From 6b11342aff0178227e2ccdd6fb00ff158fa182fc Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:24:47 -0700 Subject: [PATCH 01/11] Add streaming I/O support via FileCacheMode.streaming Introduces a new FileCacheMode.streaming that opens cloud files as Python-standard BufferedIOBase/TextIOWrapper objects backed by provider range-requests and multipart/resumable uploads, without requiring a local cache copy. New public API: - CloudBufferedIO: BufferedIOBase wrapper for binary streaming - CloudTextIO: TextIOWrapper for text streaming - FileCacheMode.streaming enum value Provider implementations: - S3: multipart upload with 5 MiB minimum non-final part size; _put_empty_object for zero-byte files - GCS: blob.open("wb") resumable write stream; _put_empty_object - Azure Blob: block-blob stage/commit pipeline; _put_empty_object - HTTP(S): single-PUT via self.opener (honours SSL context) - LocalClient: _LocalWriteStream + UUID-keyed multipart buffers for concurrent-write safety Correctness fixes included: - _CloudStorageRaw.close() propagates finalize errors via try/finally - seekable() returns False for write-only streams - Exclusive-create ("x") mode checked against cloud existence - Append/update modes ("a", "+") fall back to file-cache path - Client streaming methods default to NotImplementedError (not @abstractmethod) so custom Client subclasses don't require re-implementation - HTTP _put_data uses self.opener for SSL-aware PUT requests Co-Authored-By: Claude Sonnet 4.6 --- HISTORY.md | 7 + cloudpathlib/__init__.py | 3 + cloudpathlib/azure/__init__.py | 1 + cloudpathlib/azure/azblobclient.py | 88 ++ cloudpathlib/azure/azure_io.py | 69 + cloudpathlib/client.py | 97 ++ cloudpathlib/cloud_io.py | 468 +++++++ cloudpathlib/cloudpath.py | 91 +- cloudpathlib/enums.py | 3 + cloudpathlib/gs/__init__.py | 1 + cloudpathlib/gs/gs_io.py | 62 + cloudpathlib/gs/gsclient.py | 53 + cloudpathlib/http/__init__.py | 1 + cloudpathlib/http/http_io.py | 57 + cloudpathlib/http/httpclient.py | 90 ++ cloudpathlib/local/implementations/azure.py | 4 + cloudpathlib/local/implementations/gs.py | 4 + cloudpathlib/local/implementations/s3.py | 4 + cloudpathlib/local/localclient.py | 103 ++ cloudpathlib/s3/__init__.py | 1 + cloudpathlib/s3/s3_io.py | 106 ++ cloudpathlib/s3/s3client.py | 105 ++ docs/docs/caching.ipynb | 72 +- docs/docs/streaming_io.md | 747 ++++++++++ tests/conftest.py | 97 +- tests/http_fixtures.py | 54 +- tests/mock_clients/mock_azureblob.py | 52 +- tests/mock_clients/mock_gs.py | 72 + tests/mock_clients/mock_s3.py | 127 ++ tests/test_client.py | 11 +- tests/test_cloud_io.py | 1393 +++++++++++++++++++ tests/test_cloudpath_instantiation.py | 4 +- 32 files changed, 4006 insertions(+), 41 deletions(-) create mode 100644 cloudpathlib/azure/azure_io.py create mode 100644 cloudpathlib/cloud_io.py create mode 100644 cloudpathlib/gs/gs_io.py create mode 100644 cloudpathlib/http/http_io.py create mode 100644 cloudpathlib/s3/s3_io.py create mode 100644 docs/docs/streaming_io.md create mode 100644 tests/test_cloud_io.py diff --git a/HISTORY.md b/HISTORY.md index 64e08d5b..1d95a873 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,13 @@ - Added `AGENTS.md` with repository-specific guidance for coding agents covering contributor workflow, compatibility expectations, test rig and mock usage, live backend validation, and PR hygiene. +- Added streaming I/O support for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS via `FileCacheMode.streaming`. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) + - Added `FileCacheMode.streaming` enum value to enable direct streaming I/O without local caching. + - Added `CloudBufferedIO` class implementing `io.BufferedIOBase` for binary streaming operations. + - Added `CloudTextIO` class implementing `io.TextIOBase` for text streaming operations. + - Added provider-specific raw I/O implementations: `_S3StorageRaw`, `_AzureBlobStorageRaw`, `_GSStorageRaw`, `_HttpStorageRaw`. + - Added `register_raw_io_class` decorator for registering streaming I/O implementations. + - Added `buffer_size` parameter to `CloudPath.open()` for controlling streaming buffer size. - Fixed mypy 2.x type errors in `Client` and `CloudPath` that caused CI lint failures (Issue [#563](https://github.com/drivendataorg/cloudpathlib/issues/563), PR [#566](https://github.com/drivendataorg/cloudpathlib/pull/566)) - Changed `S3Client._get_metadata` to read object metadata with `HeadObject` instead of `GetObject`, so `stat`, `etag`, and `size` no longer open the object body. Also fixes a `KeyError` on `ContentLength` against S3-compatible gateways that drop `Content-Length` from `GetObject` responses. (Issue [#564](https://github.com/drivendataorg/cloudpathlib/issues/564), PR [#565](https://github.com/drivendataorg/cloudpathlib/pull/565)) - Added a `lazy` keyword argument to `CloudPath.walk`. By default (`lazy=False`) the existing fast behavior is preserved: the whole subtree is fetched up front with a single recursive listing. Passing `lazy=True` lists each directory on demand so that, when `top_down=True`, callers can prune subdirectories by modifying `dirnames` in-place (à la `os.walk` / `Path.walk`) to skip fetching the contents of those subtrees entirely — dramatically reducing API calls for large, sparsely-traversed trees. (Issue [#518](https://github.com/drivendataorg/cloudpathlib/issues/518)) diff --git a/cloudpathlib/__init__.py b/cloudpathlib/__init__.py index 37a3b764..cc23e215 100644 --- a/cloudpathlib/__init__.py +++ b/cloudpathlib/__init__.py @@ -4,6 +4,7 @@ from .anypath import AnyPath from .azure.azblobclient import AzureBlobClient from .azure.azblobpath import AzureBlobPath +from .cloud_io import CloudBufferedIO, CloudTextIO from .cloudpath import CloudPath, implementation_registry from .patches import patch_open, patch_os_functions, patch_glob, patch_all_builtins from .gs.gsclient import GSClient @@ -26,7 +27,9 @@ "AnyPath", "AzureBlobClient", "AzureBlobPath", + "CloudBufferedIO", "CloudPath", + "CloudTextIO", "implementation_registry", "GSClient", "GSPath", diff --git a/cloudpathlib/azure/__init__.py b/cloudpathlib/azure/__init__.py index e29ada6a..1647fe96 100644 --- a/cloudpathlib/azure/__init__.py +++ b/cloudpathlib/azure/__init__.py @@ -1,5 +1,6 @@ from .azblobclient import AzureBlobClient from .azblobpath import AzureBlobPath +from .azure_io import _AzureBlobStorageRaw # noqa: F401 - imported for registration __all__ = [ "AzureBlobClient", diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 60bd01d3..f4618632 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -497,6 +497,94 @@ def _generate_presigned_url( url = f"{self._get_public_url(cloud_path)}?{sas_token}" return url + # ====================== STREAMING I/O METHODS ====================== + + def _range_download(self, cloud_path: AzureBlobPath, start: int, end: int) -> bytes: + """Download a byte range from Azure Blob Storage.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + try: + length = end - start + 1 + downloader = blob_client.download_blob(offset=start, length=length) + return downloader.readall() + except ResourceNotFoundError: + raise FileNotFoundError(f"Azure blob not found: {cloud_path}") + except HttpResponseError as e: + if (e.error and e.error.code == "InvalidRange") or e.status_code == 416: + return b"" + raise + + def _get_content_length(self, cloud_path: AzureBlobPath) -> int: + """Get the size of an Azure blob.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + try: + properties = blob_client.get_blob_properties() + return properties.size + except ResourceNotFoundError: + raise FileNotFoundError(f"Azure blob not found: {cloud_path}") + + def _initiate_multipart_upload(self, cloud_path: AzureBlobPath) -> str: + """Start an Azure block blob upload. + + Azure doesn't need explicit initialization; return empty string. + """ + return "" + + def _upload_part( + self, cloud_path: AzureBlobPath, upload_id: str, part_number: int, data: bytes + ) -> dict: + """Upload a block in an Azure block blob upload.""" + import base64 + + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + # Azure uses base64-encoded block IDs + block_id = base64.b64encode(f"block-{part_number:06d}".encode()).decode() + blob_client.stage_block(block_id=block_id, data=data, length=len(data)) + return {"block_id": block_id} + + def _complete_multipart_upload( + self, cloud_path: AzureBlobPath, upload_id: str, parts: list + ) -> None: + """Commit an Azure block blob upload, threading content-type.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + block_ids = [part["block_id"] for part in parts] + content_settings = None + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_type or content_encoding: + content_settings = ContentSettings( + content_type=content_type, content_encoding=content_encoding + ) + blob_client.commit_block_list(block_ids, content_settings=content_settings) + + def _abort_multipart_upload(self, cloud_path: AzureBlobPath, upload_id: str) -> None: + """Abort an Azure block blob upload. + + Azure automatically expires uncommitted blocks; nothing explicit to do. + """ + pass + + def _put_empty_object(self, cloud_path: AzureBlobPath) -> None: + """Upload a zero-byte Azure blob, threading content-type.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + content_settings = None + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_type or content_encoding: + content_settings = ContentSettings( + content_type=content_type, content_encoding=content_encoding + ) + blob_client.upload_blob(b"", overwrite=True, content_settings=content_settings) + def _hns_rmtree(data_lake_client, container, directory): """Stateless implementation so can be used in test suite cleanup as well. diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py new file mode 100644 index 00000000..213b8286 --- /dev/null +++ b/cloudpathlib/azure/azure_io.py @@ -0,0 +1,69 @@ +""" +Azure Blob Storage-specific streaming I/O implementations. + +Provides efficient streaming I/O for Azure using range requests and block uploads. +""" + +from typing import Optional, Dict, Any + +from ..cloud_io import _CloudStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("azure") +class _AzureBlobStorageRaw(_CloudStorageRaw): + """ + Azure Blob Storage-specific raw I/O adapter. + + Implements efficient range-based reads and block blob uploads for Azure. + Each block is staged independently (true streaming) and committed on finalize. + """ + + def __init__(self, client, cloud_path, mode: str = "rb"): + super().__init__(client, cloud_path, mode) + + # Block blob upload state + self._upload_id: str = "" # Azure doesn't use upload IDs + self._parts: list = [] + + def _range_get(self, start: int, end: int) -> bytes: + return self._client._range_download(self._cloud_path, start, end) + + def _get_size(self) -> int: + return self._client._get_content_length(self._cloud_path) + + def _is_eof_error(self, error: Exception) -> bool: + error_str = str(error) + if "InvalidRange" in error_str or "out of range" in error_str.lower(): + return True + if hasattr(error, "error_code") and error.error_code == "InvalidRange": + return True + return False + + # ---- Write support (Azure block blob upload) ---- + + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: + if not data: + return + + if not self._upload_id: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + + part_number = len(self._parts) + 1 + part_info = self._client._upload_part(self._cloud_path, self._upload_id, part_number, data) + self._parts.append(part_info) + + def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + if not self._parts: + # No blocks staged — create an empty blob directly + self._client._put_empty_object(self._cloud_path) + return + + try: + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) + finally: + self._upload_id = "" + self._parts = [] + + def close(self) -> None: + super().close() diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index d1c36fd5..37306ce5 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -184,3 +184,100 @@ def _generate_presigned_url( self, cloud_path: BoundedCloudPath, expire_seconds: int = 60 * 60 ) -> str: pass + + # ====================== STREAMING I/O METHODS ====================== + # Methods to support efficient streaming without local caching. + # Default implementations raise NotImplementedError so that existing Client + # subclasses that don't implement streaming still instantiate normally. + # Providers override as needed. + + def _range_download(self, cloud_path: BoundedCloudPath, start: int, end: int) -> bytes: + """Download a byte range from cloud storage. + + Args: + cloud_path: Path to download from + start: Start byte position (inclusive) + end: End byte position (inclusive) + + Returns: + Bytes in the requested range + + Raises: + FileNotFoundError: If object doesn't exist + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_range_download). " + "Implement this method or use a non-streaming file_cache_mode." + ) + + def _get_content_length(self, cloud_path: BoundedCloudPath) -> int: + """Get the size of an object without downloading it. + + Args: + cloud_path: Path to query + + Returns: + Size in bytes + + Raises: + FileNotFoundError: If object doesn't exist + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_get_content_length)." + ) + + def _initiate_multipart_upload(self, cloud_path: BoundedCloudPath) -> str: + """Start a multipart/chunked upload session. + + Args: + cloud_path: Destination path + + Returns: + Upload session ID/handle (provider-specific, may be empty string) + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_initiate_multipart_upload)." + ) + + def _upload_part( + self, cloud_path: BoundedCloudPath, upload_id: str, part_number: int, data: bytes + ) -> dict: + """Upload a single part/chunk in a multipart upload. + + Args: + cloud_path: Destination path + upload_id: Upload session ID from _initiate_multipart_upload + part_number: Sequential part number (1-indexed) + data: Bytes to upload + + Returns: + Provider-specific metadata needed for finalization + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_upload_part)." + ) + + def _complete_multipart_upload( + self, cloud_path: BoundedCloudPath, upload_id: str, parts: list + ) -> None: + """Finalize a multipart upload. + + Args: + cloud_path: Destination path + upload_id: Upload session ID + parts: List of part metadata from _upload_part calls + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_complete_multipart_upload)." + ) + + def _abort_multipart_upload(self, cloud_path: BoundedCloudPath, upload_id: str) -> None: + """Cancel a multipart upload and clean up. + + Args: + cloud_path: Destination path + upload_id: Upload session ID + """ + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_abort_multipart_upload)." + ) diff --git a/cloudpathlib/cloud_io.py b/cloudpathlib/cloud_io.py new file mode 100644 index 00000000..ad63c0cd --- /dev/null +++ b/cloudpathlib/cloud_io.py @@ -0,0 +1,468 @@ +""" +Cloud storage streaming I/O implementations. + +Provides BufferedIOBase and TextIOBase compliant file-like objects for cloud storage +that support efficient streaming with range requests and multipart uploads, without +requiring full local caching. +""" + +import io +from abc import abstractmethod +from typing import Optional, Any, Type, Union, Dict + +# ============================================================================ +# Base Raw I/O Adapter (internal) +# ============================================================================ + + +class _CloudStorageRaw(io.RawIOBase): + """ + Internal raw I/O adapter for cloud storage objects. + + Implements efficient range-based reads using cloud provider APIs. + Not exposed to users - internal implementation detail. + """ + + def __init__( + self, + client: Any, + cloud_path: Any, + mode: str = "rb", + ): + """ + Initialize raw cloud storage adapter. + + Args: + client: Cloud provider client (S3Client, AzureBlobClient, etc.) + cloud_path: CloudPath instance + mode: File mode (currently only read modes supported in base) + """ + super().__init__() + self._client = client + self._cloud_path = cloud_path + self._mode = mode + self._pos = 0 + self._size: Optional[int] = None + self._closed = False + + def readable(self) -> bool: + """Return whether object was opened for reading.""" + return "r" in self._mode or "+" in self._mode + + def writable(self) -> bool: + """Return whether object was opened for writing.""" + return "w" in self._mode or "a" in self._mode or "+" in self._mode or "x" in self._mode + + def seekable(self) -> bool: + """Return whether object supports random access. + + Streaming writes are sequential-only; seeking is only valid for readable streams. + """ + return self.readable() + + def readinto(self, b: bytearray) -> int: # type: ignore[override] + """ + Read bytes into a pre-allocated buffer. + + Args: + b: Buffer to read data into + + Returns: + Number of bytes read (0 at EOF) + """ + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.readable(): + raise io.UnsupportedOperation("not readable") + if len(b) == 0: + return 0 + + # Calculate range to fetch + start = self._pos + end = start + len(b) - 1 + + # Clamp end to file size if known (prevents 416 errors) + if self._size is None: + try: + self._size = self._get_size() + except Exception: + # If we can't get size, try the request anyway + pass + + if self._size is not None and end >= self._size: + # Clamp to last valid byte + end = self._size - 1 + if start >= self._size: + # Already at EOF + return 0 + + # Fetch data from cloud storage + try: + data = self._range_get(start, end) + except Exception as e: + # If we get an error reading beyond EOF, treat as EOF + if self._is_eof_error(e): + return 0 + raise + + # Copy data into buffer + n = len(data) + if n == 0: + return 0 + + # Ensure we don't write more than the buffer can hold + n = min(n, len(b)) + + try: + b[:n] = data[:n] + except (ValueError, TypeError): + # Fall back to memoryview-based copy for non-contiguous buffer shapes + memoryview(b).cast("B")[:n] = data[:n] + + self._pos += n + return n + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + """ + Change stream position. + + Args: + offset: Offset in bytes + whence: Position to seek from (SEEK_SET, SEEK_CUR, SEEK_END) + + Returns: + New absolute position + """ + if self._closed: + raise ValueError("I/O operation on closed file") + + if whence == io.SEEK_SET: + new_pos = offset + elif whence == io.SEEK_CUR: + new_pos = self._pos + offset + elif whence == io.SEEK_END: + if self._size is None: + self._size = self._get_size() + if self._size is None: + raise OSError("Unable to determine file size for SEEK_END") + new_pos = self._size + offset + else: + raise ValueError( + f"invalid whence ({whence}, should be {io.SEEK_SET}, " + f"{io.SEEK_CUR}, or {io.SEEK_END})" + ) + + if new_pos < 0: + raise ValueError("negative seek position") + + self._pos = new_pos + return self._pos + + def tell(self) -> int: + """Return current stream position.""" + if self._closed: + raise ValueError("I/O operation on closed file") + return self._pos + + def write(self, b: bytes) -> int: # type: ignore[override] + """ + Write bytes to the stream. + + This method is required by RawIOBase for writable streams. + The actual implementation is delegated to subclasses via _upload_chunk. + + Args: + b: Bytes to write + + Returns: + Number of bytes written + """ + if not self.writable(): + raise io.UnsupportedOperation("not writable") + + # Delegate to subclass implementation + # Note: Don't check _closed here because BufferedWriter may call write() during close/flush + self._upload_chunk(bytes(b), None) + return len(b) + + def close(self) -> None: + """Close the file.""" + if self._closed: + return + + # Mark as closed FIRST to prevent recursive double-finalize + self._closed = True + + try: + if self.writable(): + self._finalize_upload(None) + finally: + # Always call parent close() to set the stdlib closed state + super().close() + + @abstractmethod + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]]) -> None: + """ + Upload a chunk of data. + + Args: + data: Bytes to upload + upload_state: Upload state dictionary (for multipart uploads) + """ + pass + + @abstractmethod + def _finalize_upload(self, upload_state: Optional[Dict[str, Any]]) -> None: + """ + Finalize the upload process. + + Args: + upload_state: Upload state dictionary (for multipart uploads) + """ + pass + + # Abstract methods to be implemented by subclasses + + @abstractmethod + def _range_get(self, start: int, end: int) -> bytes: + """ + Fetch a byte range from cloud storage. + + Args: + start: Start byte position (inclusive) + end: End byte position (inclusive) + + Returns: + Bytes in the requested range + """ + pass + + @abstractmethod + def _get_size(self) -> int: + """ + Get the total size of the cloud object. + + Returns: + Size in bytes + """ + pass + + def _is_eof_error(self, error: Exception) -> bool: + """ + Check if an error indicates EOF/out of range. + + Override in subclasses for provider-specific error handling. + """ + return False + + +# ============================================================================ +# Public Buffered Binary I/O +# ============================================================================ + + +class CloudBufferedIO(io.BufferedIOBase): + """ + Buffered binary file-like object for cloud storage. + + Wraps a raw cloud storage adapter with Python's standard buffered I/O classes + (BufferedReader, BufferedWriter, or BufferedRandom) based on the mode. + + Example: + >>> from cloudpathlib import S3Client + >>> client = S3Client() + >>> with CloudBufferedIO(client, "s3://bucket/file.bin", mode="rb") as f: + ... data = f.read(1024) + """ + + def __init__( + self, + raw_io_class: Type[_CloudStorageRaw], + client: Any, + cloud_path: Any, + mode: str = "rb", + buffer_size: int = 64 * 1024, + ): + """ + Initialize cloud buffered I/O. + + Args: + raw_io_class: The raw I/O class to use for this provider + client: Cloud provider client instance + cloud_path: CloudPath instance + mode: File mode ('rb', 'wb', 'ab', 'r+b', 'w+b', 'a+b', 'xb') + buffer_size: Size of read/write buffer in bytes (default 64 KiB) + """ + if "b" not in mode: + raise ValueError("CloudBufferedIO requires binary mode (must include 'b')") + + # Create raw adapter using provided class + raw = raw_io_class(client, cloud_path, mode) + + # Choose appropriate buffered class based on mode + if "+" in mode: + # Read and write (e.g., 'r+b', 'w+b') + self._buffer: Union[io.BufferedReader, io.BufferedWriter, io.BufferedRandom] = io.BufferedRandom(raw, buffer_size=buffer_size) # type: ignore[arg-type] + elif "r" in mode: + # Read only (e.g., 'rb') + self._buffer = io.BufferedReader(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] + else: + # Write only (e.g., 'wb', 'ab', 'xb') + self._buffer = io.BufferedWriter(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] + + # Store additional attributes + self._cloud_path = cloud_path + self._mode = mode + self._buffer_size_val = buffer_size + + @property + def name(self) -> str: + """File name (the cloud URL).""" + return str(self._cloud_path) + + @property + def mode(self) -> str: + """File mode.""" + return self._mode + + @property + def _buffer_size(self) -> int: + """Buffer size for compatibility with tests.""" + return self._buffer_size_val + + # Delegate all I/O methods to the internal buffer + def read(self, size: Optional[int] = -1) -> bytes: # type: ignore[override] + return self._buffer.read(size) + + def read1(self, size: int = -1) -> bytes: + return self._buffer.read1(size) # type: ignore[attr-defined] + + def readinto(self, b): + return self._buffer.readinto(b) + + def readinto1(self, b): + return self._buffer.readinto1(b) # type: ignore[attr-defined] + + def write(self, b): + return self._buffer.write(b) + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + return self._buffer.seek(offset, whence) + + def tell(self) -> int: + return self._buffer.tell() + + def flush(self): + return self._buffer.flush() + + def close(self): + # Delegate entirely to the stdlib buffer: flush → raw.close(). + # _CloudStorageRaw.close() guards against double-finalize via _closed, + # so calling self._buffer.close() is safe and lets exceptions propagate. + if hasattr(self, "_buffer") and not self._buffer.closed: + self._buffer.close() + + def readable(self) -> bool: + return self._buffer.readable() + + def writable(self) -> bool: + return self._buffer.writable() + + def seekable(self) -> bool: + return self._buffer.seekable() + + @property + def closed(self) -> bool: + return self._buffer.closed + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +# ============================================================================ +# Public Text I/O +# ============================================================================ + + +class CloudTextIO(io.TextIOWrapper): + """ + Text file-like object for cloud storage. + + Implements TextIOBase for seamless integration with standard Python I/O + and third-party libraries. Handles encoding/decoding and newline translation. + + Example: + >>> from cloudpathlib import S3Client + >>> client = S3Client() + >>> with CloudTextIO(client, "s3://bucket/file.txt", mode="rt") as f: + ... text = f.read() + """ + + def __init__( + self, + raw_io_class: Type[_CloudStorageRaw], + client: Any, + cloud_path: Any, + mode: str = "rt", + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + buffer_size: int = 64 * 1024, + ): + """ + Initialize cloud text I/O. + + Args: + raw_io_class: The raw I/O class to use for this provider + client: Cloud provider client instance + cloud_path: CloudPath instance + mode: File mode ('rt', 'wt', 'at', 'r+t', 'w+t', 'a+t', 'xt', or same without 't') + encoding: Text encoding (default: utf-8) + errors: Error handling strategy (default: strict) + newline: Newline handling (None, '', '\\n', '\\r', '\\r\\n') + buffer_size: Size of buffer in bytes + """ + if "b" in mode: + raise ValueError("CloudTextIO requires text mode (no 'b' in mode)") + + # Ensure mode has 't' or is text mode + if "t" not in mode and "r" in mode: + binary_mode = mode.replace("r", "rb", 1) + elif "t" not in mode and "w" in mode: + binary_mode = mode.replace("w", "wb", 1) + elif "t" not in mode and "a" in mode: + binary_mode = mode.replace("a", "ab", 1) + elif "t" not in mode and "x" in mode: + binary_mode = mode.replace("x", "xb", 1) + else: + binary_mode = mode.replace("t", "b") + + # Create underlying buffered I/O + buffered = CloudBufferedIO( + raw_io_class, client, cloud_path, mode=binary_mode, buffer_size=buffer_size + ) + + # Initialize TextIOWrapper with the buffered stream + super().__init__( + buffered, + encoding=encoding or "utf-8", + errors=errors, + newline=newline, + ) + + # Store additional attributes + self._cloud_path = cloud_path + self._mode = mode + + @property + def name(self) -> str: + """File name (the cloud URL).""" + return str(self._cloud_path) + + @property + def mode(self) -> str: + """File mode.""" + return self._mode diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 01306fa7..aeada7ec 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -108,12 +108,14 @@ def _make_selector(pattern_parts, _flavour, case_sensitive=True): # noqa: F811 class CloudImplementation: - name: str + name: Optional[str] = None dependencies_loaded: bool = True _client_class: Type["Client"] _path_class: Type["CloudPath"] + _raw_io_class: Optional[Type] = None def validate_completeness(self) -> None: + # raw_io_class is optional; streaming raises NotImplementedError when absent expected = ["client_class", "path_class"] missing = [cls for cls in expected if getattr(self, f"_{cls}") is None] if missing: @@ -121,9 +123,11 @@ def validate_completeness(self) -> None: f"Implementation is missing registered components: {missing}" ) if not self.dependencies_loaded: + # Use name if available, otherwise fall back to client class name + pkg_name = self.name if self.name else self._client_class.__name__.lower() raise MissingDependenciesError( f"Missing dependencies for {self._client_class.__name__}. You can install them " - f"with 'pip install cloudpathlib[{self.name}]'." + f"with 'pip install cloudpathlib[{pkg_name}]'." ) @property @@ -136,6 +140,11 @@ def path_class(self) -> Type["CloudPath"]: self.validate_completeness() return self._path_class + @property + def raw_io_class(self) -> Optional[Type]: + self.validate_completeness() + return self._raw_io_class + implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation) @@ -155,6 +164,23 @@ def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: return decorator +def register_raw_io_class(key: str) -> Callable[[Type[T]], Type[T]]: + """Decorator to register a raw I/O class for a cloud provider. + + Args: + key: The cloud provider key (e.g., 's3', 'azure', 'gs') + + Returns: + Decorator function + """ + + def decorator(cls: Type[T]) -> Type[T]: + implementation_registry[key]._raw_io_class = cls + return cls + + return decorator + + class CloudPathMeta(abc.ABCMeta): @overload def __call__( @@ -355,6 +381,14 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, type(self)) and str(self) == str(other) def __fspath__(self) -> str: + # Check if streaming mode is enabled + if self.client.file_cache_mode == FileCacheMode.streaming: + raise CloudPathNotImplementedError( + "fspath is not available in streaming mode. " + "Streaming mode does not create cached files on disk. " + "Use CloudPath.open() to read/write data directly." + ) + if self.is_file(): self._refresh_cache() return str(self._local) @@ -763,6 +797,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, # extra kwarg not in pathlib force_overwrite_to_cloud: Optional[bool] = None, # extra kwarg not in pathlib + buffer_size: Optional[int] = None, # extra kwarg for streaming mode ) -> "IO[Any]": # if trying to call open on a directory that exists exists_on_cloud = self.exists() @@ -777,10 +812,58 @@ def open( f"File opened for read or append, but it does not exist on cloud: {self}" ) - if mode == "x" and self.exists(): + if "x" in mode and self.exists(): raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.") - # TODO: consider streaming from client rather than DLing entire file to cache + # Use streaming I/O if file_cache_mode is streaming AND the mode is supported. + # Append (a) and update/random (+) modes cannot be done correctly as pure streaming + # over object storage; fall through to the cached path for correct semantics. + _streaming_unsupported = any(m in mode for m in ("a", "+")) + if self.client.file_cache_mode == FileCacheMode.streaming and not _streaming_unsupported: + # Import here to keep it localized to streaming functionality + from .cloud_io import CloudBufferedIO, CloudTextIO + + # Get the raw IO class from the cloud implementation + raw_io_class = self._cloud_meta.raw_io_class + if raw_io_class is None: + raise NotImplementedError( + f"Streaming I/O is not implemented for {self._cloud_meta.name}" + ) + + # Calculate buffer size from buffering or buffer_size parameter + if buffer_size is None: + if buffering == 0: + # Unbuffered binary mode + buffer_size = 1 # Minimal buffering + if "b" not in mode: + mode += "b" # Force binary mode for unbuffered + elif buffering > 0: + buffer_size = buffering + else: + buffer_size = 64 * 1024 # Default 64 KiB + + # Return appropriate streaming I/O object + if "b" in mode: + return CloudBufferedIO( # type: ignore[return-value] + raw_io_class=raw_io_class, + client=self.client, + cloud_path=self, + mode=mode, + buffer_size=buffer_size, + ) + else: + return CloudTextIO( # type: ignore[return-value] + raw_io_class=raw_io_class, + client=self.client, + cloud_path=self, + mode=mode, + encoding=encoding, + errors=errors, + newline=newline, + buffer_size=buffer_size, + ) + + # Standard cached mode self._refresh_cache(force_overwrite_from_cloud=force_overwrite_from_cloud) # create any directories that may be needed if the file is new diff --git a/cloudpathlib/enums.py b/cloudpathlib/enums.py index 55260c17..b34fd36a 100644 --- a/cloudpathlib/enums.py +++ b/cloudpathlib/enums.py @@ -15,6 +15,8 @@ class FileCacheMode(str, Enum): called by Python garbage collection. close_file (str): Cache for a `CloudPath` file is removed as soon as the file is closed. Note: you must use `CloudPath.open` whenever opening the file for this method to function. + streaming (str): No caching is used; files are streamed directly from/to cloud storage using + efficient range requests and multipart uploads. Modes can be set by passing them to the Client or by setting the `CLOUDPATHLIB_FILE_CACHE_MODE` environment variable. @@ -26,6 +28,7 @@ class FileCacheMode(str, Enum): tmp_dir = "tmp_dir" # DEFAULT: handled by deleting client, Python, or OS (usually on machine restart) cloudpath_object = "cloudpath_object" # __del__ called on the CloudPath object close_file = "close_file" # cache is cleared when file is closed + streaming = "streaming" # no caching, direct streaming I/O @classmethod def from_environment(cls) -> Optional["FileCacheMode"]: diff --git a/cloudpathlib/gs/__init__.py b/cloudpathlib/gs/__init__.py index d68a41c4..d4c5e24a 100644 --- a/cloudpathlib/gs/__init__.py +++ b/cloudpathlib/gs/__init__.py @@ -1,5 +1,6 @@ from .gsclient import GSClient from .gspath import GSPath +from .gs_io import _GSStorageRaw # noqa: F401 - imported for registration __all__ = [ "GSClient", diff --git a/cloudpathlib/gs/gs_io.py b/cloudpathlib/gs/gs_io.py new file mode 100644 index 00000000..2cd09d9b --- /dev/null +++ b/cloudpathlib/gs/gs_io.py @@ -0,0 +1,62 @@ +""" +Google Cloud Storage-specific streaming I/O implementations. + +Provides efficient streaming I/O for GCS using range requests and resumable uploads. +Upload state is held on the raw adapter instance (not the shared client) so concurrent +writers to the same client cannot collide. +""" + +from typing import Any, Dict, Optional + +from ..cloud_io import _CloudStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("gs") +class _GSStorageRaw(_CloudStorageRaw): + """ + GCS-specific raw I/O adapter. + + Implements efficient range-based reads and resumable uploads for GCS. + Write state (_writer) lives on this adapter instance, not on the client, + so concurrent writes to different paths on the same client are safe. + """ + + def __init__(self, client, cloud_path, mode: str = "rb"): + super().__init__(client, cloud_path, mode) + # Open write stream (returned by client._open_write_stream); None until first write + self._writer: Optional[Any] = None + + def _range_get(self, start: int, end: int) -> bytes: + return self._client._range_download(self._cloud_path, start, end) + + def _get_size(self) -> int: + return self._client._get_content_length(self._cloud_path) + + def _is_eof_error(self, error: Exception) -> bool: + error_str = str(error) + if "416" in error_str or "Requested Range Not Satisfiable" in error_str: + return True + if hasattr(error, "code") and error.code == 416: + return True + return False + + # ---- Write support (resumable upload via client._open_write_stream) ---- + + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: + if not data: + return + if self._writer is None: + self._writer = self._client._open_write_stream(self._cloud_path) + self._writer.write(data) + + def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + if self._writer is not None: + self._writer.close() + self._writer = None + else: + # No data was written — create an empty object + self._client._put_empty_object(self._cloud_path) + + def close(self) -> None: + super().close() diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index 96705ece..5510e1ad 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -15,12 +15,14 @@ from google.auth.credentials import Credentials from google.api_core.retry import Retry + from google.api_core.exceptions import NotFound as GCSNotFound from google.auth import default as google_default_auth from google.auth.exceptions import DefaultCredentialsError from google.cloud.storage import Client as StorageClient except ModuleNotFoundError: implementation_registry["gs"].dependencies_loaded = False + GCSNotFound = Exception # type: ignore[misc, assignment] # fallback so name is always defined try: @@ -310,5 +312,56 @@ def _generate_presigned_url(self, cloud_path: GSPath, expire_seconds: int = 60 * ) return url + # ====================== STREAMING I/O METHODS ====================== + + def _range_download(self, cloud_path: GSPath, start: int, end: int) -> bytes: + """Download a byte range from GCS.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + try: + # GCS end is exclusive in the API, our API is inclusive + return blob.download_as_bytes(start=start, end=end + 1) + except GCSNotFound: + raise FileNotFoundError(f"GCS object not found: {cloud_path}") + except Exception as e: + error_str = str(e) + if "416" in error_str or "Requested Range Not Satisfiable" in error_str: + return b"" + if hasattr(e, "code") and e.code == 416: + return b"" + raise + + def _get_content_length(self, cloud_path: GSPath) -> int: + """Get the size of a GCS object.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + try: + blob.reload() + return blob.size + except GCSNotFound: + raise FileNotFoundError(f"GCS object not found: {cloud_path}") + + def _open_write_stream(self, cloud_path: GSPath): + """Open a GCS resumable upload stream. + + Returns a file-like writer. Data written to it streams incrementally + to GCS rather than being buffered in memory. The caller must close() + the writer to finalize the upload. + """ + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + kwargs: Dict[str, Any] = {} + if self.content_type_method is not None: + content_type, _ = self.content_type_method(str(cloud_path)) + if content_type is not None: + kwargs["content_type"] = content_type + # blob_kwargs may carry timeout/retry; pass through where blob.open accepts them + for k in ("timeout", "retry"): + if k in self.blob_kwargs: + kwargs[k] = self.blob_kwargs[k] + return blob.open("wb", **kwargs) + + def _put_empty_object(self, cloud_path: GSPath) -> None: + """Upload a zero-byte GCS object.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + blob.upload_from_string(b"", **self.blob_kwargs) + GSClient.GSPath = GSClient.CloudPath # type: ignore diff --git a/cloudpathlib/http/__init__.py b/cloudpathlib/http/__init__.py index ccf7452e..00085d9c 100644 --- a/cloudpathlib/http/__init__.py +++ b/cloudpathlib/http/__init__.py @@ -1,5 +1,6 @@ from .httpclient import HttpClient, HttpsClient from .httppath import HttpPath, HttpsPath +from .http_io import _HttpStorageRaw # noqa: F401 __all__ = [ "HttpClient", diff --git a/cloudpathlib/http/http_io.py b/cloudpathlib/http/http_io.py new file mode 100644 index 00000000..6dc1fb5d --- /dev/null +++ b/cloudpathlib/http/http_io.py @@ -0,0 +1,57 @@ +""" +HTTP-specific streaming I/O implementations. + +Provides streaming I/O for HTTP/HTTPS using range requests and single-PUT uploads. +""" + +from typing import Optional, Dict, Any + +from ..cloud_io import _CloudStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("http") +@register_raw_io_class("https") +class _HttpStorageRaw(_CloudStorageRaw): + """ + HTTP-specific raw I/O adapter. + + Implements efficient range-based reads and single-PUT uploads for HTTP/HTTPS. + Write operations require the server to support PUT requests. + Data is buffered in _upload_buffer on the adapter instance (not on the client) + and flushed as a single PUT on close. + """ + + def __init__(self, client, cloud_path, mode: str = "rb"): + super().__init__(client, cloud_path, mode) + self._upload_buffer: list = [] + + def _range_get(self, start: int, end: int) -> bytes: + return self._client._range_download(self._cloud_path, start, end) + + def _get_size(self) -> int: + return self._client._get_content_length(self._cloud_path) + + def _is_eof_error(self, error: Exception) -> bool: + error_str = str(error).lower() + return ( + "416" in error_str + or "requested range not satisfiable" in error_str + or "invalid range" in error_str + ) + + # ---- Write support (single PUT on finalize) ---- + + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: + if not data: + return + self._upload_buffer.append(data) + + def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + # Concatenate buffered chunks (may be empty for an empty write) + complete_data = b"".join(self._upload_buffer) + self._upload_buffer.clear() + self._client._put_data(self._cloud_path, complete_data) + + def close(self) -> None: + super().close() diff --git a/cloudpathlib/http/httpclient.py b/cloudpathlib/http/httpclient.py index a67690ea..379a169b 100644 --- a/cloudpathlib/http/httpclient.py +++ b/cloudpathlib/http/httpclient.py @@ -203,6 +203,96 @@ def request( # the connection is closed when we exit the context manager. return response, response.read() + # ====================== STREAMING I/O METHODS ====================== + + def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes: + """Download a byte range from HTTP. + + Verifies the response is 206 Partial Content. If the server returns 200 + (ignoring the Range header), slices the full body locally so callers + always receive exactly the requested bytes. + """ + headers = {"Range": f"bytes={start}-{end}"} + request = urllib.request.Request(str(cloud_path), headers=headers) + try: + with self.opener.open(request) as response: + status = response.status + data = response.read() + if status == 206: + return data + elif status == 200: + # Server ignored the Range header; slice locally + return data[start : end + 1] + else: + raise OSError(f"Unexpected status {status} for range request on {cloud_path}") + except urllib.error.HTTPError as e: + if e.code == 404: + raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") + elif e.code == 416: # Range not satisfiable + return b"" + raise + + def _get_content_length(self, cloud_path: "HttpPath") -> int: + """Get the size of an HTTP resource.""" + request = urllib.request.Request(str(cloud_path), method="HEAD") + try: + with self.opener.open(request) as response: + content_length = response.headers.get("Content-Length") + if content_length: + return int(content_length) + raise ValueError(f"HTTP resource does not provide Content-Length: {cloud_path}") + except urllib.error.HTTPError as e: + if e.code == 404: + raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") + raise + + def _initiate_multipart_upload(self, cloud_path: "HttpPath") -> str: + """HTTP uploads are single-shot PUT; no session needed.""" + return "" + + def _upload_part( + self, cloud_path: "HttpPath", upload_id: str, part_number: int, data: bytes + ) -> dict: + """HTTP does not support true multipart; use _put_data for a single PUT.""" + raise NotImplementedError( + "HTTP uses a single PUT for uploads; multipart is not supported. " + "Use _put_data instead." + ) + + def _complete_multipart_upload( + self, cloud_path: "HttpPath", upload_id: str, parts: list + ) -> None: + """HTTP does not support true multipart; use _put_data for a single PUT.""" + raise NotImplementedError( + "HTTP uses a single PUT for uploads; multipart is not supported." + ) + + def _abort_multipart_upload(self, cloud_path: "HttpPath", upload_id: str) -> None: + """Nothing to abort for HTTP single-PUT uploads.""" + pass + + def _put_data(self, cloud_path: "HttpPath", data: bytes) -> None: + """Upload data to HTTP server using a PUT request. + + Uses self.opener so that any SSL context or auth handlers configured on + this client are applied (important for HttpsClient with self-signed certs). + """ + url = str(cloud_path) + request = urllib.request.Request(url, data=data, method="PUT") + request.add_header("Content-Type", "application/octet-stream") + request.add_header("Content-Length", str(len(data))) + + try: + with self.opener.open(request) as response: + if response.status not in (200, 201, 204): + raise OSError( + f"HTTP PUT failed with status {response.status}: {response.reason}" + ) + except urllib.error.HTTPError as e: + if e.code == 405: # Method Not Allowed + raise NotImplementedError(f"HTTP server does not support PUT requests for {url}") + raise OSError(f"HTTP PUT failed: {e}") + HttpClient.HttpPath = HttpClient.CloudPath # type: ignore diff --git a/cloudpathlib/local/implementations/azure.py b/cloudpathlib/local/implementations/azure.py index 2b44814f..c273341b 100644 --- a/cloudpathlib/local/implementations/azure.py +++ b/cloudpathlib/local/implementations/azure.py @@ -6,6 +6,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...azure.azure_io import _AzureBlobStorageRaw # noqa: F401 + local_azure_blob_implementation = CloudImplementation() """Replacement for "azure" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -81,3 +84,4 @@ def md5(self) -> str: local_azure_blob_implementation.name = "azure" local_azure_blob_implementation._client_class = LocalAzureBlobClient local_azure_blob_implementation._path_class = LocalAzureBlobPath +local_azure_blob_implementation._raw_io_class = _AzureBlobStorageRaw diff --git a/cloudpathlib/local/implementations/gs.py b/cloudpathlib/local/implementations/gs.py index 8633ff4e..4ca78873 100644 --- a/cloudpathlib/local/implementations/gs.py +++ b/cloudpathlib/local/implementations/gs.py @@ -4,6 +4,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...gs.gs_io import _GSStorageRaw # noqa: F401 + local_gs_implementation = CloudImplementation() """Replacement for "gs" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -64,3 +67,4 @@ def md5(self) -> str: local_gs_implementation.name = "gs" local_gs_implementation._client_class = LocalGSClient local_gs_implementation._path_class = LocalGSPath +local_gs_implementation._raw_io_class = _GSStorageRaw diff --git a/cloudpathlib/local/implementations/s3.py b/cloudpathlib/local/implementations/s3.py index edc8c67e..e74c72fe 100644 --- a/cloudpathlib/local/implementations/s3.py +++ b/cloudpathlib/local/implementations/s3.py @@ -4,6 +4,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...s3.s3_io import _S3StorageRaw # noqa: F401 + local_s3_implementation = CloudImplementation() """Replacement for "s3" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -60,3 +63,4 @@ def etag(self): local_s3_implementation.name = "s3" local_s3_implementation._client_class = LocalS3Client local_s3_implementation._path_class = LocalS3Path +local_s3_implementation._raw_io_class = _S3StorageRaw diff --git a/cloudpathlib/local/localclient.py b/cloudpathlib/local/localclient.py index b3f73932..e1d541bc 100644 --- a/cloudpathlib/local/localclient.py +++ b/cloudpathlib/local/localclient.py @@ -215,6 +215,109 @@ def _generate_presigned_url( query["signature"] = "local" return urlunsplit(parts._replace(query=urlencode(query))) + # ====================== STREAMING I/O METHODS ====================== + # For local clients, streaming uses local file operations. + + def _range_download(self, cloud_path: LocalPath, start: int, end: int) -> bytes: + """Download a byte range from local storage.""" + local_path = self._cloud_path_to_local(cloud_path) + if not local_path.exists(): + raise FileNotFoundError(f"File not found: {cloud_path}") + + with open(local_path, "rb") as f: + f.seek(start) + length = end - start + 1 + return f.read(length) + + def _get_content_length(self, cloud_path: LocalPath) -> int: + """Get the size of a local file.""" + local_path = self._cloud_path_to_local(cloud_path) + if not local_path.exists(): + raise FileNotFoundError(f"File not found: {cloud_path}") + return local_path.stat().st_size + + def _initiate_multipart_upload(self, cloud_path: LocalPath) -> str: + """Return a unique upload ID so concurrent uploads don't share a buffer.""" + import uuid + + return str(uuid.uuid4()) + + def _upload_part( + self, cloud_path: LocalPath, upload_id: str, part_number: int, data: bytes + ) -> dict: + """Buffer a part keyed by upload_id (not by path) for concurrent-write safety.""" + if not hasattr(self, "_local_upload_buffers"): + self._local_upload_buffers: dict = {} + if upload_id not in self._local_upload_buffers: + self._local_upload_buffers[upload_id] = [] + self._local_upload_buffers[upload_id].append((part_number, data)) + return {"part_number": part_number} + + def _complete_multipart_upload( + self, cloud_path: LocalPath, upload_id: str, parts: list + ) -> None: + """Complete local file upload by joining all buffered parts.""" + if ( + not hasattr(self, "_local_upload_buffers") + or upload_id not in self._local_upload_buffers + ): + return + + buffer = self._local_upload_buffers.pop(upload_id, []) + buffer.sort(key=lambda x: x[0]) + complete_data = b"".join([data for _, data in buffer]) + + local_path = self._cloud_path_to_local(cloud_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.write_bytes(complete_data) + + def _abort_multipart_upload(self, cloud_path: LocalPath, upload_id: str) -> None: + """Abort local file upload by cleaning up the buffer.""" + if hasattr(self, "_local_upload_buffers"): + self._local_upload_buffers.pop(upload_id, None) + + def _open_write_stream(self, cloud_path: LocalPath) -> "_LocalWriteStream": + """Return a write stream that buffers data and writes to the local file on close.""" + local_path = self._cloud_path_to_local(cloud_path) + return _LocalWriteStream(local_path) + + def _put_empty_object(self, cloud_path: LocalPath) -> None: + """Create a zero-byte local file.""" + local_path = self._cloud_path_to_local(cloud_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.write_bytes(b"") + + +class _LocalWriteStream: + """File-like writer that accumulates bytes and flushes to a local path on close. + + Used by LocalClient._open_write_stream so that _GSStorageRaw (and other adapters + that call _open_write_stream) can work correctly against local test clients. + """ + + def __init__(self, local_path: Path) -> None: + self._local_path = local_path + self._buf: bytearray = bytearray() + self._closed: bool = False + + def write(self, data: bytes) -> int: + if self._closed: + raise ValueError("I/O operation on closed stream") + self._buf.extend(data) + return len(data) + + def close(self) -> None: + if not self._closed: + self._closed = True + self._local_path.parent.mkdir(parents=True, exist_ok=True) + self._local_path.write_bytes(bytes(self._buf)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + _temp_dirs_to_clean: List[TemporaryDirectory] = [] diff --git a/cloudpathlib/s3/__init__.py b/cloudpathlib/s3/__init__.py index 77d27176..cb71b813 100644 --- a/cloudpathlib/s3/__init__.py +++ b/cloudpathlib/s3/__init__.py @@ -1,5 +1,6 @@ from .s3client import S3Client from .s3path import S3Path +from .s3_io import _S3StorageRaw # noqa: F401 - imported for registration __all__ = [ "S3Client", diff --git a/cloudpathlib/s3/s3_io.py b/cloudpathlib/s3/s3_io.py new file mode 100644 index 00000000..df0b9d50 --- /dev/null +++ b/cloudpathlib/s3/s3_io.py @@ -0,0 +1,106 @@ +""" +S3-specific streaming I/O implementations. + +Provides efficient streaming I/O for S3 using range requests and multipart uploads. +""" + +from typing import Optional, Dict, Any + +from ..cloud_io import _CloudStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("s3") +class _S3StorageRaw(_CloudStorageRaw): + """ + S3-specific raw I/O adapter. + + Implements efficient range-based reads and multipart uploads for S3. + S3 requires non-final parts to be at least 5 MiB; this class accumulates + chunks in _write_buffer and only uploads a part once _MIN_PART_SIZE is reached. + """ + + # S3 minimum part size for non-final parts: 5 MiB + _MIN_PART_SIZE = 5 * 1024 * 1024 + + def __init__(self, client, cloud_path, mode: str = "rb"): + super().__init__(client, cloud_path, mode) + + # Multipart upload state + self._upload_id: Optional[str] = None + self._parts: list = [] + self._part_number: int = 1 + # Accumulation buffer — we only flush a part when >= _MIN_PART_SIZE bytes + self._write_buffer: bytearray = bytearray() + + def _range_get(self, start: int, end: int) -> bytes: + return self._client._range_download(self._cloud_path, start, end) + + def _get_size(self) -> int: + return self._client._get_content_length(self._cloud_path) + + def _is_eof_error(self, error: Exception) -> bool: + error_str = str(error) + return ( + "InvalidRange" in error_str + or "InvalidObjectState" in error_str + or hasattr(error, "__class__") + and "InvalidRange" in error.__class__.__name__ + ) + + # ---- Write support (multipart upload with 5 MiB minimum part size) ---- + + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: + if not data: + return + + self._write_buffer.extend(data) + + # Upload full-sized parts whenever we have enough buffered data + while len(self._write_buffer) >= self._MIN_PART_SIZE: + chunk = bytes(self._write_buffer[: self._MIN_PART_SIZE]) + del self._write_buffer[: self._MIN_PART_SIZE] + if self._upload_id is None: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + part_info = self._client._upload_part( + self._cloud_path, self._upload_id, self._part_number, chunk + ) + self._parts.append(part_info) + self._part_number += 1 + + def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + # Flush remaining buffer as the final part (exempt from 5 MiB floor) + if self._write_buffer: + if self._upload_id is None: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + part_info = self._client._upload_part( + self._cloud_path, + self._upload_id, + self._part_number, + bytes(self._write_buffer), + ) + self._parts.append(part_info) + self._part_number += 1 + self._write_buffer = bytearray() + + if self._upload_id is None: + # No data written — create an empty object directly + self._client._put_empty_object(self._cloud_path) + return + + try: + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) + except Exception: + try: + self._client._abort_multipart_upload(self._cloud_path, self._upload_id) + except Exception: + pass # best-effort abort + raise + finally: + self._upload_id = None + self._parts = [] + self._part_number = 1 + self._write_buffer = bytearray() + + def close(self) -> None: + super().close() diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 9fa4bd75..53f9b590 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -400,5 +400,110 @@ def _generate_presigned_url(self, cloud_path: S3Path, expire_seconds: int = 60 * ) return url + # ====================== STREAMING I/O METHODS ====================== + + def _range_download(self, cloud_path: S3Path, start: int, end: int) -> bytes: + """Download a byte range from S3.""" + try: + response = self.client.get_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + Range=f"bytes={start}-{end}", + **self.boto3_dl_extra_args, + ) + body = response["Body"] + data = body.read() + body.close() + return data + except ClientError as e: + code = e.response["Error"]["Code"] + if code in ("404", "NoSuchKey"): + raise FileNotFoundError(f"S3 object not found: {cloud_path}") + if code in ("InvalidRange", "416"): + return b"" + raise + except Exception as e: + if "InvalidRange" in str(e): + return b"" + raise + + def _get_content_length(self, cloud_path: S3Path) -> int: + """Get the size of an S3 object. + + head_object raises ClientError with code 404, not NoSuchKey. + """ + try: + response = self.client.head_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + **self.boto3_dl_extra_args, + ) + return response["ContentLength"] + except ClientError as e: + code = e.response["Error"]["Code"] + if code in ("404", "NoSuchKey"): + raise FileNotFoundError(f"S3 object not found: {cloud_path}") + raise + + def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: + """Start an S3 multipart upload, threading content-type and upload extra args.""" + extra_args = self.boto3_ul_extra_args.copy() + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_type is not None: + extra_args["ContentType"] = content_type + if content_encoding is not None: + extra_args["ContentEncoding"] = content_encoding + response = self.client.create_multipart_upload( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + **extra_args, + ) + return response["UploadId"] + + def _upload_part( + self, cloud_path: S3Path, upload_id: str, part_number: int, data: bytes + ) -> dict: + """Upload a part in an S3 multipart upload.""" + response = self.client.upload_part( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + UploadId=upload_id, + PartNumber=part_number, + Body=data, + ) + return {"PartNumber": part_number, "ETag": response["ETag"]} + + def _complete_multipart_upload(self, cloud_path: S3Path, upload_id: str, parts: list) -> None: + """Complete an S3 multipart upload.""" + self.client.complete_multipart_upload( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + + def _abort_multipart_upload(self, cloud_path: S3Path, upload_id: str) -> None: + """Abort an S3 multipart upload.""" + self.client.abort_multipart_upload( + Bucket=cloud_path.bucket, Key=cloud_path.key, UploadId=upload_id + ) + + def _put_empty_object(self, cloud_path: S3Path) -> None: + """Upload a zero-byte object, threading content-type and upload extra args.""" + extra_args = self.boto3_ul_extra_args.copy() + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_type is not None: + extra_args["ContentType"] = content_type + if content_encoding is not None: + extra_args["ContentEncoding"] = content_encoding + self.client.put_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + Body=b"", + **extra_args, + ) + S3Client.S3Path = S3Client.CloudPath # type: ignore diff --git a/docs/docs/caching.ipynb b/docs/docs/caching.ipynb index f2be92c5..c94099e8 100644 --- a/docs/docs/caching.ipynb +++ b/docs/docs/caching.ipynb @@ -443,16 +443,15 @@ "\n", "### Automatically\n", "\n", - "We provide a number of different ways for the cache to get cleared automatically for you depending on your use case. These range from no cache clearing done by `cloudpathlib` (`\"persistent\"`), to the most aggressive (`\"close_file\"`), which deletes a file from the cache as soon as the file handle is closed and the file is uploaded to the cloud, if it was changed).\n", + "We provide a number of different ways for the cache to get cleared automatically for you depending on your use case. These range from no cache clearing done by `cloudpathlib` (`\"persistent\"`), to the most aggressive (`\"close_file\"`), which deletes a file from the cache as soon as the file handle is closed and the file is uploaded to the cloud, if it was changed). There is also a `\"streaming\"` mode that bypasses caching entirely for direct I/O.\n", "\n", "The modes are defined in the `FileCacheMode` enum, which you can use directly or you can use the corresponding string value. Examples of both methods are included below.\n", "\n", - "Note: There is not currently a cache mode that _never_ writes a file to disk and only keeps it in memory.\n", - "\n", " - `\"persistent\"` - `cloudpathlib` does not clear the cache at all. In this case, you must also pass a `local_cache_dir` when you instantiate the client.\n", " - `\"tmp_dir\"` (_default_) - Cached files are saved using Python's [`TemporaryDirectory`](https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryDirectory). This provides three potential avenues for the cache to get cleared. First, cached files are removed by `cloudpathlib` when the `*Client` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called. Second, Python clears a temporary directory if all references to that directory leave scope. Finally since the folder is in an operating system temp directory, it will be cleared by the OS (which, depending on the OS, may not happen until system restart).\n", " - `\"cloudpath_object\"` - cached files are removed when the `CloudPath` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called.\n", " - `\"close_file\"` - since we only download a file to the cache on read/write, we can ensure the cache is empty by removing the cached file as soon as the read/write is finished. Reading/writing the same `CloudPath` multiple times will result in re-downloading the file from the cloud. Note: For this to work, `cloudpath` needs to be in control of the reading/writing of files. This means your code base should use the `CloudPath.write_*`, `CloudPath.read_*`, and `CloudPath.open` methods. Using `CloudPath.fspath` (or passing the `CloudPath` as a `PathLike` object to another library) will not clear the cache on file close since it was not opened by `cloudpathlib`.\n", + " - `\"streaming\"` - files are never written to disk. Data is streamed directly from/to cloud storage using range requests for reads and multipart/block uploads for writes. This mode uses only small in-memory buffers and is ideal for large files or memory-constrained environments. Note: `.fspath` and similar properties are not available in streaming mode. Currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n", "\n", "Note: Although we use it in the examples below, for `\"cloudpath_object\"` and `\"tmp_dir\"` you normally shouldn't need to explicitly call `del`. Letting Python garbage collection run on its own once all references to the object leave scope should be sufficient. See details [in the Python docs](https://docs.python.org/3/reference/datamodel.html?highlight=__del__#object.__del__)).\n" ] @@ -712,6 +711,73 @@ "shutil.rmtree(client_cache_dir)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### File cache mode: `\"streaming\"`\n", + "\n", + "The `\"streaming\"` mode provides direct streaming I/O without any local caching. This is ideal for:\n", + "\n", + "- **Large files** that don't fit in memory or disk\n", + "- **Partial reads** where you only need part of a file\n", + "- **Sequential processing** where you read/write once\n", + "- **Memory-constrained environments**\n", + "\n", + "Unlike other cache modes, streaming mode:\n", + "- Reads data directly from cloud storage using range requests\n", + "- Writes data directly to cloud storage using multipart/block uploads\n", + "- Never creates cached files on disk (only uses small in-memory buffers)\n", + "- Works with standard Python file-like interfaces\n", + "\n", + "**Note:** Streaming mode is currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Streaming a large file\n", + "streaming_client = S3Client(file_cache_mode=FileCacheMode.streaming)\n", + "\n", + "flood_image = streaming_client.CloudPath(\n", + " \"s3://ladi/Images/FEMA_CAP/2020/70349/DSC_0002_a89f1b79-786f-4dac-9dcc-609fb1a977b1.jpg\"\n", + ")\n", + "\n", + "# Read the image in streaming mode - no cache file created\n", + "with flood_image.open(\"rb\") as f:\n", + " i = Image.open(f)\n", + " print(\"Image loaded via streaming...\")\n", + "\n", + "# No cache file exists - streaming mode doesn't create one\n", + "print(\"Cache file exists: \", flood_image._local.exists())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Reading in chunks for memory efficiency\n", + "streaming_client = S3Client(file_cache_mode=FileCacheMode.streaming)\n", + "\n", + "flood_image = streaming_client.CloudPath(\n", + " \"s3://ladi/Images/FEMA_CAP/2020/70349/DSC_0002_a89f1b79-786f-4dac-9dcc-609fb1a977b1.jpg\"\n", + ")\n", + "\n", + "# Read file in 8KB chunks without loading entire file\n", + "print(\"Reading file in chunks:\")\n", + "chunk_count = 0\n", + "with flood_image.open(\"rb\", buffer_size=8192) as f:\n", + " while chunk := f.read(8192):\n", + " chunk_count += 1\n", + "\n", + "print(f\"Read {chunk_count} chunks without caching the file\")" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/docs/streaming_io.md b/docs/docs/streaming_io.md new file mode 100644 index 00000000..e17156b5 --- /dev/null +++ b/docs/docs/streaming_io.md @@ -0,0 +1,747 @@ +# Streaming I/O + +CloudPathLib provides high-performance streaming I/O capabilities for cloud storage that work seamlessly with Python's standard I/O interfaces and third-party libraries. + +## Overview + +By default, CloudPathLib downloads files to a local cache before opening them. While this works well for many use cases, it can be inefficient for: + +- **Large files** that don't fit in memory or disk +- **Partial reads** where you only need to access part of a file +- **Sequential processing** where you read a file once and discard it +- **Write-only workflows** where you're generating data to upload + +The streaming I/O system solves these problems by: + +- Reading data directly from cloud storage using range requests +- Writing data directly to cloud storage using multipart/block uploads +- Providing standard Python file-like objects that work with any library +- Eliminating the need for local disk caching + +## Quick Start + +### Enable Streaming Mode + +To use streaming I/O, set your client's `file_cache_mode` to `FileCacheMode.streaming`: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Option 1: Set streaming mode on the client +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + for line in f: + print(line.strip()) + +# Option 2: Change mode on existing client +client = S3Client() +client.file_cache_mode = FileCacheMode.streaming + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + content = f.read() + +# Option 3: Temporarily enable streaming +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +original_mode = path.client.file_cache_mode +path.client.file_cache_mode = FileCacheMode.streaming + +with path.open("rt") as f: + content = f.read() + +path.client.file_cache_mode = original_mode # Restore +``` + +### Basic Examples + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Create a client with streaming enabled +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read a text file +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + for line in f: + print(line.strip()) + +# Write a binary file +path = S3Path("s3://bucket/output.bin", client=client) +with path.open("wb") as f: + f.write(b"Hello, cloud!") + +# Read binary data in chunks +path = S3Path("s3://bucket/large-file.bin", client=client) +with path.open("rb") as f: + while chunk := f.read(8192): + process(chunk) +``` + +## API Reference + +!!! important "Always use `CloudPath.open()`" + The recommended way to use streaming I/O is through `CloudPath.open()` with `FileCacheMode.streaming`. The `CloudBufferedIO` and `CloudTextIO` classes are implementation details returned by `open()` and should not be instantiated directly. + +### `FileCacheMode` Enum + +Controls how `CloudPath.open()` handles file caching: + +- `FileCacheMode.cloudpath_object`: Default - cache files in CloudPath object +- `FileCacheMode.tmp_dir`: Cache files in temporary directory +- `FileCacheMode.persistent`: Cache files persistently +- `FileCacheMode.close_file`: Close file after reading +- **`FileCacheMode.streaming`**: Stream directly without caching + +```python +from cloudpathlib.enums import FileCacheMode + +# Set on client initialization +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Or change dynamically +client.file_cache_mode = FileCacheMode.streaming +``` + +### `CloudPath.open()` + +Opens a cloud file in streaming mode when `file_cache_mode` is set to `FileCacheMode.streaming`. + +```python +CloudPath.open( + mode: str = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + *, + buffer_size: Optional[int] = None, +) -> Union[CloudBufferedIO, CloudTextIO, IO] +``` + +**Parameters:** + +- `mode`: File mode - binary (`'rb'`, `'wb'`, etc.) or text (`'r'`, `'w'`, `'rt'`, `'wt'`, etc.) +- `buffering`: Buffer size (deprecated, use `buffer_size` instead) +- `encoding`: Text encoding (default: `"utf-8"`, text mode only) +- `errors`: Error handling strategy (default: `"strict"`, text mode only) +- `newline`: Newline handling (text mode only) +- `buffer_size`: Size of read/write buffer in bytes (default: 64 KiB) + +**Returns:** + +- `CloudBufferedIO` for binary modes (when streaming) +- `CloudTextIO` for text modes (when streaming) +- Standard file object (when not streaming) + +### `CloudBufferedIO` + +Binary file-like object implementing `io.BufferedIOBase`. + +!!! note "Use `CloudPath.open()` instead" + **Do not instantiate `CloudBufferedIO` directly.** Always use `CloudPath.open()` with the appropriate mode and `FileCacheMode.streaming` to get streaming file objects. The streaming I/O classes are implementation details that are returned by `open()`. + +**Key Methods:** + +- `read(size=-1)`: Read up to size bytes (all if size is -1) +- `read1(size=-1)`: Read up to size bytes with one underlying read call +- `readinto(b)`: Read bytes into a pre-allocated buffer +- `write(b)`: Write bytes +- `flush()`: Flush write buffer to cloud storage +- `seek(offset, whence=SEEK_SET)`: Change stream position +- `tell()`: Return current stream position +- `close()`: Close file and finalize upload + +**Properties:** + +- `name`: The cloud path +- `mode`: File mode (e.g., `"rb"`, `"wb"`) +- `closed`: Whether the file is closed + +**Capability Flags:** + +- `readable()`: Returns True for read modes +- `writable()`: Returns True for write modes +- `seekable()`: Returns True (random access supported) + +### `CloudTextIO` + +Text file-like object implementing `io.TextIOBase`. + +!!! note "Use `CloudPath.open()` instead" + **Do not instantiate `CloudTextIO` directly.** Always use `CloudPath.open()` with text mode (e.g., `"r"`, `"rt"`, `"w"`, `"wt"`) and `FileCacheMode.streaming` to get streaming text file objects. The streaming I/O classes are implementation details that are returned by `open()`. + +**Key Methods:** + +- `read(size=-1)`: Read up to size characters +- `readline(size=-1)`: Read one line +- `readlines(hint=-1)`: Read list of lines +- `write(s)`: Write string +- `writelines(lines)`: Write list of strings +- `flush()`: Flush write buffer +- `seek(offset, whence=SEEK_SET)`: Change position +- `tell()`: Return current position +- `close()`: Close file + +**Properties:** + +- `name`: The cloud path +- `mode`: File mode (e.g., `"rt"`, `"wt"`) +- `encoding`: Text encoding +- `errors`: Error handling strategy +- `newlines`: Newline(s) encountered +- `buffer`: Underlying binary buffer (CloudBufferedIO) +- `closed`: Whether the file is closed + +**Iteration:** + +CloudTextIO supports iteration: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + for line in f: + process(line) +``` + +## Usage Examples + +### Reading Large Files in Chunks + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/huge-file.csv", client=client) + +# Process a large file without loading it entirely into memory +with path.open("rt") as f: + header = f.readline() + for line in f: + process_csv_line(line) +``` + +### Partial File Reads + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.bin", client=client) + +# Read just the header of a file +with path.open("rb") as f: + header = f.read(1024) # Read first 1KB + parse_header(header) + + # Seek to specific position + f.seek(10000) + chunk = f.read(100) +``` + +### Streaming Uploads + +```python +from cloudpathlib import AzureBlobPath, AzureBlobClient +from cloudpathlib.enums import FileCacheMode +import json + +client = AzureBlobClient(file_cache_mode=FileCacheMode.streaming) +path = AzureBlobPath("az://container/output.json", client=client) + +# Write data directly to cloud without local file +with path.open("wt") as f: + f.write('{"items": [\n') + for i, item in enumerate(generate_items()): + if i > 0: + f.write(',\n') + f.write(json.dumps(item)) + f.write('\n]}') +``` + +### Using with pandas + +```python +import pandas as pd +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read CSV directly from cloud +read_path = S3Path("s3://bucket/data.csv", client=client) +with read_path.open("rt") as f: + df = pd.read_csv(f) + +# Write CSV directly to cloud +write_path = S3Path("s3://bucket/output.csv", client=client) +with write_path.open("wt") as f: + df.to_csv(f, index=False) +``` + +### Using with PIL/Pillow + +```python +from PIL import Image +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read image +read_path = S3Path("s3://bucket/image.jpg", client=client) +with read_path.open("rb") as f: + img = Image.open(f) + img.show() + +# Write image +write_path = S3Path("s3://bucket/output.png", client=client) +with write_path.open("wb") as f: + img.save(f, format="PNG") +``` + +### Custom Buffer Size + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Use larger buffer for better throughput on fast connections +path = S3Path("s3://bucket/large-file.bin", client=client) +with path.open("rb", buffer_size=1024*1024) as f: + data = f.read() + +# Use smaller buffer for memory-constrained environments +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt", buffer_size=8192) as f: + for line in f: + process(line) +``` + +## Performance Considerations + +### Buffer Size + +The `buffer_size` parameter controls how much data is fetched from/written to cloud storage in each request: + +- **Larger buffers** (256 KiB - 1 MiB): Better throughput, fewer requests, more memory +- **Smaller buffers** (8 KiB - 64 KiB): Lower memory usage, more requests, lower throughput +- **Default** (64 KiB): Good balance for most use cases + +### Read Patterns + +- **Sequential reads**: Optimal performance - data is fetched ahead as needed +- **Random seeks**: Each seek may trigger a new range request - less efficient +- **Small random reads**: Consider downloading the file to cache instead + +### Write Contract + +Object storage is fundamentally a **sequential, write-once** medium. +Streaming mode reflects that contract: + +| Mode | Streaming behaviour | +|------|---------------------| +| `wb`, `w`, `xb`, `x` | True streaming — data is forwarded to the provider as it arrives | +| `ab`, `a`, `r+b`, `r+`, `w+b`, `w+` | **Falls back to cache** — the object is downloaded, mutated locally, then re-uploaded on close. Semantics are correct; performance matches the cached path. | + +Attempting to seek backward on a write-only streaming stream raises +`io.UnsupportedOperation` because the provider has already accepted +the earlier bytes. + +### Multipart/Block Uploads + +For write operations, the streaming I/O system automatically handles: + +- **S3**: Multipart upload; non-final parts are buffered until they reach + the provider minimum of **5 MiB** (S3 rejects smaller non-final parts). + The final part may be smaller than 5 MiB. +- **Azure**: Block blob staging — each flushed chunk is staged as a block + and committed on close. +- **GCS**: Resumable upload (`blob.open("wb")`) — data streams + incrementally to GCS without in-memory buffering. + +## Provider-Specific Behavior + +### AWS S3 + +- Uses boto3 `get_object()` with `Range` header for reads +- Uses boto3 multipart upload API for writes +- Supports all S3-compatible storage (MinIO, Ceph, etc.) + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +### Azure Blob Storage + +- Uses Azure SDK `download_blob()` with offset/length for reads +- Uses block blob staging and commit for writes +- Compatible with Azure Data Lake Storage Gen2 + +```python +from cloudpathlib import AzureBlobPath, AzureBlobClient +from cloudpathlib.enums import FileCacheMode + +client = AzureBlobClient(file_cache_mode=FileCacheMode.streaming) +path = AzureBlobPath("az://container/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +### Google Cloud Storage + +- Uses GCS SDK `download_as_bytes()` with start/end for reads +- Uses `upload_from_string()` for writes +- Supports GCS-specific features through client configuration + +```python +from cloudpathlib import GSPath, GSClient +from cloudpathlib.enums import FileCacheMode + +client = GSClient(file_cache_mode=FileCacheMode.streaming) +path = GSPath("gs://bucket/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +## Comparison with Cached Mode + +| Feature | Streaming (`FileCacheMode.streaming`) | Cached (default) | +|---------|--------------------------------------|------------------| +| **Disk usage** | Minimal (only buffer) | Full file size | +| **Memory usage** | Configurable buffer | Varies | +| **Read performance** | Sequential: Good
Random: Moderate | Fast (local disk) | +| **Write performance** | Good (direct upload) | Fast write, slower close | +| **Partial reads** | Efficient | Downloads full file | +| **Large files** | Excellent | Limited by disk space | +| **Offline access** | No | Yes (after download) | +| **Compatibility** | Standard I/O interfaces | Standard I/O interfaces | + +## Best Practices + +### When to Use Streaming I/O + +✅ **Good use cases:** + +- Large files that don't fit in memory/disk +- Reading only part of a file (e.g., headers, metadata) +- Sequential processing (one-pass reads) +- Direct upload of generated content +- Integration with libraries that accept file-like objects + +❌ **Consider caching instead:** + +- Small files (< 10 MB) +- Frequent random access to same file +- Multiple passes over the same data +- Offline processing +- Maximum read performance required +- Libraries that require file paths (`.fspath` not available in streaming mode) + +### Error Handling + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +try: + with path.open("rt") as f: + content = f.read() +except FileNotFoundError: + print("File not found in cloud storage") +except PermissionError: + print("Access denied") +except Exception as e: + print(f"Error: {e}") +``` + +### Resource Management + +Always use context managers to ensure proper cleanup: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# ✅ Good - file is automatically closed +with path.open("rt") as f: + content = f.read() + +# ❌ Bad - must remember to close manually +f = path.open("rt") +content = f.read() +f.close() # Easy to forget! +``` + +### Streaming Mode Limitations + +When using `FileCacheMode.streaming`, certain CloudPath features are not available because streaming mode doesn't create cached files on disk: + +**Not Available:** +- `.fspath` property - Raises `CloudPathNotImplementedError` +- `.__fspath__()` method - Raises `CloudPathNotImplementedError` +- Passing CloudPath as `os.PathLike` to libraries that need file paths + +**Workaround:** +Use `CloudPath.open()` and pass the file-like object to libraries that accept file handles instead of file paths. + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode +import pandas as pd + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.csv", client=client) + +# ❌ This will raise an error in streaming mode +# df = pd.read_csv(path.fspath) + +# ✅ Use this instead - pass the open file handle +with path.open("rt") as f: + df = pd.read_csv(f) +``` + +## Compatibility + +### Python I/O Interfaces + +The streaming I/O classes are fully compatible with Python's I/O hierarchy: + +```python +import io +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +path = S3Path("s3://bucket/file.bin", client=client) +with path.open("rb") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.BufferedIOBase) + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.TextIOBase) +``` + +### Third-Party Libraries + +Works with any library that accepts file-like objects: + +- **Data processing**: pandas, NumPy, PyArrow +- **Images**: PIL/Pillow, OpenCV +- **Compression**: gzip, zipfile, tarfile +- **Serialization**: pickle, json, yaml +- **Scientific**: h5py, netCDF4 + +## Troubleshooting + +### "File not found" errors + +Ensure the file exists and you have read permissions: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +if path.exists(): + with path.open("rt") as f: + content = f.read() +``` + +### Slow performance + +Try increasing buffer size: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# Larger buffer for faster networks +with path.open("rb", buffer_size=1024*1024) as f: + data = f.read() +``` + +### Out of memory + +Try smaller buffer size or process in chunks: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/huge.bin", client=client) + +# Process large file in chunks +with path.open("rb", buffer_size=8192) as f: + while chunk := f.read(8192): + process_chunk(chunk) +``` + +## Migration Guide + +### From Cached to Streaming + +Before: + +```python +from cloudpathlib import S3Path + +path = S3Path("s3://bucket/file.txt") +with path.open("rt") as f: # Downloads to cache + content = f.read() +``` + +After: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Option 1: Set on client initialization +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: # Streams directly + content = f.read() + +# Option 2: Change client mode +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +path.client.file_cache_mode = FileCacheMode.streaming +with path.open("rt") as f: # Streams directly + content = f.read() +``` + +## Advanced Topics + +### Custom Clients + +Pass custom clients with specific configurations: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode +from botocore.config import Config + +# Custom S3 client with retry configuration +client = S3Client( + file_cache_mode=FileCacheMode.streaming, + boto3_config=Config( + retries={'max_attempts': 10, 'mode': 'adaptive'} + ) +) + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + content = f.read() +``` + +### Multiple Files + +Process multiple files efficiently: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +bucket = S3Path("s3://bucket/", client=client) + +for file_path in bucket.glob("*.csv"): + with file_path.open("rt") as f: + process_csv(f) +``` + +### Encoding Detection + +For files with unknown encoding: + +```python +import chardet +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# Read a small sample to detect encoding +with path.open("rb") as f: + sample = f.read(10000) + detected = chardet.detect(sample) + encoding = detected['encoding'] + +# Re-open with detected encoding +with path.open("rt", encoding=encoding) as f: + content = f.read() +``` + +### Context Manager for Temporary Streaming + +Use a context manager to temporarily enable streaming mode: + +```python +from contextlib import contextmanager +from cloudpathlib import S3Client +from cloudpathlib.enums import FileCacheMode + +@contextmanager +def streaming_mode(client): + """Temporarily enable streaming mode on a client.""" + original_mode = client.file_cache_mode + try: + client.file_cache_mode = FileCacheMode.streaming + yield client + finally: + client.file_cache_mode = original_mode + +# Usage +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +with streaming_mode(client): + with path.open("rt") as f: + content = f.read() # Uses streaming + +# Back to cached mode +with path.open("rt") as f: + content = f.read() # Uses caching +``` diff --git a/tests/conftest.py b/tests/conftest.py index 9732c39a..e35c6016 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,10 +21,13 @@ from shortuuid import uuid from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed -from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path -from cloudpathlib.cloudpath import implementation_registry +from cloudpathlib.azure import AzureBlobClient, AzureBlobPath, _AzureBlobStorageRaw +from cloudpathlib.gs import GSClient, GSPath, _GSStorageRaw +from cloudpathlib.s3 import S3Client, S3Path, _S3StorageRaw +from cloudpathlib.cloudpath import implementation_registry, CloudImplementation from cloudpathlib.http.httpclient import HttpClient, HttpsClient from cloudpathlib.http.httppath import HttpPath, HttpsPath +from cloudpathlib.http.http_io import _HttpStorageRaw from cloudpathlib.local import ( local_azure_blob_implementation, LocalAzureBlobClient, @@ -80,8 +83,7 @@ class CloudProviderTestRig: def __init__( self, - path_class: type, - client_class: type, + cloud_implementation: CloudImplementation, drive: str = "drive", test_dir: str = "", live_server: bool = False, @@ -92,8 +94,7 @@ def __init__( path_class (type): CloudPath subclass client_class (type): Client subclass """ - self.path_class = path_class - self.client_class = client_class + self.cloud_implementation = cloud_implementation self.drive = drive self.test_dir = test_dir self.live_server = live_server # if the server is a live server @@ -101,6 +102,18 @@ def __init__( required_client_kwargs if required_client_kwargs is not None else {} ) + @property + def path_class(self): + return self.cloud_implementation.path_class + + @property + def client_class(self): + return self.cloud_implementation.client_class + + @property + def raw_io_class(self): + return self.cloud_implementation.raw_io_class + @property def cloud_prefix(self): return self.path_class.cloud_prefix @@ -200,9 +213,13 @@ def _azure_fixture(conn_str_env_var, adls_gen2, request, monkeypatch, assets_dir MockedDataLakeServiceClient, ) + azure_blob_implementation = CloudImplementation() + azure_blob_implementation._client_class = AzureBlobClient + azure_blob_implementation._path_class = AzureBlobPath + azure_blob_implementation._raw_io_class = _AzureBlobStorageRaw + rig = CloudProviderTestRig( - path_class=AzureBlobPath, - client_class=AzureBlobClient, + cloud_implementation=azure_blob_implementation, drive=drive, test_dir=test_dir, live_server=live_server, @@ -284,9 +301,13 @@ def gs_rig(request, monkeypatch, assets_dir, live_server): ) monkeypatch.setattr(cloudpathlib.gs.gsclient, "google_default_auth", mock_default_auth) + gs_implementation = CloudImplementation() + gs_implementation._client_class = GSClient + gs_implementation._path_class = GSPath + gs_implementation._raw_io_class = _GSStorageRaw + rig = CloudProviderTestRig( - path_class=GSPath, - client_class=GSClient, + cloud_implementation=gs_implementation, drive=drive, test_dir=test_dir, live_server=live_server, @@ -334,9 +355,13 @@ def s3_rig(request, monkeypatch, assets_dir, live_server): mocked_session_class_factory(test_dir), ) + s3_implementation = CloudImplementation() + s3_implementation._client_class = S3Client + s3_implementation._path_class = S3Path + s3_implementation._raw_io_class = _S3StorageRaw + rig = CloudProviderTestRig( - path_class=S3Path, - client_class=S3Client, + cloud_implementation=s3_implementation, drive=drive, test_dir=test_dir, live_server=live_server, @@ -418,9 +443,13 @@ def _spin_up_bucket(): mocked_session_class_factory(test_dir), ) + custom_s3_implementation = CloudImplementation() + custom_s3_implementation._client_class = S3Client + custom_s3_implementation._path_class = S3Path + custom_s3_implementation._raw_io_class = _S3StorageRaw + rig = CloudProviderTestRig( - path_class=S3Path, - client_class=S3Client, + cloud_implementation=custom_s3_implementation, drive=drive, test_dir=test_dir, live_server=live_server, @@ -457,9 +486,13 @@ def local_azure_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "azure", local_azure_blob_implementation) + local_azure_blob_cloud_implementation = CloudImplementation() + local_azure_blob_cloud_implementation._client_class = LocalAzureBlobClient + local_azure_blob_cloud_implementation._path_class = LocalAzureBlobPath + local_azure_blob_cloud_implementation._raw_io_class = _AzureBlobStorageRaw + rig = CloudProviderTestRig( - path_class=LocalAzureBlobPath, - client_class=LocalAzureBlobClient, + cloud_implementation=local_azure_blob_cloud_implementation, drive=drive, test_dir=test_dir, ) @@ -488,9 +521,13 @@ def local_gs_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "gs", local_gs_implementation) + local_gs_cloud_implementation = CloudImplementation() + local_gs_cloud_implementation._client_class = LocalGSClient + local_gs_cloud_implementation._path_class = LocalGSPath + local_gs_cloud_implementation._raw_io_class = _GSStorageRaw + rig = CloudProviderTestRig( - path_class=LocalGSPath, - client_class=LocalGSClient, + cloud_implementation=local_gs_cloud_implementation, drive=drive, test_dir=test_dir, ) @@ -518,9 +555,13 @@ def local_s3_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "s3", local_s3_implementation) + local_s3_cloud_implementation = CloudImplementation() + local_s3_cloud_implementation._client_class = LocalS3Client + local_s3_cloud_implementation._path_class = LocalS3Path + local_s3_cloud_implementation._raw_io_class = _S3StorageRaw + rig = CloudProviderTestRig( - path_class=LocalS3Path, - client_class=LocalS3Client, + cloud_implementation=local_s3_implementation, drive=drive, test_dir=test_dir, ) @@ -557,9 +598,13 @@ def http_rig(request, assets_dir, http_server): # noqa: F811 shutil.copytree(assets_dir, server_dir / test_dir) _sync_filesystem() + http_implementation = CloudImplementation() + http_implementation._client_class = HttpClient + http_implementation._path_class = HttpPath + http_implementation._raw_io_class = _HttpStorageRaw + rig = CloudProviderTestRig( - path_class=HttpPath, - client_class=HttpClient, + cloud_implementation=http_implementation, drive=drive, test_dir=test_dir, ) @@ -589,9 +634,13 @@ def https_rig(request, assets_dir, https_server): # noqa: F811 skip_verify_ctx.check_hostname = False skip_verify_ctx.load_verify_locations(utilities_dir / "insecure-test.pem") + https_implementation = CloudImplementation() + https_implementation._client_class = HttpsClient + https_implementation._path_class = HttpsPath + https_implementation._raw_io_class = _HttpStorageRaw + rig = CloudProviderTestRig( - path_class=HttpsPath, - client_class=HttpsClient, + cloud_implementation=https_implementation, drive=drive, test_dir=test_dir, required_client_kwargs=dict( diff --git a/tests/http_fixtures.py b/tests/http_fixtures.py index d43ce236..dec3479a 100644 --- a/tests/http_fixtures.py +++ b/tests/http_fixtures.py @@ -75,7 +75,59 @@ def do_POST(self): @retry(stop=stop_after_attempt(5), wait=wait_fixed(0.1)) def do_GET(self): - super().do_GET() + """Handle GET requests with optional Range header support.""" + # Check if this is a range request + range_header = self.headers.get("Range") + if range_header: + self._handle_range_request(range_header) + else: + super().do_GET() + + def _handle_range_request(self, range_header): + """Handle Range requests for partial content.""" + path = Path(self.translate_path(self.path)) + + if not path.exists() or not path.is_file(): + self.send_error(404, "File not found") + return + + # Parse the Range header (format: "bytes=start-end") + try: + range_spec = range_header.replace("bytes=", "").strip() + parts = range_spec.split("-") + start = int(parts[0]) if parts[0] else 0 + + file_size = path.stat().st_size + + # Handle end byte + if len(parts) > 1 and parts[1]: + end = int(parts[1]) + else: + end = file_size - 1 + + # Validate range + if start < 0 or end >= file_size or start > end: + self.send_error(416, "Requested Range Not Satisfiable") + self.send_header("Content-Range", f"bytes */{file_size}") + self.end_headers() + return + + # Read the requested range + with path.open("rb") as f: + f.seek(start) + content = f.read(end - start + 1) + + # Send partial content response + self.send_response(206) # Partial Content + self.send_header("Content-Type", self.guess_type(str(path))) + self.send_header("Content-Length", str(len(content))) + self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}") + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + self.wfile.write(content) + + except (ValueError, IndexError) as e: + self.send_error(400, f"Bad Range header: {e}") @retry(stop=stop_after_attempt(5), wait=wait_fixed(0.1)) def do_HEAD(self): diff --git a/tests/mock_clients/mock_azureblob.py b/tests/mock_clients/mock_azureblob.py index 2fc8f7eb..8370d05f 100644 --- a/tests/mock_clients/mock_azureblob.py +++ b/tests/mock_clients/mock_azureblob.py @@ -61,6 +61,9 @@ def __init__(self, test_dir, adls): self.metadata_cache = _JsonCache(self.root / ".metadata") self.adls_gen2 = adls + # For block blob uploads (multipart) - shared across all blob clients + self._staged_blocks = {} + @classmethod def from_connection_string(cls, conn_str, credential): # configured in conftest.py @@ -114,7 +117,7 @@ def url(self): def get_blob_properties(self): path = self.root / self.key if path.exists() and path.is_file(): - return BlobProperties( + props = BlobProperties( **{ "name": self.key, "Last-Modified": datetime.fromtimestamp(path.stat().st_mtime), @@ -125,11 +128,14 @@ def get_blob_properties(self): "metadata": dict(), } ) + # Set size directly as BlobProperties doesn't accept it in constructor + props.size = path.stat().st_size + return props else: raise ResourceNotFoundError - def download_blob(self): - return MockStorageStreamDownloader(self.root, self.key) + def download_blob(self, offset=None, length=None): + return MockStorageStreamDownloader(self.root, self.key, offset=offset, length=length) def set_blob_metadata(self, metadata): path = self.root / self.key @@ -148,21 +154,55 @@ def delete_blob(self): def upload_blob(self, data, overwrite, content_settings=None): path = self.root / self.key path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data.read()) + if isinstance(data, bytes): + path.write_bytes(data) + else: + path.write_bytes(data.read()) if content_settings is not None: self.service_client.metadata_cache[self.root / self.key] = ( content_settings.content_type ) + def stage_block(self, block_id, data, length): + """Stage a block for block blob upload.""" + # Store the block data indexed by blob key in service client's staged blocks + if self.key not in self.service_client._staged_blocks: + self.service_client._staged_blocks[self.key] = {} + self.service_client._staged_blocks[self.key][block_id] = data + + def commit_block_list(self, block_ids, content_settings=None): + """Commit a list of staged blocks to create a blob.""" + path = self.root / self.key + path.parent.mkdir(parents=True, exist_ok=True) + + # Concatenate blocks in order + if self.key in self.service_client._staged_blocks: + complete_data = b"" + for block_id in block_ids: + complete_data += self.service_client._staged_blocks[self.key][block_id] + + path.write_bytes(complete_data) + + # Clean up staged blocks + del self.service_client._staged_blocks[self.key] + class MockStorageStreamDownloader: - def __init__(self, root, key): + def __init__(self, root, key, offset=None, length=None): self.root = root self.key = key + self.offset = offset + self.length = length def readall(self): - return (self.root / self.key).read_bytes() + data = (self.root / self.key).read_bytes() + if self.offset is not None: + if self.length is not None: + return data[self.offset : self.offset + self.length] + else: + return data[self.offset :] + return data def content_as_bytes(self): return self.readall() diff --git a/tests/mock_clients/mock_gs.py b/tests/mock_clients/mock_gs.py index 26487767..d56bb9ee 100644 --- a/tests/mock_clients/mock_gs.py +++ b/tests/mock_clients/mock_gs.py @@ -72,6 +72,27 @@ def download_to_filename(self, filename, timeout=None, retry=None): to_path.parent.mkdir(exist_ok=True, parents=True) to_path.write_bytes(from_path.read_bytes()) + def download_as_bytes(self, start=None, end=None, timeout=None, retry=None): + """Download blob content as bytes with optional byte range.""" + # if timeout is not None, assume that the test wants a timeout and throw it + if timeout is not None: + raise TimeoutError("Download timed out") + + # indicate that retry object made it through to the GS lib + if retry is not None: + retry.mocked_retries = 1 + + from_path = self.bucket / self.name + data = from_path.read_bytes() + + # Handle byte range if specified + if start is not None: + if end is not None: + return data[start : end + 1] + else: + return data[start:] + return data + def patch(self): if "updated" in self.metadata: (self.bucket / self.name).touch() @@ -107,6 +128,25 @@ def upload_from_filename(self, filename, content_type=None, timeout=None, retry= self.client.metadata_cache[self.bucket / self.name] = content_type + def upload_from_string(self, data, content_type=None, timeout=None, retry=None): + """Upload from bytes/string data.""" + # if timeout is not None, assume that the test wants a timeout and throw it + if timeout is not None: + raise TimeoutError("Upload timed out") + + # indicate that retry object made it through to the GS lib + if retry is not None: + retry.mocked_retries = 1 + + path = self.bucket / self.name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(data, str): + path.write_text(data) + else: + path.write_bytes(data) + + self.client.metadata_cache[self.bucket / self.name] = content_type + @property def etag(self): return "etag" @@ -136,6 +176,38 @@ def public_url(self) -> str: def generate_signed_url(self, version: str, expiration: timedelta, method: str): return f"https://storage.googleapis.com{self.bucket}/{self.name}?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=TEST&X-Goog-Date=20240131T185515Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=TEST" + def open(self, mode="rb", **kwargs): + """Return a file-like writer/reader for resumable uploads (mock implementation).""" + if mode == "wb": + return _MockBlobWriter(self) + raise NotImplementedError(f"Mock blob.open() only supports 'wb', not {mode!r}") + + +class _MockBlobWriter: + """Simulates a GCS resumable upload stream (blob.open('wb')).""" + + def __init__(self, blob: "MockBlob") -> None: + self._blob = blob + self._buf: bytearray = bytearray() + self._closed: bool = False + + def write(self, data: bytes) -> int: + if self._closed: + raise ValueError("I/O operation on closed stream") + self._buf.extend(data) + return len(data) + + def close(self) -> None: + if not self._closed: + self._closed = True + self._blob.upload_from_string(bytes(self._buf)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + class MockBucket: def __init__(self, name, bucket_name, client=None): diff --git a/tests/mock_clients/mock_s3.py b/tests/mock_clients/mock_s3.py index a2f850ca..da181703 100644 --- a/tests/mock_clients/mock_s3.py +++ b/tests/mock_clients/mock_s3.py @@ -257,6 +257,133 @@ def head_object(self, Bucket, Key, **kwargs): "Metadata": {}, } + def get_object(self, Bucket, Key, Range=None, **kwargs): + """Get an S3 object with optional byte range.""" + if ( + not (self.root / Key).exists() + or (self.root / Key).is_dir() + or Bucket != DEFAULT_S3_BUCKET_NAME + ): + raise ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "The specified key does not exist."}}, + {}, + ) + + path = self.root / Key + data = path.read_bytes() + + if Range: + import re + + match = re.match(r"bytes=(\d+)-(\d+)", Range) + if match: + start, end = int(match.group(1)), int(match.group(2)) + data = data[start : end + 1] + else: + raise ClientError( + { + "Error": { + "Code": "InvalidRange", + "Message": "The requested range is not satisfiable", + } + }, + {}, + ) + + from io import BytesIO + + return {"Body": BytesIO(data), "ContentLength": len(data)} + + def create_multipart_upload(self, Bucket, Key, **kwargs): + """Start a multipart upload.""" + import uuid + + upload_id = str(uuid.uuid4()) + if not hasattr(self, "_uploads"): + self._uploads = {} + self._uploads[upload_id] = {"Bucket": Bucket, "Key": Key, "Parts": []} + return {"UploadId": upload_id} + + def upload_part(self, Bucket, Key, UploadId, PartNumber, Body, **kwargs): + """Upload a part in a multipart upload.""" + if not hasattr(self, "_uploads") or UploadId not in self._uploads: + raise ClientError( + { + "Error": { + "Code": "NoSuchUpload", + "Message": "The specified upload does not exist.", + } + }, + {}, + ) + + upload = self._uploads[UploadId] + if isinstance(Body, bytes): + data = Body + else: + data = Body.read() if hasattr(Body, "read") else Body + + upload["Parts"].append({"PartNumber": PartNumber, "Data": data}) + + import hashlib + + etag = hashlib.md5(data).hexdigest() + return {"ETag": etag} + + def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload, **kwargs): + """Complete a multipart upload.""" + if not hasattr(self, "_uploads") or UploadId not in self._uploads: + raise ClientError( + { + "Error": { + "Code": "NoSuchUpload", + "Message": "The specified upload does not exist.", + } + }, + {}, + ) + + upload = self._uploads[UploadId] + parts = sorted(upload["Parts"], key=lambda p: p["PartNumber"]) + complete_data = b"".join([p["Data"] for p in parts]) + + path = self.root / Key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(complete_data) + + del self._uploads[UploadId] + + return {"Location": f"https://{Bucket}.s3.amazonaws.com/{Key}"} + + def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): + """Abort a multipart upload.""" + if hasattr(self, "_uploads") and UploadId in self._uploads: + del self._uploads[UploadId] + + def put_object(self, Bucket, Key, Body=b"", **kwargs): + """Upload data directly (single PUT).""" + if Bucket != DEFAULT_S3_BUCKET_NAME and ".mrap" not in Bucket: + raise ClientError( + { + "Error": { + "Code": "NoSuchBucket", + "Message": "The specified bucket does not exist.", + } + }, + "PutObject", + ) + path = self.root / Key + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(Body, bytes): + path.write_bytes(Body) + else: + path.write_bytes(Body.read() if hasattr(Body, "read") else b"") + if "ContentType" in kwargs and self.session is not None: + self.session.metadata_cache[path] = kwargs["ContentType"] + import hashlib + + return {"ETag": f'"{hashlib.md5(path.read_bytes()).hexdigest()}"'} + def generate_presigned_url(self, op: str, Params: dict, ExpiresIn: int): mock_presigned_url = f"https://{Params['Bucket']}.s3.amazonaws.com/{Params['Key']}?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=TEST%2FTEST%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240131T194721Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=TEST" return mock_presigned_url diff --git a/tests/test_client.py b/tests/test_client.py index 3eceafc8..1d05645c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -8,10 +8,15 @@ from cloudpathlib import CloudPath from cloudpathlib.client import register_client_class -from cloudpathlib.cloudpath import implementation_registry, register_path_class +from cloudpathlib.cloudpath import ( + implementation_registry, + register_path_class, + register_raw_io_class, +) from cloudpathlib.http.httpclient import HttpClient, HttpsClient from cloudpathlib.s3.s3client import S3Client from cloudpathlib.s3.s3path import S3Path +from cloudpathlib.s3.s3_io import _S3StorageRaw def test_default_client_instantiation(rig): @@ -140,6 +145,10 @@ class MyS3Path(S3Path): class MyS3Client(S3Client): pass + @register_raw_io_class("mys3") + class MyS3StorageRaw(_S3StorageRaw): + pass + yield (MyS3Path, MyS3Client) # cleanup after use diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py new file mode 100644 index 00000000..69cfd9bc --- /dev/null +++ b/tests/test_cloud_io.py @@ -0,0 +1,1393 @@ +""" +Tests for cloud storage streaming I/O. + +Tests CloudBufferedIO, CloudTextIO, and streaming mode for direct +streaming without local caching. +""" + +import io +import threading +import pytest + +from cloudpathlib import S3Path, AzureBlobPath, GSPath +from cloudpathlib import CloudBufferedIO, CloudTextIO +from cloudpathlib.enums import FileCacheMode + +# Sample test data +BINARY_DATA = b"Hello, World! This is binary data.\n" * 100 +TEXT_DATA = "Hello, World! This is text data.\n" * 100 +MULTILINE_TEXT = """Line 1 +Line 2 +Line 3 +Line 4 with special chars: éñ中文 +""" + + +@pytest.fixture +def temp_cloud_file(rig): + """Create a temporary cloud file for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_io.txt") + path.write_text(TEXT_DATA) + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +@pytest.fixture +def temp_cloud_binary_file(rig): + """Create a temporary cloud binary file for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_io.bin") + path.write_bytes(BINARY_DATA) + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +@pytest.fixture +def temp_cloud_multiline_file(rig): + """Create a temporary cloud file with multiple lines for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_multiline.txt") + path.write_text(MULTILINE_TEXT) + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# CloudBufferedIO tests (binary streaming) +# ============================================================================ + + +def test_read_binary_stream(temp_cloud_binary_file): + """Test reading binary data via streaming.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Verify it's the right type + assert isinstance(f, CloudBufferedIO) + assert isinstance(f, io.BufferedIOBase) + + # Read all data + data = f.read() + assert data == BINARY_DATA + + +def test_read_chunks(temp_cloud_binary_file): + """Test reading data in chunks.""" + chunk_size = 100 + with temp_cloud_binary_file.open(mode="rb") as f: + chunks = [] + while True: + chunk = f.read(chunk_size) + if not chunk: + break + chunks.append(chunk) + assert len(chunk) <= chunk_size + + # Verify we got all data + assert b"".join(chunks) == BINARY_DATA + + +def test_read1(temp_cloud_binary_file): + """Test read1 method.""" + with temp_cloud_binary_file.open(mode="rb") as f: + chunk = f.read1(50) + assert len(chunk) <= 50 + assert len(chunk) > 0 + + +def test_readinto(temp_cloud_binary_file): + """Test readinto method.""" + with temp_cloud_binary_file.open(mode="rb") as f: + buf = bytearray(100) + n = f.readinto(buf) + assert n > 0 + assert n <= 100 + assert buf[:n] == BINARY_DATA[:n] + + +def test_seek_tell(temp_cloud_binary_file): + """Test seek and tell operations.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Initial position + assert f.tell() == 0 + + # Read some data + f.read(50) + assert f.tell() == 50 + + # Seek to beginning + pos = f.seek(0) + assert pos == 0 + assert f.tell() == 0 + + # Seek relative + pos = f.seek(10, io.SEEK_CUR) + assert pos == 10 + + # Seek from end + pos = f.seek(-10, io.SEEK_END) + assert pos == len(BINARY_DATA) - 10 + + +def test_seekable_readable_writable(temp_cloud_binary_file): + """Test capability flags.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert f.readable() + assert not f.writable() + assert f.seekable() + + +def test_buffered_io_properties(temp_cloud_binary_file): + """Test file properties.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert f.name == str(temp_cloud_binary_file) + assert f.mode == "rb" + assert not f.closed + + assert f.closed + + +def test_buffered_io_context_manager(temp_cloud_binary_file): + """Test context manager protocol.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert not f.closed + data = f.read(10) + assert len(data) == 10 + + assert f.closed + + +def test_write_binary_stream(rig): + """Test writing binary data via streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_binary.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Write data + with path.open(mode="wb") as f: + assert isinstance(f, CloudBufferedIO) + assert f.writable() + assert not f.readable() + + n = f.write(BINARY_DATA) + assert n == len(BINARY_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify data was written + assert path.exists() + assert path.read_bytes() == BINARY_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_write_chunks(rig): + """Test writing data in chunks.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_chunks.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + chunk_size = 100 + with path.open(mode="wb", buffer_size=chunk_size) as f: + for i in range(0, len(BINARY_DATA), chunk_size): + chunk = BINARY_DATA[i : i + chunk_size] + f.write(chunk) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify + assert path.read_bytes() == BINARY_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_flush(rig): + """Test explicit flush.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_flush.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + f.write(b"First chunk") + f.flush() + f.write(b" Second chunk") + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_bytes() == b"First chunk Second chunk" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_buffered_io_isinstance_checks(temp_cloud_binary_file): + """Test that instances pass isinstance checks.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.BufferedIOBase) + assert not isinstance(f, io.TextIOBase) + + +def test_not_found_error(rig): + """Test error when file doesn't exist.""" + path = rig.create_cloud_path("nonexistent.bin") + + with pytest.raises(FileNotFoundError): + with path.open(mode="rb") as f: + f.read() + + +# ============================================================================ +# CloudTextIO tests (text streaming) +# ============================================================================ + + +def test_read_text_stream(temp_cloud_file): + """Test reading text data via streaming.""" + with temp_cloud_file.open(mode="rt") as f: + # Verify it's the right type + assert isinstance(f, CloudTextIO) + assert isinstance(f, io.TextIOBase) + + # Read all data + data = f.read() + assert data == TEXT_DATA + + +def test_read_text_mode_without_t(temp_cloud_file): + """Test reading text with mode 'r' (without explicit 't').""" + with temp_cloud_file.open(mode="r") as f: + assert isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + + +def test_readline(temp_cloud_multiline_file): + """Test readline method.""" + with temp_cloud_multiline_file.open(mode="rt") as f: + line1 = f.readline() + assert line1 == "Line 1\n" + + line2 = f.readline() + assert line2 == "Line 2\n" + + +def test_readlines(temp_cloud_multiline_file): + """Test readlines method.""" + with temp_cloud_multiline_file.open(mode="rt") as f: + lines = f.readlines() + assert len(lines) == 4 + assert lines[0] == "Line 1\n" + assert "special chars" in lines[3] + + +def test_iteration(temp_cloud_multiline_file): + """Test iterating over lines.""" + with temp_cloud_multiline_file.open(mode="rt") as f: + lines = list(f) + assert len(lines) == 4 + assert lines[0] == "Line 1\n" + + +def test_encoding(rig): + """Test different encodings.""" + path = rig.create_cloud_path("test_encoding.txt") + utf8_text = "Hello 世界 🌍" + + try: + # Write with UTF-8 + path.write_text(utf8_text, encoding="utf-8") + + # Read with UTF-8 + with path.open(mode="rt", encoding="utf-8") as f: + assert f.encoding == "utf-8" + data = f.read() + assert data == utf8_text + finally: + try: + path.unlink() + except Exception: + pass + + +def test_text_properties(temp_cloud_file): + """Test text mode properties.""" + with temp_cloud_file.open(mode="rt", encoding="utf-8", errors="strict") as f: + assert f.encoding == "utf-8" + assert f.errors == "strict" + assert f.name == str(temp_cloud_file) + assert "r" in f.mode + + +def test_write_text_stream(rig): + """Test writing text data via streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_text.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt") as f: + assert isinstance(f, CloudTextIO) + n = f.write(TEXT_DATA) + assert n == len(TEXT_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify + assert path.read_text() == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_writelines(rig): + """Test writelines method.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_writelines.txt") + lines = ["Line 1\n", "Line 2\n", "Line 3\n"] + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt") as f: + f.writelines(lines) + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_text() == "".join(lines) + finally: + try: + path.unlink() + except Exception: + pass + + +def test_text_io_isinstance_checks(temp_cloud_file): + """Test that instances pass isinstance checks.""" + with temp_cloud_file.open(mode="rt") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.TextIOBase) + assert not isinstance(f, io.BufferedIOBase) + + +def test_buffer_property(temp_cloud_file): + """Test access to underlying binary buffer.""" + with temp_cloud_file.open(mode="rt") as f: + assert hasattr(f, "buffer") + assert isinstance(f.buffer, CloudBufferedIO) + + +# ============================================================================ +# CloudPath.open streaming integration tests +# ============================================================================ + + +def test_cloudpath_stream_read(temp_cloud_file): + """Test CloudPath.open with streaming mode for reading.""" + # The temp_cloud_file fixture already sets streaming mode + with temp_cloud_file.open(mode="r") as f: + assert isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + + +def test_cloudpath_stream_write(rig): + """Test CloudPath.open with streaming mode for writing.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_stream_write.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="w") as f: + assert isinstance(f, CloudTextIO) + f.write(TEXT_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_text() == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_cloudpath_stream_binary(temp_cloud_binary_file): + """Test CloudPath.open with streaming mode for binary.""" + # The temp_cloud_binary_file fixture already sets streaming mode + with temp_cloud_binary_file.open(mode="rb") as f: + assert isinstance(f, CloudBufferedIO) + data = f.read() + assert data == BINARY_DATA + + +def test_cloudpath_stream_false_uses_cache(rig): + """Test that non-streaming mode uses traditional caching.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_caching.txt") + path.write_text(TEXT_DATA) + + try: + # Default mode (not streaming) should use caching + assert path.client.file_cache_mode != FileCacheMode.streaming + + with path.open(mode="r") as f: + # Should not be a CloudTextIO instance + assert not isinstance(f, CloudTextIO) + # Should still read correctly + data = f.read() + assert data == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_cloudpath_default_no_streaming(rig): + """Test that default behavior uses caching, not streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_default.txt") + path.write_text(TEXT_DATA) + + try: + # Default client mode should not be streaming + assert path.client.file_cache_mode != FileCacheMode.streaming + + with path.open(mode="r") as f: + # Default should not use streaming + assert not isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# CloudPath.open factory tests +# ============================================================================ + + +def test_auto_client_s3(temp_cloud_file): + """Test auto-detection of S3 client.""" + if not isinstance(temp_cloud_file, S3Path): + pytest.skip("Not testing S3") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_auto_client_azure(temp_cloud_file): + """Test auto-detection of Azure client.""" + if not isinstance(temp_cloud_file, AzureBlobPath): + pytest.skip("Not testing Azure") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_auto_client_gs(temp_cloud_file): + """Test auto-detection of GCS client.""" + if not isinstance(temp_cloud_file, GSPath): + pytest.skip("Not testing GCS") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_explicit_client(temp_cloud_file): + """Test passing explicit client.""" + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_buffer_size_parameter(temp_cloud_binary_file): + """Test custom buffer size.""" + buffer_size = 1024 + with temp_cloud_binary_file.open(mode="rb", buffer_size=buffer_size) as f: + assert f._buffer_size == buffer_size + + +def test_text_parameters(rig): + """Test text-specific parameters.""" + path = rig.create_cloud_path("test_params.txt") + text = "Test data" + + try: + path.write_text(text) + + with path.open(mode="rt", encoding="utf-8", errors="strict", newline=None) as f: + assert f.encoding == "utf-8" + assert f.errors == "strict" + data = f.read() + assert data == text + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Edge cases and error conditions +# ============================================================================ + + +def test_empty_file_read(rig): + """Test reading an empty file.""" + path = rig.create_cloud_path("test_empty.txt") + + try: + path.write_text("") + + with path.open(mode="rt") as f: + data = f.read() + assert data == "" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_empty_file_write(rig): + """Test writing an empty file.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_empty_write.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt"): + pass # Write nothing + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.exists() + assert path.read_text() == "" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_large_file_streaming(rig): + """Test streaming a larger file.""" + path = rig.create_cloud_path("test_large.bin") + # 1 MB of data + large_data = b"X" * (1024 * 1024) + + try: + path.write_bytes(large_data) + + # Read in chunks + with path.open(mode="rb", buffer_size=8192) as f: + chunks = [] + while True: + chunk = f.read(8192) + if not chunk: + break + chunks.append(chunk) + + result = b"".join(chunks) + assert len(result) == len(large_data) + assert result == large_data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_seek_beyond_eof(temp_cloud_binary_file): + """Test seeking beyond end of file.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek beyond EOF + size = len(BINARY_DATA) + pos = f.seek(size + 1000) + assert pos == size + 1000 + + # Reading should return empty + data = f.read(10) + assert data == b"" + + +def test_closed_file_operations(temp_cloud_file): + """Test operations on closed file raise errors.""" + with temp_cloud_file.open(mode="rt") as f: + pass # Just open and close + + # Now f is closed + with pytest.raises(ValueError): + f.read() + + with pytest.raises(ValueError): + f.readline() + + +def test_binary_mode_required_for_buffered(temp_cloud_file): + """Test that CloudBufferedIO requires binary mode.""" + # Get the raw IO class + raw_io_class = temp_cloud_file._cloud_meta.raw_io_class + if raw_io_class is None: + pytest.skip("No raw IO class registered") + + # This should raise an error + with pytest.raises(ValueError, match="binary mode"): + CloudBufferedIO( + raw_io_class=raw_io_class, + client=temp_cloud_file.client, + cloud_path=temp_cloud_file, + mode="r", + ) + + +def test_text_mode_required_for_text(temp_cloud_file): + """Test that CloudTextIO requires text mode.""" + # Get the raw IO class + raw_io_class = temp_cloud_file._cloud_meta.raw_io_class + if raw_io_class is None: + pytest.skip("No raw IO class registered") + + with pytest.raises(ValueError, match="text mode"): + CloudTextIO( + raw_io_class=raw_io_class, + client=temp_cloud_file.client, + cloud_path=temp_cloud_file, + mode="rb", + ) + + +def test_unsupported_operations(temp_cloud_file): + """Test unsupported operations raise appropriate errors.""" + with temp_cloud_file.open(mode="rt") as f: + # fileno() should raise + with pytest.raises(OSError): + f.fileno() + + # isatty() should return False + assert not f.isatty() + + +def test_read_write_mode_not_implemented(temp_cloud_file): + """Test that read/write modes work as expected.""" + # For now, r+ and w+ may have limitations + # Test basic write mode + with temp_cloud_file.open(mode="wt") as f: + assert f.writable() + assert not f.readable() + + +# ============================================================================ +# Provider-specific tests +# ============================================================================ + + +def test_s3_multipart_upload(rig): + """Test that S3 multipart upload is triggered for large writes.""" + if not hasattr(rig, "s3_path"): + pytest.skip("Not testing S3") + + path = rig.create_cloud_path("test_multipart.bin") + # Write enough data to trigger multiple parts (> 64KB buffer) + large_data = b"X" * (200 * 1024) # 200 KB + + try: + with path.open(mode="wb", buffer_size=64 * 1024) as f: + f.write(large_data) + + # Verify data was uploaded correctly + assert path.read_bytes() == large_data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_azure_block_upload(rig): + """Test that Azure block upload works.""" + if not hasattr(rig, "azure_path"): + pytest.skip("Not testing Azure") + + path = rig.create_cloud_path("test_blocks.bin") + data = b"Block data " * 1000 + + try: + with path.open(mode="wb") as f: + f.write(data) + + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_gs_resumable_upload(rig): + """Test that GCS upload works.""" + if not hasattr(rig, "gs_path"): + pytest.skip("Not testing GCS") + + path = rig.create_cloud_path("test_resumable.bin") + data = b"GCS data " * 1000 + + try: + with path.open(mode="wb") as f: + f.write(data) + + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Performance and efficiency tests +# ============================================================================ + + +def test_small_buffer_many_reads(temp_cloud_binary_file): + """Test reading with small buffer size.""" + with temp_cloud_binary_file.open(mode="rb", buffer_size=128) as f: + data = f.read() + assert data == BINARY_DATA + + +def test_large_buffer_few_reads(temp_cloud_binary_file): + """Test reading with large buffer size.""" + with temp_cloud_binary_file.open(mode="rb", buffer_size=1024 * 1024) as f: + data = f.read() + assert data == BINARY_DATA + + +def test_sequential_reads(temp_cloud_binary_file): + """Test sequential reading pattern.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pos = 0 + while pos < len(BINARY_DATA): + chunk = f.read(100) + if not chunk: + break + assert chunk == BINARY_DATA[pos : pos + 100] + pos += len(chunk) + + +def test_random_seeks(temp_cloud_binary_file): + """Test random seek pattern.""" + positions = [0, 100, 50, 200, 10] + + with temp_cloud_binary_file.open(mode="rb") as f: + for pos in positions: + f.seek(pos) + assert f.tell() == pos + chunk = f.read(10) + assert chunk == BINARY_DATA[pos : pos + 10] + + +# ============================================================================ +# Additional coverage tests for error paths and edge cases +# ============================================================================ + + +def test_readinto_on_closed_file(temp_cloud_binary_file): + """Test readinto on closed file raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pass + + buf = bytearray(100) + with pytest.raises(ValueError, match="closed file"): + f.readinto(buf) + + +def test_read_on_write_only_file(rig): + """Test reading from write-only file raises error.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_only.bin") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + # Try to read from write-only file + with pytest.raises(io.UnsupportedOperation): + f.read() + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_readinto_empty_buffer(temp_cloud_binary_file): + """Test readinto with empty buffer returns 0.""" + with temp_cloud_binary_file.open(mode="rb") as f: + buf = bytearray(0) + n = f.readinto(buf) + assert n == 0 + + +def test_seek_with_invalid_whence(temp_cloud_binary_file): + """Test seek with invalid whence raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + with pytest.raises((ValueError, OSError)): + f.seek(0, 999) # Invalid whence value + + +def test_negative_seek_position(temp_cloud_binary_file): + """Test seeking to negative position raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + with pytest.raises(ValueError, match="negative seek position"): + f.seek(-10, io.SEEK_SET) + + +def test_seek_on_closed_file(temp_cloud_binary_file): + """Test seek on closed file raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pass + + with pytest.raises(ValueError, match="closed file"): + f.seek(0) + + +def test_write_empty_chunks(rig): + """Test that empty write chunks are handled correctly.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_empty_chunks.bin") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + # Write empty data - should be no-op + f.write(b"") + # Write actual data + f.write(b"real data") + # Write more empty data + f.write(b"") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"real data" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_write_error_cleanup(rig): + """Test that write errors are handled gracefully.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + # This test just verifies that writing and closing work correctly + # The error handling paths are tested by the actual upload implementations + path = rig.create_cloud_path("test_error_cleanup.bin") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Write some data successfully + with path.open(mode="wb") as f: + f.write(b"test data") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"test data" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_http_write_empty_file(rig): + """Test HTTP write for empty file.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("Test is specific to HTTP/HTTPS") + + # HTTP writes aren't fully supported in tests, but we can test the code path + # Skip for now since HTTP test server doesn't support PUT + pytest.skip("HTTP write not supported by test server") + + +def test_seek_from_end_without_size(rig, monkeypatch): + """Test SEEK_END when size cannot be determined.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_no_size.bin") + path.write_bytes(b"test data") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Monkeypatch _get_size to raise an error + def mock_get_size(): + raise OSError("Cannot determine size") + + with path.open(mode="rb") as f: + # CloudBufferedIO has a _buffer attribute that wraps the raw IO + # Access the raw IO object through _buffer + raw = f._buffer.raw if hasattr(f, "_buffer") else f.raw + monkeypatch.setattr(raw, "_get_size", mock_get_size) + monkeypatch.setattr(raw, "_size", None) + + # Try to seek from end - should raise error (either from mock or from handler) + with pytest.raises(OSError): + f.seek(-5, io.SEEK_END) + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_read_at_eof_returns_empty(temp_cloud_binary_file): + """Test that reading at EOF returns empty bytes.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek to end + f.seek(0, io.SEEK_END) + # Try to read + data = f.read(100) + assert data == b"" + + +def test_readinto_at_eof_returns_zero(temp_cloud_binary_file): + """Test that readinto at EOF returns 0.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek to end + f.seek(0, io.SEEK_END) + # Try to readinto + buf = bytearray(100) + n = f.readinto(buf) + assert n == 0 + + +def test_fspath_raises_in_streaming_mode(rig): + """Test that fspath raises an error in streaming mode.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_fspath.txt") + path.write_text("test data") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Try to access fspath - should raise error + from cloudpathlib.exceptions import CloudPathNotImplementedError + + with pytest.raises( + CloudPathNotImplementedError, match="fspath is not available in streaming mode" + ): + _ = path.fspath + + # Also test __fspath__ directly + with pytest.raises( + CloudPathNotImplementedError, match="fspath is not available in streaming mode" + ): + _ = path.__fspath__() + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Step 8 regression tests — one test per bug from the plan +# ============================================================================ + + +# H1 — finalize-error propagates (no silent data loss) +def test_finalize_error_propagates(rig): + """A failed upload must raise out of the with-block; silent data loss is not allowed.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_finalize_error.bin") + raw_io_class = path._cloud_meta.raw_io_class + + class _FailingRaw(raw_io_class): + def _finalize_upload(self, upload_state=None): + raise RuntimeError("simulated upload failure") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(RuntimeError, match="simulated upload failure"): + with CloudBufferedIO( + raw_io_class=_FailingRaw, + client=path.client, + cloud_path=path, + mode="wb", + ) as f: + f.write(b"data that should not survive") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H2 — exclusive create raises for 'xb' and 'xt' when object already exists +def test_exclusive_create_xb_raises_when_exists(rig): + """open('xb') must raise CloudPathFileExistsError when the object already exists.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = rig.create_cloud_path("test_exclusive_create.bin") + path.write_bytes(b"existing content") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(CloudPathFileExistsError): + path.open("xb") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +def test_exclusive_create_xt_raises_when_exists(rig): + """open('xt') must also raise CloudPathFileExistsError (mode='x' alone was not enough).""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = rig.create_cloud_path("test_exclusive_create_xt.txt") + path.write_text("existing") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(CloudPathFileExistsError): + path.open("xt") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H2 — append/r+ fall back to cache (correct semantics over streaming) +def test_append_mode_uses_cache_fallback(rig): + """Append mode with streaming file_cache_mode must fall back to the cached path.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_append_fallback.bin") + path.write_bytes(b"hello ") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("ab") as f: + assert not isinstance(f, CloudBufferedIO), "append mode must use cache, not streaming" + f.write(b"world") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"hello world" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +def test_rplus_mode_uses_cache_fallback(rig): + """r+b mode with streaming file_cache_mode must fall back to the cached path.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_rplus_fallback.bin") + path.write_bytes(b"hello world") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("r+b") as f: + assert not isinstance(f, CloudBufferedIO), "r+b must use cache, not streaming" + f.seek(6) + f.write(b"there") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"hello there" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H3 — S3 part-size floor: no non-final part < 5 MiB +def test_s3_no_small_non_final_parts(rig): + """All non-final S3 multipart parts must be >= 5 MiB.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific test") + + from cloudpathlib.s3.s3_io import _S3StorageRaw + + path = rig.create_cloud_path("test_part_size.bin") + data = b"X" * (12 * 1024 * 1024) # 12 MiB → two 5 MiB parts + one 2 MiB final + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + uploaded_parts = [] + real_upload_part = path.client._upload_part + + def spy_upload_part(cloud_path, upload_id, part_number, part_data): + uploaded_parts.append(len(part_data)) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + path.client._upload_part = spy_upload_part + + try: + with path.open("wb") as f: + f.write(data) + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == data + + min_size = _S3StorageRaw._MIN_PART_SIZE + for part_size in uploaded_parts[:-1]: # all except last + assert ( + part_size >= min_size + ), f"Non-final part is {part_size} bytes, below 5 MiB minimum" + finally: + path.client._upload_part = real_upload_part + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H4 — concurrent writes to two paths on one client don't cross buffers +def test_concurrent_writes_dont_cross_buffers(rig): + """Two simultaneous streaming writers must not share upload state.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path_a = rig.create_cloud_path("test_concurrent_a.bin") + path_b = rig.create_cloud_path("test_concurrent_b.bin") + data_a = b"AAAA" * 1024 + data_b = b"BBBB" * 1024 + errors = [] + + def write_path(path, data): + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + with path.open("wb") as f: + f.write(data) + path.client.file_cache_mode = original_mode + except Exception as e: + errors.append(e) + + t_a = threading.Thread(target=write_path, args=(path_a, data_a)) + t_b = threading.Thread(target=write_path, args=(path_b, data_b)) + t_a.start() + t_b.start() + t_a.join() + t_b.join() + + assert not errors, f"Concurrent write errors: {errors}" + + try: + assert path_a.read_bytes() == data_a, "path_a has wrong data (buffer cross)" + assert path_b.read_bytes() == data_b, "path_b has wrong data (buffer cross)" + finally: + for p in (path_a, path_b): + try: + p.unlink() + except Exception: + pass + + +# M6/M7 — custom Client without raw_io_class still instantiates in cached mode +def test_custom_client_without_raw_io_class_instantiates(rig): + """A CloudImplementation with no raw_io_class must not raise IncompleteImplementationError.""" + from cloudpathlib.cloudpath import CloudImplementation + from cloudpathlib.local.localclient import LocalClient + from cloudpathlib.local.localpath import LocalPath + + minimal = CloudImplementation() + minimal._client_class = LocalClient + minimal._path_class = LocalPath + minimal._raw_io_class = None # no streaming + + # Must not raise + minimal.validate_completeness() + assert minimal.raw_io_class is None + + +# M5 — HTTP range reads return the correct slice +def test_http_range_read_returns_correct_bytes(rig): + """HTTP range reads must return exactly the requested byte slice.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("HTTP/HTTPS-specific test") + + path = rig.create_cloud_path("test_range_slice.bin") + content = b"0123456789abcdef" + path.write_bytes(content) + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("rb", buffer_size=4) as f: + chunk = f.read(4) + assert chunk == b"0123", f"Expected first 4 bytes, got {chunk!r}" + chunk2 = f.read(4) + assert chunk2 == b"4567", f"Expected bytes 4-7, got {chunk2!r}" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass diff --git a/tests/test_cloudpath_instantiation.py b/tests/test_cloudpath_instantiation.py index bbdfa3c3..7a9826d1 100644 --- a/tests/test_cloudpath_instantiation.py +++ b/tests/test_cloudpath_instantiation.py @@ -103,14 +103,14 @@ def test_idempotency(rig): def test_dependencies_not_loaded(rig, monkeypatch): - monkeypatch.setattr(rig.path_class._cloud_meta, "dependencies_loaded", False) + monkeypatch.setattr(rig.cloud_implementation, "dependencies_loaded", False) with pytest.raises(MissingDependenciesError): CloudPath(f"{rig.cloud_prefix}{rig.drive}/{rig.test_dir}/dir_0/file0_0.txt") with pytest.raises(MissingDependenciesError): rig.create_cloud_path("dir_0/file0_0.txt") # manual reset for teardown order so teardown doesn't fail - monkeypatch.setattr(rig.path_class._cloud_meta, "dependencies_loaded", True) + monkeypatch.setattr(rig.cloud_implementation, "dependencies_loaded", True) def test_is_pathlike(rig): From 40a83b22d928fa3eca08be2a8e924bcec11c0443 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:35:56 -0700 Subject: [PATCH 02/11] Fix test skip conditions and mypy errors in streaming I/O MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_s3_multipart_upload, test_azure_block_upload, test_gs_resumable_upload were skipping on ALL rigs because hasattr(rig, "s3_path") etc. is always False; switch to rig.path_class.cloud_prefix comparisons so they run on the correct rigs - gs/gsclient.py GCSNotFound fallback: type: ignore[misc, assignment] - azure/azblobclient.py: e.error_code → e.error.code (guarded for None) - gs/gs_io.py: annotate _writer as Optional[Any] to satisfy mypy Co-Authored-By: Claude Sonnet 4.6 --- tests/test_cloud_io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index 69cfd9bc..a2db7144 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -794,7 +794,7 @@ def test_read_write_mode_not_implemented(temp_cloud_file): def test_s3_multipart_upload(rig): """Test that S3 multipart upload is triggered for large writes.""" - if not hasattr(rig, "s3_path"): + if rig.path_class.cloud_prefix != "s3://": pytest.skip("Not testing S3") path = rig.create_cloud_path("test_multipart.bin") @@ -816,7 +816,7 @@ def test_s3_multipart_upload(rig): def test_azure_block_upload(rig): """Test that Azure block upload works.""" - if not hasattr(rig, "azure_path"): + if rig.path_class.cloud_prefix != "az://": pytest.skip("Not testing Azure") path = rig.create_cloud_path("test_blocks.bin") @@ -836,7 +836,7 @@ def test_azure_block_upload(rig): def test_gs_resumable_upload(rig): """Test that GCS upload works.""" - if not hasattr(rig, "gs_path"): + if rig.path_class.cloud_prefix != "gs://": pytest.skip("Not testing GCS") path = rig.create_cloud_path("test_resumable.bin") From b129981cd44e9bf1325c71131d59eb486e27c31c Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:18:48 -0700 Subject: [PATCH 03/11] update tests --- tests/test_cloud_io.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index a2db7144..353362f5 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -1307,6 +1307,49 @@ def spy_upload_part(cloud_path, upload_id, part_number, part_data): pass +# H3b — S3 aborts multipart upload when complete fails +def test_s3_abort_multipart_on_complete_failure(rig): + """If _complete_multipart_upload fails, _abort_multipart_upload is attempted.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific test") + + path = rig.create_cloud_path("test_abort_on_complete_fail.bin") + data = b"X" * (6 * 1024 * 1024) # 6 MiB → one 5 MiB part + final part + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + real_complete = path.client._complete_multipart_upload + real_abort = path.client._abort_multipart_upload + abort_calls = [] + + def fail_complete(cloud_path, upload_id, parts): + raise RuntimeError("complete failed") + + def spy_abort(cloud_path, upload_id): + abort_calls.append(upload_id) + return real_abort(cloud_path, upload_id) + + path.client._complete_multipart_upload = fail_complete + path.client._abort_multipart_upload = spy_abort + + try: + with pytest.raises(RuntimeError, match="complete failed"): + with path.open("wb") as f: + f.write(data) + + assert len(abort_calls) == 1 + assert not path.exists() + finally: + path.client._complete_multipart_upload = real_complete + path.client._abort_multipart_upload = real_abort + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + # H4 — concurrent writes to two paths on one client don't cross buffers def test_concurrent_writes_dont_cross_buffers(rig): """Two simultaneous streaming writers must not share upload state.""" From 5f3612c502a011fae7553c04090ebfd92c2cf194 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:11:26 -0700 Subject: [PATCH 04/11] code review --- cloudpathlib/azure/azure_io.py | 37 ++++++++++++++++++++++++++++------ cloudpathlib/cloudpath.py | 7 +++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py index 213b8286..37f078e1 100644 --- a/cloudpathlib/azure/azure_io.py +++ b/cloudpathlib/azure/azure_io.py @@ -16,15 +16,21 @@ class _AzureBlobStorageRaw(_CloudStorageRaw): Azure Blob Storage-specific raw I/O adapter. Implements efficient range-based reads and block blob uploads for Azure. - Each block is staged independently (true streaming) and committed on finalize. + Writes are accumulated and staged in larger blocks to stay under Azure's + 50,000 committed-block limit per blob. """ + # Target block size before staging (Azure allows up to 50,000 blocks per blob) + _BLOCK_SIZE = 4 * 1024 * 1024 + def __init__(self, client, cloud_path, mode: str = "rb"): super().__init__(client, cloud_path, mode) # Block blob upload state self._upload_id: str = "" # Azure doesn't use upload IDs self._parts: list = [] + self._part_number: int = 1 + self._write_buffer: bytearray = bytearray() def _range_get(self, start: int, end: int) -> bytes: return self._client._range_download(self._cloud_path, start, end) @@ -46,14 +52,32 @@ def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = No if not data: return - if not self._upload_id: - self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + self._write_buffer.extend(data) - part_number = len(self._parts) + 1 - part_info = self._client._upload_part(self._cloud_path, self._upload_id, part_number, data) - self._parts.append(part_info) + while len(self._write_buffer) >= self._BLOCK_SIZE: + chunk = bytes(self._write_buffer[: self._BLOCK_SIZE]) + del self._write_buffer[: self._BLOCK_SIZE] + if not self._upload_id: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + part_info = self._client._upload_part( + self._cloud_path, self._upload_id, self._part_number, chunk + ) + self._parts.append(part_info) + self._part_number += 1 def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + if self._write_buffer: + if not self._upload_id: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + part_info = self._client._upload_part( + self._cloud_path, + self._upload_id, + self._part_number, + bytes(self._write_buffer), + ) + self._parts.append(part_info) + self._write_buffer.clear() + if not self._parts: # No blocks staged — create an empty blob directly self._client._put_empty_object(self._cloud_path) @@ -64,6 +88,7 @@ def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> Non finally: self._upload_id = "" self._parts = [] + self._part_number = 1 def close(self) -> None: super().close() diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index aeada7ec..1bf27263 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -714,6 +714,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "TextIOWrapper": ... @overload @@ -726,6 +727,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "FileIO": ... @overload @@ -738,6 +740,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedRandom": ... @overload @@ -750,6 +753,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedWriter": ... @overload @@ -762,6 +766,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedReader": ... @overload @@ -774,6 +779,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BinaryIO": ... @overload @@ -786,6 +792,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "IO[Any]": ... def open( From 3a42c07c3abb6ca568d8f5009a73991aab9f1e9b Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:15:24 -0700 Subject: [PATCH 05/11] Fix streaming I/O correctness and failure handling --- cloudpathlib/azure/azure_io.py | 28 ++- cloudpathlib/cloud_io.py | 69 +++++++- cloudpathlib/cloudpath.py | 30 +++- cloudpathlib/gs/gs_io.py | 7 + cloudpathlib/gs/gsclient.py | 6 +- cloudpathlib/http/http_io.py | 19 ++- cloudpathlib/http/httpclient.py | 30 ++-- cloudpathlib/s3/s3_io.py | 36 ++-- cloudpathlib/s3/s3client.py | 67 +++++++- docs/docs/streaming_io.md | 46 ++--- docs/mkdocs.yml | 1 + tests/conftest.py | 70 +------- tests/mock_clients/mock_gs.py | 4 + tests/test_cloud_io.py | 291 +++++++++++++++++++++++++++++--- 14 files changed, 541 insertions(+), 163 deletions(-) diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py index 37f078e1..e23495e3 100644 --- a/cloudpathlib/azure/azure_io.py +++ b/cloudpathlib/azure/azure_io.py @@ -22,6 +22,9 @@ class _AzureBlobStorageRaw(_CloudStorageRaw): # Target block size before staging (Azure allows up to 50,000 blocks per blob) _BLOCK_SIZE = 4 * 1024 * 1024 + _MAX_BLOCK_SIZE = 4_000 * 1024 * 1024 + _MAX_BLOCKS = 50_000 + _BLOCKS_PER_SIZE_TIER = 1_000 def __init__(self, client, cloud_path, mode: str = "rb"): super().__init__(client, cloud_path, mode) @@ -48,25 +51,35 @@ def _is_eof_error(self, error: Exception) -> bool: # ---- Write support (Azure block blob upload) ---- + def _target_block_size(self) -> int: + tier = (self._part_number - 1) // self._BLOCKS_PER_SIZE_TIER + return min(self._BLOCK_SIZE * (2**tier), self._MAX_BLOCK_SIZE) + def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: if not data: return self._write_buffer.extend(data) - while len(self._write_buffer) >= self._BLOCK_SIZE: - chunk = bytes(self._write_buffer[: self._BLOCK_SIZE]) - del self._write_buffer[: self._BLOCK_SIZE] + target_block_size = self._target_block_size() + while len(self._write_buffer) >= target_block_size: + if self._part_number > self._MAX_BLOCKS: + raise OSError("Azure block upload exceeded the 50,000-block limit") + chunk = bytes(self._write_buffer[:target_block_size]) if not self._upload_id: self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) part_info = self._client._upload_part( self._cloud_path, self._upload_id, self._part_number, chunk ) + del self._write_buffer[:target_block_size] self._parts.append(part_info) self._part_number += 1 + target_block_size = self._target_block_size() def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: if self._write_buffer: + if self._part_number > self._MAX_BLOCKS: + raise OSError("Azure block upload exceeded the 50,000-block limit") if not self._upload_id: self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) part_info = self._client._upload_part( @@ -90,5 +103,14 @@ def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> Non self._parts = [] self._part_number = 1 + def _abort_upload(self) -> None: + try: + self._client._abort_multipart_upload(self._cloud_path, self._upload_id) + finally: + self._upload_id = "" + self._parts = [] + self._part_number = 1 + self._write_buffer.clear() + def close(self) -> None: super().close() diff --git a/cloudpathlib/cloud_io.py b/cloudpathlib/cloud_io.py index ad63c0cd..242f48a4 100644 --- a/cloudpathlib/cloud_io.py +++ b/cloudpathlib/cloud_io.py @@ -10,6 +10,21 @@ from abc import abstractmethod from typing import Optional, Any, Type, Union, Dict + +def _validate_file_mode(mode: str) -> None: + """Validate an ``open`` mode using the same grammar as the stdlib.""" + if not isinstance(mode, str): + raise TypeError(f"mode must be a string, not {type(mode).__name__}") + if not mode or any(character not in "rwaxbt+" for character in mode): + raise ValueError(f"invalid mode: {mode!r}") + if sum(mode.count(character) for character in "rwax") != 1: + raise ValueError("must have exactly one of create/read/write/append mode") + if mode.count("+") > 1 or mode.count("b") > 1 or mode.count("t") > 1: + raise ValueError(f"invalid mode: {mode!r}") + if "b" in mode and "t" in mode: + raise ValueError("can't have text and binary mode at once") + + # ============================================================================ # Base Raw I/O Adapter (internal) # ============================================================================ @@ -44,6 +59,7 @@ def __init__( self._pos = 0 self._size: Optional[int] = None self._closed = False + self._upload_error: Optional[BaseException] = None def readable(self) -> bool: """Return whether object was opened for reading.""" @@ -177,12 +193,18 @@ def write(self, b: bytes) -> int: # type: ignore[override] Returns: Number of bytes written """ + if self._closed: + raise ValueError("I/O operation on closed file") if not self.writable(): raise io.UnsupportedOperation("not writable") + if self._upload_error is not None: + raise self._upload_error - # Delegate to subclass implementation - # Note: Don't check _closed here because BufferedWriter may call write() during close/flush - self._upload_chunk(bytes(b), None) + try: + self._upload_chunk(bytes(b), None) + except BaseException as error: + self._upload_error = error + raise return len(b) def close(self) -> None: @@ -194,12 +216,30 @@ def close(self) -> None: self._closed = True try: + if self.writable() and self._upload_error is not None: + try: + self._abort_upload() + except Exception: + pass + finally: + raise self._upload_error if self.writable(): - self._finalize_upload(None) + try: + self._finalize_upload(None) + except BaseException: + try: + self._abort_upload() + except Exception: + pass + raise finally: # Always call parent close() to set the stdlib closed state super().close() + def _abort_upload(self) -> None: + """Best-effort cleanup after a write or finalization failure.""" + pass + @abstractmethod def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]]) -> None: """ @@ -290,11 +330,16 @@ def __init__( raw_io_class: The raw I/O class to use for this provider client: Cloud provider client instance cloud_path: CloudPath instance - mode: File mode ('rb', 'wb', 'ab', 'r+b', 'w+b', 'a+b', 'xb') + mode: Streaming file mode ('rb', 'wb', or 'xb') buffer_size: Size of read/write buffer in bytes (default 64 KiB) """ + _validate_file_mode(mode) if "b" not in mode: raise ValueError("CloudBufferedIO requires binary mode (must include 'b')") + if "a" in mode or "+" in mode: + raise io.UnsupportedOperation( + "append and update modes require the local-cache implementation" + ) # Create raw adapter using provided class raw = raw_io_class(client, cloud_path, mode) @@ -411,6 +456,7 @@ def __init__( errors: Optional[str] = None, newline: Optional[str] = None, buffer_size: int = 64 * 1024, + line_buffering: bool = False, ): """ Initialize cloud text I/O. @@ -419,14 +465,20 @@ def __init__( raw_io_class: The raw I/O class to use for this provider client: Cloud provider client instance cloud_path: CloudPath instance - mode: File mode ('rt', 'wt', 'at', 'r+t', 'w+t', 'a+t', 'xt', or same without 't') - encoding: Text encoding (default: utf-8) + mode: Streaming file mode ('rt', 'wt', 'xt', or the same without 't') + encoding: Text encoding (default: platform locale, matching ``open``) errors: Error handling strategy (default: strict) newline: Newline handling (None, '', '\\n', '\\r', '\\r\\n') buffer_size: Size of buffer in bytes + line_buffering: Flush text output whenever a newline is written """ + _validate_file_mode(mode) if "b" in mode: raise ValueError("CloudTextIO requires text mode (no 'b' in mode)") + if "a" in mode or "+" in mode: + raise io.UnsupportedOperation( + "append and update modes require the local-cache implementation" + ) # Ensure mode has 't' or is text mode if "t" not in mode and "r" in mode: @@ -448,9 +500,10 @@ def __init__( # Initialize TextIOWrapper with the buffered stream super().__init__( buffered, - encoding=encoding or "utf-8", + encoding=encoding, errors=errors, newline=newline, + line_buffering=line_buffering, ) # Store additional attributes diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 1bf27263..9024b0fd 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -806,6 +806,23 @@ def open( force_overwrite_to_cloud: Optional[bool] = None, # extra kwarg not in pathlib buffer_size: Optional[int] = None, # extra kwarg for streaming mode ) -> "IO[Any]": + from .cloud_io import _validate_file_mode + + _validate_file_mode(mode) + binary_mode = "b" in mode + if binary_mode and encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + if binary_mode and errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + if binary_mode and newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + if not binary_mode and buffering == 0: + raise ValueError("can't have unbuffered text I/O") + if buffering < -1: + raise ValueError("invalid buffering size") + if buffer_size is not None and buffer_size <= 0: + raise ValueError("buffer_size must be greater than zero") + # if trying to call open on a directory that exists exists_on_cloud = self.exists() @@ -814,12 +831,12 @@ def open( f"Cannot open directory, only files. Tried to open ({self})" ) - if not exists_on_cloud and any(m in mode for m in ("r", "a")): + if not exists_on_cloud and "r" in mode: raise CloudPathFileNotFoundError( - f"File opened for read or append, but it does not exist on cloud: {self}" + f"File opened for read, but it does not exist on cloud: {self}" ) - if "x" in mode and self.exists(): + if "x" in mode and exists_on_cloud: raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.") # Use streaming I/O if file_cache_mode is streaming AND the mode is supported. @@ -840,10 +857,8 @@ def open( # Calculate buffer size from buffering or buffer_size parameter if buffer_size is None: if buffering == 0: - # Unbuffered binary mode - buffer_size = 1 # Minimal buffering - if "b" not in mode: - mode += "b" # Force binary mode for unbuffered + # A raw provider adapter is the streaming equivalent of FileIO. + return raw_io_class(self.client, self, mode) # type: ignore[return-value] elif buffering > 0: buffer_size = buffering else: @@ -868,6 +883,7 @@ def open( errors=errors, newline=newline, buffer_size=buffer_size, + line_buffering=buffering == 1, ) # Standard cached mode diff --git a/cloudpathlib/gs/gs_io.py b/cloudpathlib/gs/gs_io.py index 2cd09d9b..60d91e10 100644 --- a/cloudpathlib/gs/gs_io.py +++ b/cloudpathlib/gs/gs_io.py @@ -58,5 +58,12 @@ def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> Non # No data was written — create an empty object self._client._put_empty_object(self._cloud_path) + def _abort_upload(self) -> None: + if self._writer is not None: + try: + self._writer.terminate() + finally: + self._writer = None + def close(self) -> None: super().close() diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index 5510e1ad..99e1ca6f 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -318,8 +318,8 @@ def _range_download(self, cloud_path: GSPath, start: int, end: int) -> bytes: """Download a byte range from GCS.""" blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) try: - # GCS end is exclusive in the API, our API is inclusive - return blob.download_as_bytes(start=start, end=end + 1) + # GCS and our internal API both use an inclusive end offset. + return blob.download_as_bytes(start=start, end=end, **self.blob_kwargs) except GCSNotFound: raise FileNotFoundError(f"GCS object not found: {cloud_path}") except Exception as e: @@ -334,7 +334,7 @@ def _get_content_length(self, cloud_path: GSPath) -> int: """Get the size of a GCS object.""" blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) try: - blob.reload() + blob.reload(**self.blob_kwargs) return blob.size except GCSNotFound: raise FileNotFoundError(f"GCS object not found: {cloud_path}") diff --git a/cloudpathlib/http/http_io.py b/cloudpathlib/http/http_io.py index 6dc1fb5d..c5af16ab 100644 --- a/cloudpathlib/http/http_io.py +++ b/cloudpathlib/http/http_io.py @@ -4,6 +4,7 @@ Provides streaming I/O for HTTP/HTTPS using range requests and single-PUT uploads. """ +import tempfile from typing import Optional, Dict, Any from ..cloud_io import _CloudStorageRaw @@ -24,7 +25,7 @@ class _HttpStorageRaw(_CloudStorageRaw): def __init__(self, client, cloud_path, mode: str = "rb"): super().__init__(client, cloud_path, mode) - self._upload_buffer: list = [] + self._upload_buffer: Any = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024) def _range_get(self, start: int, end: int) -> bytes: return self._client._range_download(self._cloud_path, start, end) @@ -45,13 +46,19 @@ def _is_eof_error(self, error: Exception) -> bool: def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: if not data: return - self._upload_buffer.append(data) + self._upload_buffer.write(data) def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: - # Concatenate buffered chunks (may be empty for an empty write) - complete_data = b"".join(self._upload_buffer) - self._upload_buffer.clear() - self._client._put_data(self._cloud_path, complete_data) + self._upload_buffer.seek(0, 2) + content_length = self._upload_buffer.tell() + self._upload_buffer.seek(0) + try: + self._client._put_data(self._cloud_path, self._upload_buffer, content_length) + finally: + self._upload_buffer.close() + + def _abort_upload(self) -> None: + self._upload_buffer.close() def close(self) -> None: super().close() diff --git a/cloudpathlib/http/httpclient.py b/cloudpathlib/http/httpclient.py index 379a169b..52807b2e 100644 --- a/cloudpathlib/http/httpclient.py +++ b/cloudpathlib/http/httpclient.py @@ -6,7 +6,7 @@ import urllib.parse import urllib.error from pathlib import Path -from typing import Iterable, Optional, Tuple, Union, Callable +from typing import BinaryIO, Iterable, Optional, Tuple, Union, Callable import shutil import mimetypes import warnings @@ -217,12 +217,13 @@ def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes try: with self.opener.open(request) as response: status = response.status - data = response.read() if status == 206: - return data + return response.read(end - start + 1) elif status == 200: - # Server ignored the Range header; slice locally - return data[start : end + 1] + raise OSError( + f"HTTP server ignored the Range header for {cloud_path}; " + "streaming reads require byte-range support" + ) else: raise OSError(f"Unexpected status {status} for range request on {cloud_path}") except urllib.error.HTTPError as e: @@ -271,16 +272,19 @@ def _abort_multipart_upload(self, cloud_path: "HttpPath", upload_id: str) -> Non """Nothing to abort for HTTP single-PUT uploads.""" pass - def _put_data(self, cloud_path: "HttpPath", data: bytes) -> None: - """Upload data to HTTP server using a PUT request. + def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) -> None: + """Upload a file-like body using the client's configured write method. Uses self.opener so that any SSL context or auth handlers configured on this client are applied (important for HttpsClient with self-signed certs). """ url = str(cloud_path) - request = urllib.request.Request(url, data=data, method="PUT") - request.add_header("Content-Type", "application/octet-stream") - request.add_header("Content-Length", str(len(data))) + request = urllib.request.Request(url, data=data, method=self.write_file_http_method) + content_type = None + if self.content_type_method is not None: + content_type, _ = self.content_type_method(str(cloud_path)) + request.add_header("Content-Type", content_type or "application/octet-stream") + request.add_header("Content-Length", str(content_length)) try: with self.opener.open(request) as response: @@ -290,8 +294,10 @@ def _put_data(self, cloud_path: "HttpPath", data: bytes) -> None: ) except urllib.error.HTTPError as e: if e.code == 405: # Method Not Allowed - raise NotImplementedError(f"HTTP server does not support PUT requests for {url}") - raise OSError(f"HTTP PUT failed: {e}") + raise NotImplementedError( + f"HTTP server does not support {self.write_file_http_method} requests for {url}" + ) + raise OSError(f"HTTP upload failed: {e}") HttpClient.HttpPath = HttpClient.CloudPath # type: ignore diff --git a/cloudpathlib/s3/s3_io.py b/cloudpathlib/s3/s3_io.py index df0b9d50..ae151d0e 100644 --- a/cloudpathlib/s3/s3_io.py +++ b/cloudpathlib/s3/s3_io.py @@ -22,6 +22,9 @@ class _S3StorageRaw(_CloudStorageRaw): # S3 minimum part size for non-final parts: 5 MiB _MIN_PART_SIZE = 5 * 1024 * 1024 + _MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + _MAX_PARTS = 10_000 + _PARTS_PER_SIZE_TIER = 1_000 def __init__(self, client, cloud_path, mode: str = "rb"): super().__init__(client, cloud_path, mode) @@ -43,11 +46,14 @@ def _is_eof_error(self, error: Exception) -> bool: error_str = str(error) return ( "InvalidRange" in error_str - or "InvalidObjectState" in error_str or hasattr(error, "__class__") and "InvalidRange" in error.__class__.__name__ ) + def _target_part_size(self) -> int: + tier = (self._part_number - 1) // self._PARTS_PER_SIZE_TIER + return min(self._MIN_PART_SIZE * (2**tier), self._MAX_PART_SIZE) + # ---- Write support (multipart upload with 5 MiB minimum part size) ---- def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: @@ -57,20 +63,26 @@ def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = No self._write_buffer.extend(data) # Upload full-sized parts whenever we have enough buffered data - while len(self._write_buffer) >= self._MIN_PART_SIZE: - chunk = bytes(self._write_buffer[: self._MIN_PART_SIZE]) - del self._write_buffer[: self._MIN_PART_SIZE] + target_part_size = self._target_part_size() + while len(self._write_buffer) >= target_part_size: + if self._part_number > self._MAX_PARTS: + raise OSError("S3 multipart upload exceeded the 10,000-part limit") + chunk = bytes(self._write_buffer[:target_part_size]) if self._upload_id is None: self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) part_info = self._client._upload_part( self._cloud_path, self._upload_id, self._part_number, chunk ) + del self._write_buffer[:target_part_size] self._parts.append(part_info) self._part_number += 1 + target_part_size = self._target_part_size() def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: # Flush remaining buffer as the final part (exempt from 5 MiB floor) if self._write_buffer: + if self._part_number > self._MAX_PARTS: + raise OSError("S3 multipart upload exceeded the 10,000-part limit") if self._upload_id is None: self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) part_info = self._client._upload_part( @@ -88,19 +100,21 @@ def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> Non self._client._put_empty_object(self._cloud_path) return + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) + self._upload_id = None + self._parts = [] + self._part_number = 1 + self._write_buffer.clear() + + def _abort_upload(self) -> None: try: - self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) - except Exception: - try: + if self._upload_id is not None: self._client._abort_multipart_upload(self._cloud_path, self._upload_id) - except Exception: - pass # best-effort abort - raise finally: self._upload_id = None self._parts = [] self._part_number = 1 - self._write_buffer = bytearray() + self._write_buffer.clear() def close(self) -> None: super().close() diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 53f9b590..9cd6730c 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -445,9 +445,66 @@ def _get_content_length(self, cloud_path: S3Path) -> int: raise FileNotFoundError(f"S3 object not found: {cloud_path}") raise + def _streaming_extra_args(self, operation_name: str) -> dict: + """Return upload extras accepted by a specific low-level S3 operation.""" + try: + operation = self.client.meta.service_model.operation_model(operation_name) + allowed = set(operation.input_shape.members) + except AttributeError: + # The test client intentionally implements only a small boto3 surface. + fallback_allowed = { + "CreateMultipartUpload": { + "ACL", + "CacheControl", + "ChecksumAlgorithm", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation", + }, + "UploadPart": { + "ChecksumAlgorithm", + "ExpectedBucketOwner", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + }, + "CompleteMultipartUpload": { + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ExpectedBucketOwner", + "MpuObjectSize", + "RequestPayer", + }, + "PutObject": set(self.boto3_ul_extra_args), + } + allowed = fallback_allowed[operation_name] + return {key: value for key, value in self.boto3_ul_extra_args.items() if key in allowed} + def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: """Start an S3 multipart upload, threading content-type and upload extra args.""" - extra_args = self.boto3_ul_extra_args.copy() + extra_args = self._streaming_extra_args("CreateMultipartUpload") if self.content_type_method is not None: content_type, content_encoding = self.content_type_method(str(cloud_path)) if content_type is not None: @@ -471,8 +528,11 @@ def _upload_part( UploadId=upload_id, PartNumber=part_number, Body=data, + **self._streaming_extra_args("UploadPart"), ) - return {"PartNumber": part_number, "ETag": response["ETag"]} + part = {"PartNumber": part_number, "ETag": response["ETag"]} + part.update({key: value for key, value in response.items() if key.startswith("Checksum")}) + return part def _complete_multipart_upload(self, cloud_path: S3Path, upload_id: str, parts: list) -> None: """Complete an S3 multipart upload.""" @@ -481,6 +541,7 @@ def _complete_multipart_upload(self, cloud_path: S3Path, upload_id: str, parts: Key=cloud_path.key, UploadId=upload_id, MultipartUpload={"Parts": parts}, + **self._streaming_extra_args("CompleteMultipartUpload"), ) def _abort_multipart_upload(self, cloud_path: S3Path, upload_id: str) -> None: @@ -491,7 +552,7 @@ def _abort_multipart_upload(self, cloud_path: S3Path, upload_id: str) -> None: def _put_empty_object(self, cloud_path: S3Path) -> None: """Upload a zero-byte object, threading content-type and upload extra args.""" - extra_args = self.boto3_ul_extra_args.copy() + extra_args = self._streaming_extra_args("PutObject") if self.content_type_method is not None: content_type, content_encoding = self.content_type_method(str(cloud_path)) if content_type is not None: diff --git a/docs/docs/streaming_io.md b/docs/docs/streaming_io.md index e17156b5..e0fcddf6 100644 --- a/docs/docs/streaming_io.md +++ b/docs/docs/streaming_io.md @@ -1,6 +1,6 @@ # Streaming I/O -CloudPathLib provides high-performance streaming I/O capabilities for cloud storage that work seamlessly with Python's standard I/O interfaces and third-party libraries. +cloudpathlib provides streaming I/O capabilities for cloud storage through Python's standard I/O interfaces. ## Overview @@ -86,8 +86,8 @@ with path.open("rb") as f: ## API Reference -!!! important "Always use `CloudPath.open()`" - The recommended way to use streaming I/O is through `CloudPath.open()` with `FileCacheMode.streaming`. The `CloudBufferedIO` and `CloudTextIO` classes are implementation details returned by `open()` and should not be instantiated directly. +!!! tip "Prefer `CloudPath.open()`" + The usual entry point is `CloudPath.open()` with `FileCacheMode.streaming`. `CloudBufferedIO` and `CloudTextIO` are also public for integrations that need to construct a provider-backed file object directly. ### `FileCacheMode` Enum @@ -128,8 +128,8 @@ CloudPath.open( **Parameters:** - `mode`: File mode - binary (`'rb'`, `'wb'`, etc.) or text (`'r'`, `'w'`, `'rt'`, `'wt'`, etc.) -- `buffering`: Buffer size (deprecated, use `buffer_size` instead) -- `encoding`: Text encoding (default: `"utf-8"`, text mode only) +- `buffering`: Standard Python buffering control. Binary mode supports `0` for an unbuffered raw stream. +- `encoding`: Text encoding (default: platform locale, text mode only) - `errors`: Error handling strategy (default: `"strict"`, text mode only) - `newline`: Newline handling (text mode only) - `buffer_size`: Size of read/write buffer in bytes (default: 64 KiB) @@ -144,8 +144,8 @@ CloudPath.open( Binary file-like object implementing `io.BufferedIOBase`. -!!! note "Use `CloudPath.open()` instead" - **Do not instantiate `CloudBufferedIO` directly.** Always use `CloudPath.open()` with the appropriate mode and `FileCacheMode.streaming` to get streaming file objects. The streaming I/O classes are implementation details that are returned by `open()`. +!!! note "Usually returned by `CloudPath.open()`" + Most applications should let `CloudPath.open()` construct this class. Direct construction is supported when implementing file-object integrations. **Key Methods:** @@ -168,14 +168,14 @@ Binary file-like object implementing `io.BufferedIOBase`. - `readable()`: Returns True for read modes - `writable()`: Returns True for write modes -- `seekable()`: Returns True (random access supported) +- `seekable()`: Returns `True` for readable streams. Streaming writes are sequential and return `False`. ### `CloudTextIO` Text file-like object implementing `io.TextIOBase`. -!!! note "Use `CloudPath.open()` instead" - **Do not instantiate `CloudTextIO` directly.** Always use `CloudPath.open()` with text mode (e.g., `"r"`, `"rt"`, `"w"`, `"wt"`) and `FileCacheMode.streaming` to get streaming text file objects. The streaming I/O classes are implementation details that are returned by `open()`. +!!! note "Usually returned by `CloudPath.open()`" + Most applications should let `CloudPath.open()` construct this class. Direct construction is supported when implementing file-object integrations. **Key Methods:** @@ -246,7 +246,7 @@ path = S3Path("s3://bucket/data.bin", client=client) with path.open("rb") as f: header = f.read(1024) # Read first 1KB parse_header(header) - + # Seek to specific position f.seek(10000) chunk = f.read(100) @@ -370,8 +370,8 @@ For write operations, the streaming I/O system automatically handles: - **S3**: Multipart upload; non-final parts are buffered until they reach the provider minimum of **5 MiB** (S3 rejects smaller non-final parts). The final part may be smaller than 5 MiB. -- **Azure**: Block blob staging — each flushed chunk is staged as a block - and committed on close. +- **Azure**: Block blob staging — blocks grow adaptively during very large uploads + and are committed on close. - **GCS**: Resumable upload (`blob.open("wb")`) — data streams incrementally to GCS without in-memory buffering. @@ -414,7 +414,7 @@ with path.open("rt") as f: ### Google Cloud Storage - Uses GCS SDK `download_as_bytes()` with start/end for reads -- Uses `upload_from_string()` for writes +- Uses a resumable `blob.open("wb")` stream for writes - Supports GCS-specific features through client configuration ```python @@ -428,11 +428,17 @@ with path.open("rt") as f: content = f.read() ``` +### HTTP and HTTPS + +- Requires servers to honor byte-range requests for streaming reads +- Uses the client's configured `write_file_http_method` for writes +- Spools single-request upload bodies with bounded memory, using a temporary file above 8 MiB + ## Comparison with Cached Mode | Feature | Streaming (`FileCacheMode.streaming`) | Cached (default) | |---------|--------------------------------------|------------------| -| **Disk usage** | Minimal (only buffer) | Full file size | +| **Disk usage** | None for cloud providers; HTTP uploads may use a temporary spool | Full file size | | **Memory usage** | Configurable buffer | Varies | | **Read performance** | Sequential: Good
Random: Moderate | Fast (local disk) | | **Write performance** | Good (direct upload) | Fast write, slower close | @@ -660,14 +666,12 @@ Pass custom clients with specific configurations: ```python from cloudpathlib import S3Path, S3Client from cloudpathlib.enums import FileCacheMode -from botocore.config import Config - -# Custom S3 client with retry configuration +# Custom S3-compatible endpoint and upload metadata client = S3Client( file_cache_mode=FileCacheMode.streaming, - boto3_config=Config( - retries={'max_attempts': 10, 'mode': 'adaptive'} - ) + endpoint_url="https://objects.example.com", + addressing_style="path", + extra_args={"ServerSideEncryption": "AES256"}, ) path = S3Path("s3://bucket/file.txt", client=client) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index cd917ce3..1fcbd3d7 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -21,6 +21,7 @@ nav: - AnyPath: "anypath-polymorphism.md" - HTTP URLs: "http.md" - Caching: "caching.ipynb" + - Streaming I/O: "streaming_io.md" - Compatibility: "patching_builtins.ipynb" - Other Client settings: "other_client_settings.md" - Testing code that uses cloudpathlib: "testing_mocked_cloudpathlib.ipynb" diff --git a/tests/conftest.py b/tests/conftest.py index e35c6016..dddb0a4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,23 +21,14 @@ from shortuuid import uuid from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed -from cloudpathlib.azure import AzureBlobClient, AzureBlobPath, _AzureBlobStorageRaw -from cloudpathlib.gs import GSClient, GSPath, _GSStorageRaw -from cloudpathlib.s3 import S3Client, S3Path, _S3StorageRaw from cloudpathlib.cloudpath import implementation_registry, CloudImplementation -from cloudpathlib.http.httpclient import HttpClient, HttpsClient -from cloudpathlib.http.httppath import HttpPath, HttpsPath -from cloudpathlib.http.http_io import _HttpStorageRaw from cloudpathlib.local import ( local_azure_blob_implementation, LocalAzureBlobClient, - LocalAzureBlobPath, local_gs_implementation, LocalGSClient, - LocalGSPath, local_s3_implementation, LocalS3Client, - LocalS3Path, ) import cloudpathlib.azure.azblobclient from cloudpathlib.azure.azblobclient import _hns_rmtree @@ -213,13 +204,8 @@ def _azure_fixture(conn_str_env_var, adls_gen2, request, monkeypatch, assets_dir MockedDataLakeServiceClient, ) - azure_blob_implementation = CloudImplementation() - azure_blob_implementation._client_class = AzureBlobClient - azure_blob_implementation._path_class = AzureBlobPath - azure_blob_implementation._raw_io_class = _AzureBlobStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=azure_blob_implementation, + cloud_implementation=implementation_registry["azure"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -301,13 +287,8 @@ def gs_rig(request, monkeypatch, assets_dir, live_server): ) monkeypatch.setattr(cloudpathlib.gs.gsclient, "google_default_auth", mock_default_auth) - gs_implementation = CloudImplementation() - gs_implementation._client_class = GSClient - gs_implementation._path_class = GSPath - gs_implementation._raw_io_class = _GSStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=gs_implementation, + cloud_implementation=implementation_registry["gs"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -355,13 +336,8 @@ def s3_rig(request, monkeypatch, assets_dir, live_server): mocked_session_class_factory(test_dir), ) - s3_implementation = CloudImplementation() - s3_implementation._client_class = S3Client - s3_implementation._path_class = S3Path - s3_implementation._raw_io_class = _S3StorageRaw - rig = CloudProviderTestRig( - cloud_implementation=s3_implementation, + cloud_implementation=implementation_registry["s3"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -443,13 +419,8 @@ def _spin_up_bucket(): mocked_session_class_factory(test_dir), ) - custom_s3_implementation = CloudImplementation() - custom_s3_implementation._client_class = S3Client - custom_s3_implementation._path_class = S3Path - custom_s3_implementation._raw_io_class = _S3StorageRaw - rig = CloudProviderTestRig( - cloud_implementation=custom_s3_implementation, + cloud_implementation=implementation_registry["s3"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -486,13 +457,8 @@ def local_azure_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "azure", local_azure_blob_implementation) - local_azure_blob_cloud_implementation = CloudImplementation() - local_azure_blob_cloud_implementation._client_class = LocalAzureBlobClient - local_azure_blob_cloud_implementation._path_class = LocalAzureBlobPath - local_azure_blob_cloud_implementation._raw_io_class = _AzureBlobStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=local_azure_blob_cloud_implementation, + cloud_implementation=local_azure_blob_implementation, drive=drive, test_dir=test_dir, ) @@ -521,13 +487,8 @@ def local_gs_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "gs", local_gs_implementation) - local_gs_cloud_implementation = CloudImplementation() - local_gs_cloud_implementation._client_class = LocalGSClient - local_gs_cloud_implementation._path_class = LocalGSPath - local_gs_cloud_implementation._raw_io_class = _GSStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=local_gs_cloud_implementation, + cloud_implementation=local_gs_implementation, drive=drive, test_dir=test_dir, ) @@ -555,11 +516,6 @@ def local_s3_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "s3", local_s3_implementation) - local_s3_cloud_implementation = CloudImplementation() - local_s3_cloud_implementation._client_class = LocalS3Client - local_s3_cloud_implementation._path_class = LocalS3Path - local_s3_cloud_implementation._raw_io_class = _S3StorageRaw - rig = CloudProviderTestRig( cloud_implementation=local_s3_implementation, drive=drive, @@ -598,13 +554,8 @@ def http_rig(request, assets_dir, http_server): # noqa: F811 shutil.copytree(assets_dir, server_dir / test_dir) _sync_filesystem() - http_implementation = CloudImplementation() - http_implementation._client_class = HttpClient - http_implementation._path_class = HttpPath - http_implementation._raw_io_class = _HttpStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=http_implementation, + cloud_implementation=implementation_registry["http"], drive=drive, test_dir=test_dir, ) @@ -634,13 +585,8 @@ def https_rig(request, assets_dir, https_server): # noqa: F811 skip_verify_ctx.check_hostname = False skip_verify_ctx.load_verify_locations(utilities_dir / "insecure-test.pem") - https_implementation = CloudImplementation() - https_implementation._client_class = HttpsClient - https_implementation._path_class = HttpsPath - https_implementation._raw_io_class = _HttpStorageRaw - rig = CloudProviderTestRig( - cloud_implementation=https_implementation, + cloud_implementation=implementation_registry["https"], drive=drive, test_dir=test_dir, required_client_kwargs=dict( diff --git a/tests/mock_clients/mock_gs.py b/tests/mock_clients/mock_gs.py index d56bb9ee..165034ba 100644 --- a/tests/mock_clients/mock_gs.py +++ b/tests/mock_clients/mock_gs.py @@ -202,6 +202,10 @@ def close(self) -> None: self._closed = True self._blob.upload_from_string(bytes(self._buf)) + def terminate(self) -> None: + self._closed = True + self._buf.clear() + def __enter__(self): return self diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index 353362f5..56517724 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -195,7 +195,7 @@ def test_buffered_io_context_manager(temp_cloud_binary_file): def test_write_binary_stream(rig): """Test writing binary data via streaming.""" # Skip if streaming IO is not implemented for this provider - if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") path = rig.create_cloud_path("test_write_binary.bin") @@ -230,7 +230,7 @@ def test_write_binary_stream(rig): def test_write_chunks(rig): """Test writing data in chunks.""" # Skip if streaming IO is not implemented for this provider - if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") path = rig.create_cloud_path("test_write_chunks.bin") @@ -389,7 +389,7 @@ def test_text_properties(temp_cloud_file): def test_write_text_stream(rig): """Test writing text data via streaming.""" # Skip if streaming IO is not implemented for this provider - if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") path = rig.create_cloud_path("test_write_text.txt") @@ -655,7 +655,7 @@ def test_empty_file_read(rig): def test_empty_file_write(rig): """Test writing an empty file.""" # Skip if streaming IO is not implemented for this provider - if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") path = rig.create_cloud_path("test_empty_write.txt") @@ -996,25 +996,42 @@ def test_write_empty_chunks(rig): def test_write_error_cleanup(rig): - """Test that write errors are handled gracefully.""" - if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): - pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + """A failed part upload must abort rather than commit earlier parts.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific failure injection") - # This test just verifies that writing and closing work correctly - # The error handling paths are tested by the actual upload implementations path = rig.create_cloud_path("test_error_cleanup.bin") + original_upload_part = path.client._upload_part + original_abort = path.client._abort_multipart_upload + abort_calls = [] + + def fail_second_part(cloud_path, upload_id, part_number, data): + if part_number == 2: + raise RuntimeError("simulated part failure") + return original_upload_part(cloud_path, upload_id, part_number, data) + + def record_abort(cloud_path, upload_id): + abort_calls.append(upload_id) + return original_abort(cloud_path, upload_id) try: original_mode = path.client.file_cache_mode path.client.file_cache_mode = FileCacheMode.streaming + path.client._upload_part = fail_second_part + path.client._abort_multipart_upload = record_abort - # Write some data successfully - with path.open(mode="wb") as f: - f.write(b"test data") + stream = path.open(mode="wb") + with pytest.raises(RuntimeError, match="simulated part failure"): + stream.write(b"x" * (11 * 1024 * 1024)) + with pytest.raises(RuntimeError, match="simulated part failure"): + stream.close() - path.client.file_cache_mode = original_mode - assert path.read_bytes() == b"test data" + assert len(abort_calls) == 1 + assert not path.exists() finally: + path.client._upload_part = original_upload_part + path.client._abort_multipart_upload = original_abort + path.client.file_cache_mode = original_mode try: path.unlink() except Exception: @@ -1026,9 +1043,17 @@ def test_http_write_empty_file(rig): if rig.path_class.cloud_prefix not in ("http://", "https://"): pytest.skip("Test is specific to HTTP/HTTPS") - # HTTP writes aren't fully supported in tests, but we can test the code path - # Skip for now since HTTP test server doesn't support PUT - pytest.skip("HTTP write not supported by test server") + path = rig.create_cloud_path("test_http_empty.bin") + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("wb"): + pass + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"" + finally: + path.client.file_cache_mode = original_mode + path.unlink(missing_ok=True) def test_seek_from_end_without_size(rig, monkeypatch): @@ -1236,6 +1261,17 @@ def test_append_mode_uses_cache_fallback(rig): pass +def test_append_mode_creates_missing_file(local_s3_rig): + path = local_s3_rig.create_cloud_path("new-append.txt") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("a") as stream: + stream.write("created") + + path.client.file_cache_mode = FileCacheMode.cloudpath_object + assert path.read_text() == "created" + + def test_rplus_mode_uses_cache_fallback(rig): """r+b mode with streaming file_cache_mode must fall back to the cached path.""" if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): @@ -1393,20 +1429,24 @@ def write_path(path, data): # M6/M7 — custom Client without raw_io_class still instantiates in cached mode -def test_custom_client_without_raw_io_class_instantiates(rig): - """A CloudImplementation with no raw_io_class must not raise IncompleteImplementationError.""" +def test_custom_client_without_raw_io_class_instantiates(local_s3_rig, monkeypatch): + """A cached custom provider need not implement the optional streaming hooks.""" from cloudpathlib.cloudpath import CloudImplementation - from cloudpathlib.local.localclient import LocalClient - from cloudpathlib.local.localpath import LocalPath minimal = CloudImplementation() - minimal._client_class = LocalClient - minimal._path_class = LocalPath - minimal._raw_io_class = None # no streaming + minimal.name = "minimal" + minimal._client_class = local_s3_rig.client_class + minimal._path_class = local_s3_rig.path_class + minimal._raw_io_class = None + monkeypatch.setattr(local_s3_rig.path_class, "_cloud_meta", minimal) - # Must not raise - minimal.validate_completeness() - assert minimal.raw_io_class is None + path = local_s3_rig.create_cloud_path("no-raw-io.txt") + path.write_text("cached") + assert path.read_text() == "cached" + + path.client.file_cache_mode = FileCacheMode.streaming + with pytest.raises(NotImplementedError, match="Streaming I/O is not implemented"): + path.open("r") # M5 — HTTP range reads return the correct slice @@ -1434,3 +1474,200 @@ def test_http_range_read_returns_correct_bytes(rig): path.unlink() except Exception: pass + + +@pytest.mark.parametrize("mode", ["", "rw", "rr", "r++", "rbt", "q"]) +def test_streaming_open_rejects_invalid_modes_without_mutating(local_s3_rig, mode): + path = local_s3_rig.create_cloud_path("invalid-mode.txt") + path.write_text("preserve me") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError): + path.open(mode) + + path.client.file_cache_mode = FileCacheMode.cloudpath_object + assert path.read_text() == "preserve me" + + +@pytest.mark.parametrize( + "keyword,value,message", + [ + ("encoding", "utf-8", "encoding"), + ("errors", "ignore", "errors"), + ("newline", "", "newline"), + ], +) +def test_streaming_binary_mode_rejects_text_arguments(local_s3_rig, keyword, value, message): + path = local_s3_rig.create_cloud_path("binary-arguments.bin") + path.write_bytes(b"data") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError, match=message): + path.open("rb", **{keyword: value}) + + +def test_streaming_text_mode_rejects_unbuffered_io(local_s3_rig): + path = local_s3_rig.create_cloud_path("unbuffered.txt") + path.write_text("data") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError, match="unbuffered text"): + path.open("r", buffering=0) + + +def test_streaming_text_mode_honors_line_buffering(local_s3_rig): + path = local_s3_rig.create_cloud_path("line-buffered.txt") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("w", buffering=1) as stream: + assert stream.line_buffering + stream.write("line\n") + + +def test_streaming_binary_mode_supports_unbuffered_io(local_s3_rig): + path = local_s3_rig.create_cloud_path("unbuffered.bin") + path.write_bytes(b"data") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("rb", buffering=0) as stream: + assert isinstance(stream, io.RawIOBase) + assert stream.read() == b"data" + + +def test_s3_streaming_routes_extra_args_by_operation(s3_rig): + path = s3_rig.create_cloud_path("streaming-extra-args.bin") + client = path.client + original_extra_args = client.boto3_ul_extra_args + original_create = client.client.create_multipart_upload + original_upload = client.client.upload_part + original_complete = client.client.complete_multipart_upload + calls = {} + + def record_create(**kwargs): + calls["create"] = kwargs.copy() + return original_create(**kwargs) + + def record_upload(**kwargs): + calls["upload"] = kwargs.copy() + return original_upload(**kwargs) + + def record_complete(**kwargs): + calls["complete"] = kwargs.copy() + return original_complete(**kwargs) + + client.boto3_ul_extra_args = { + "ChecksumCRC32": "whole-object-checksum", + "SSECustomerAlgorithm": "AES256", + "SSECustomerKey": "secret", + } + client.client.create_multipart_upload = record_create + client.client.upload_part = record_upload + client.client.complete_multipart_upload = record_complete + client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("wb") as stream: + stream.write(b"x" * (6 * 1024 * 1024)) + + assert "ChecksumCRC32" not in calls["create"] + assert calls["create"]["SSECustomerKey"] == "secret" + assert calls["upload"]["SSECustomerKey"] == "secret" + assert calls["complete"]["ChecksumCRC32"] == "whole-object-checksum" + finally: + client.boto3_ul_extra_args = original_extra_args + client.client.create_multipart_upload = original_create + client.client.upload_part = original_upload + client.client.complete_multipart_upload = original_complete + path.unlink(missing_ok=True) + + +def test_gs_streaming_range_is_inclusive_and_forwards_options(gs_rig, monkeypatch): + from tests.mock_clients.mock_gs import MockBlob + + path = gs_rig.create_cloud_path("range-options.bin") + path.write_bytes(b"0123456789") + calls = {} + + def record_download(self, start=None, end=None, **kwargs): + calls.update(start=start, end=end, **kwargs) + return b"2345" + + monkeypatch.setattr(MockBlob, "download_as_bytes", record_download) + retry = object() + original_kwargs = path.client.blob_kwargs + path.client.blob_kwargs = {"timeout": 12, "retry": retry} + try: + assert path.client._range_download(path, 2, 5) == b"2345" + assert calls == {"start": 2, "end": 5, "timeout": 12, "retry": retry} + finally: + path.client.blob_kwargs = original_kwargs + + +def test_http_streaming_rejects_servers_that_ignore_ranges(http_rig, monkeypatch): + class FullResponse(io.BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def ignore_range(request): + assert request.headers["Range"] == "bytes=2-5" + return FullResponse(b"0123456789") + + path = http_rig.create_cloud_path("ignored-range.bin") + monkeypatch.setattr(path.client.opener, "open", ignore_range) + + with pytest.raises(OSError, match="ignored the Range header"): + path.client._range_download(path, 2, 5) + + +def test_http_streaming_upload_uses_client_configuration(http_rig, monkeypatch): + class CreatedResponse: + status = 201 + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + calls = {} + + def record_request(request): + calls["request"] = request + calls["body"] = request.data.read() + return CreatedResponse() + + path = http_rig.create_cloud_path("configured.txt") + path.client.write_file_http_method = "PATCH" + monkeypatch.setattr(path.client.opener, "open", record_request) + path.client._put_data(path, io.BytesIO(b"abc"), 3) + + request = calls["request"] + assert request.method == "PATCH" + assert request.headers["Content-type"] == "text/plain" + assert request.headers["Content-length"] == "3" + assert calls["body"] == b"abc" + + +def test_provider_part_sizes_grow_for_large_streams(): + from cloudpathlib.azure.azure_io import _AzureBlobStorageRaw + from cloudpathlib.s3.s3_io import _S3StorageRaw + + s3_raw = object.__new__(_S3StorageRaw) + s3_raw._closed = True + s3_raw._part_number = s3_raw._PARTS_PER_SIZE_TIER + 1 + assert s3_raw._target_part_size() == 2 * s3_raw._MIN_PART_SIZE + + azure_raw = object.__new__(_AzureBlobStorageRaw) + azure_raw._closed = True + azure_raw._part_number = azure_raw._BLOCKS_PER_SIZE_TIER + 1 + assert azure_raw._target_block_size() == 2 * azure_raw._BLOCK_SIZE + + +def test_s3_invalid_object_state_is_not_eof(s3_rig): + path = s3_rig.create_cloud_path("archived.bin") + raw = path._cloud_meta.raw_io_class(path.client, path, "rb") + assert not raw._is_eof_error(Exception("InvalidObjectState")) From 741424ba1ccd20e23ecbec235e48517ba3afd8ef Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:21:40 -0700 Subject: [PATCH 06/11] Fix adaptive part size test portability --- cloudpathlib/azure/azure_io.py | 8 ++++++-- cloudpathlib/s3/s3_io.py | 8 ++++++-- tests/test_cloud_io.py | 18 +++++++++--------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py index e23495e3..8bcccaa4 100644 --- a/cloudpathlib/azure/azure_io.py +++ b/cloudpathlib/azure/azure_io.py @@ -51,9 +51,13 @@ def _is_eof_error(self, error: Exception) -> bool: # ---- Write support (Azure block blob upload) ---- + @classmethod + def _block_size_for_number(cls, block_number: int) -> int: + tier = (block_number - 1) // cls._BLOCKS_PER_SIZE_TIER + return min(cls._BLOCK_SIZE * (2**tier), cls._MAX_BLOCK_SIZE) + def _target_block_size(self) -> int: - tier = (self._part_number - 1) // self._BLOCKS_PER_SIZE_TIER - return min(self._BLOCK_SIZE * (2**tier), self._MAX_BLOCK_SIZE) + return self._block_size_for_number(self._part_number) def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: if not data: diff --git a/cloudpathlib/s3/s3_io.py b/cloudpathlib/s3/s3_io.py index ae151d0e..9d7d0509 100644 --- a/cloudpathlib/s3/s3_io.py +++ b/cloudpathlib/s3/s3_io.py @@ -50,9 +50,13 @@ def _is_eof_error(self, error: Exception) -> bool: and "InvalidRange" in error.__class__.__name__ ) + @classmethod + def _part_size_for_number(cls, part_number: int) -> int: + tier = (part_number - 1) // cls._PARTS_PER_SIZE_TIER + return min(cls._MIN_PART_SIZE * (2**tier), cls._MAX_PART_SIZE) + def _target_part_size(self) -> int: - tier = (self._part_number - 1) // self._PARTS_PER_SIZE_TIER - return min(self._MIN_PART_SIZE * (2**tier), self._MAX_PART_SIZE) + return self._part_size_for_number(self._part_number) # ---- Write support (multipart upload with 5 MiB minimum part size) ---- diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index 56517724..b4637a69 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -1656,15 +1656,15 @@ def test_provider_part_sizes_grow_for_large_streams(): from cloudpathlib.azure.azure_io import _AzureBlobStorageRaw from cloudpathlib.s3.s3_io import _S3StorageRaw - s3_raw = object.__new__(_S3StorageRaw) - s3_raw._closed = True - s3_raw._part_number = s3_raw._PARTS_PER_SIZE_TIER + 1 - assert s3_raw._target_part_size() == 2 * s3_raw._MIN_PART_SIZE - - azure_raw = object.__new__(_AzureBlobStorageRaw) - azure_raw._closed = True - azure_raw._part_number = azure_raw._BLOCKS_PER_SIZE_TIER + 1 - assert azure_raw._target_block_size() == 2 * azure_raw._BLOCK_SIZE + assert ( + _S3StorageRaw._part_size_for_number(_S3StorageRaw._PARTS_PER_SIZE_TIER + 1) + == 2 * _S3StorageRaw._MIN_PART_SIZE + ) + + assert ( + _AzureBlobStorageRaw._block_size_for_number(_AzureBlobStorageRaw._BLOCKS_PER_SIZE_TIER + 1) + == 2 * _AzureBlobStorageRaw._BLOCK_SIZE + ) def test_s3_invalid_object_state_is_not_eof(s3_rig): From b125231ecf93a77a88a31b9026d6ebb2421edfd7 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:47:50 -0700 Subject: [PATCH 07/11] Refactor streaming I/O abstractions --- cloudpathlib/azure/azblobclient.py | 53 ++--- cloudpathlib/azure/azure_io.py | 125 ++--------- cloudpathlib/client.py | 119 +++++------ cloudpathlib/cloud_io.py | 332 ++++++++++++----------------- cloudpathlib/gs/gs_io.py | 62 ++---- cloudpathlib/gs/gsclient.py | 15 +- cloudpathlib/http/http_io.py | 64 ++---- cloudpathlib/http/httpclient.py | 44 +--- cloudpathlib/local/localclient.py | 69 +++--- cloudpathlib/s3/s3_io.py | 123 +---------- cloudpathlib/s3/s3client.py | 32 ++- tests/test_cloud_io.py | 2 +- 12 files changed, 328 insertions(+), 712 deletions(-) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index f4618632..720cc912 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -3,7 +3,7 @@ import os from http import HTTPStatus from pathlib import Path -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union from itertools import islice try: @@ -11,7 +11,7 @@ except ImportError: from typing_extensions import cast -from ..client import Client, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode from ..exceptions import MissingCredentialsError @@ -497,8 +497,6 @@ def _generate_presigned_url( url = f"{self._get_public_url(cloud_path)}?{sas_token}" return url - # ====================== STREAMING I/O METHODS ====================== - def _range_download(self, cloud_path: AzureBlobPath, start: int, end: int) -> bytes: """Download a byte range from Azure Blob Storage.""" blob_client = self.service_client.get_blob_client( @@ -527,48 +525,46 @@ def _get_content_length(self, cloud_path: AzureBlobPath) -> int: raise FileNotFoundError(f"Azure blob not found: {cloud_path}") def _initiate_multipart_upload(self, cloud_path: AzureBlobPath) -> str: - """Start an Azure block blob upload. - - Azure doesn't need explicit initialization; return empty string. - """ + """Return the stateless Azure upload ID.""" return "" def _upload_part( self, cloud_path: AzureBlobPath, upload_id: str, part_number: int, data: bytes - ) -> dict: + ) -> _UploadPart: """Upload a block in an Azure block blob upload.""" import base64 blob_client = self.service_client.get_blob_client( container=cloud_path.container, blob=cloud_path.blob ) - # Azure uses base64-encoded block IDs block_id = base64.b64encode(f"block-{part_number:06d}".encode()).decode() blob_client.stage_block(block_id=block_id, data=data, length=len(data)) return {"block_id": block_id} def _complete_multipart_upload( - self, cloud_path: AzureBlobPath, upload_id: str, parts: list + self, cloud_path: AzureBlobPath, upload_id: str, parts: Sequence[_UploadPart] ) -> None: """Commit an Azure block blob upload, threading content-type.""" blob_client = self.service_client.get_blob_client( container=cloud_path.container, blob=cloud_path.blob ) block_ids = [part["block_id"] for part in parts] - content_settings = None - if self.content_type_method is not None: - content_type, content_encoding = self.content_type_method(str(cloud_path)) - if content_type or content_encoding: - content_settings = ContentSettings( - content_type=content_type, content_encoding=content_encoding - ) - blob_client.commit_block_list(block_ids, content_settings=content_settings) + blob_client.commit_block_list( + block_ids, content_settings=self._streaming_content_settings(cloud_path) + ) - def _abort_multipart_upload(self, cloud_path: AzureBlobPath, upload_id: str) -> None: - """Abort an Azure block blob upload. + def _streaming_content_settings( + self, cloud_path: AzureBlobPath + ) -> Optional["ContentSettings"]: + if self.content_type_method is None: + return None + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if not content_type and not content_encoding: + return None + return ContentSettings(content_type=content_type, content_encoding=content_encoding) - Azure automatically expires uncommitted blocks; nothing explicit to do. - """ + def _abort_multipart_upload(self, cloud_path: AzureBlobPath, upload_id: str) -> None: + """Let Azure expire uncommitted blocks.""" pass def _put_empty_object(self, cloud_path: AzureBlobPath) -> None: @@ -576,14 +572,9 @@ def _put_empty_object(self, cloud_path: AzureBlobPath) -> None: blob_client = self.service_client.get_blob_client( container=cloud_path.container, blob=cloud_path.blob ) - content_settings = None - if self.content_type_method is not None: - content_type, content_encoding = self.content_type_method(str(cloud_path)) - if content_type or content_encoding: - content_settings = ContentSettings( - content_type=content_type, content_encoding=content_encoding - ) - blob_client.upload_blob(b"", overwrite=True, content_settings=content_settings) + blob_client.upload_blob( + b"", overwrite=True, content_settings=self._streaming_content_settings(cloud_path) + ) def _hns_rmtree(data_lake_client, container, directory): diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py index 8bcccaa4..ee68fe3a 100644 --- a/cloudpathlib/azure/azure_io.py +++ b/cloudpathlib/azure/azure_io.py @@ -1,120 +1,23 @@ -""" -Azure Blob Storage-specific streaming I/O implementations. +"""Azure Blob Storage streaming I/O.""" -Provides efficient streaming I/O for Azure using range requests and block uploads. -""" - -from typing import Optional, Dict, Any - -from ..cloud_io import _CloudStorageRaw +from ..cloud_io import _CloudMultipartStorageRaw from ..cloudpath import register_raw_io_class @register_raw_io_class("azure") -class _AzureBlobStorageRaw(_CloudStorageRaw): - """ - Azure Blob Storage-specific raw I/O adapter. - - Implements efficient range-based reads and block blob uploads for Azure. - Writes are accumulated and staged in larger blocks to stay under Azure's - 50,000 committed-block limit per blob. - """ - - # Target block size before staging (Azure allows up to 50,000 blocks per blob) - _BLOCK_SIZE = 4 * 1024 * 1024 - _MAX_BLOCK_SIZE = 4_000 * 1024 * 1024 - _MAX_BLOCKS = 50_000 +class _AzureBlobStorageRaw(_CloudMultipartStorageRaw): + """Azure range reads and block writes.""" + + # Azure permits at most 50,000 committed blocks. + _INITIAL_PART_SIZE = 4 * 1024 * 1024 + _BLOCK_SIZE = _INITIAL_PART_SIZE + _MAX_PART_SIZE = 4_000 * 1024 * 1024 + _MAX_BLOCK_SIZE = _MAX_PART_SIZE + _MAX_PARTS = 50_000 _BLOCKS_PER_SIZE_TIER = 1_000 - - def __init__(self, client, cloud_path, mode: str = "rb"): - super().__init__(client, cloud_path, mode) - - # Block blob upload state - self._upload_id: str = "" # Azure doesn't use upload IDs - self._parts: list = [] - self._part_number: int = 1 - self._write_buffer: bytearray = bytearray() - - def _range_get(self, start: int, end: int) -> bytes: - return self._client._range_download(self._cloud_path, start, end) - - def _get_size(self) -> int: - return self._client._get_content_length(self._cloud_path) - - def _is_eof_error(self, error: Exception) -> bool: - error_str = str(error) - if "InvalidRange" in error_str or "out of range" in error_str.lower(): - return True - if hasattr(error, "error_code") and error.error_code == "InvalidRange": - return True - return False - - # ---- Write support (Azure block blob upload) ---- + _PARTS_PER_SIZE_TIER = _BLOCKS_PER_SIZE_TIER + _PROVIDER_NAME = "Azure block" @classmethod def _block_size_for_number(cls, block_number: int) -> int: - tier = (block_number - 1) // cls._BLOCKS_PER_SIZE_TIER - return min(cls._BLOCK_SIZE * (2**tier), cls._MAX_BLOCK_SIZE) - - def _target_block_size(self) -> int: - return self._block_size_for_number(self._part_number) - - def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: - if not data: - return - - self._write_buffer.extend(data) - - target_block_size = self._target_block_size() - while len(self._write_buffer) >= target_block_size: - if self._part_number > self._MAX_BLOCKS: - raise OSError("Azure block upload exceeded the 50,000-block limit") - chunk = bytes(self._write_buffer[:target_block_size]) - if not self._upload_id: - self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) - part_info = self._client._upload_part( - self._cloud_path, self._upload_id, self._part_number, chunk - ) - del self._write_buffer[:target_block_size] - self._parts.append(part_info) - self._part_number += 1 - target_block_size = self._target_block_size() - - def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: - if self._write_buffer: - if self._part_number > self._MAX_BLOCKS: - raise OSError("Azure block upload exceeded the 50,000-block limit") - if not self._upload_id: - self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) - part_info = self._client._upload_part( - self._cloud_path, - self._upload_id, - self._part_number, - bytes(self._write_buffer), - ) - self._parts.append(part_info) - self._write_buffer.clear() - - if not self._parts: - # No blocks staged — create an empty blob directly - self._client._put_empty_object(self._cloud_path) - return - - try: - self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) - finally: - self._upload_id = "" - self._parts = [] - self._part_number = 1 - - def _abort_upload(self) -> None: - try: - self._client._abort_multipart_upload(self._cloud_path, self._upload_id) - finally: - self._upload_id = "" - self._parts = [] - self._part_number = 1 - self._write_buffer.clear() - - def close(self) -> None: - super().close() + return cls._part_size_for_number(block_number) diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 37306ce5..86a7a9d9 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -4,13 +4,35 @@ from pathlib import Path import shutil from tempfile import TemporaryDirectory -from typing import ClassVar, Generic, Callable, Iterable, Optional, Tuple, TypeVar, Union +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Generic, + Iterable, + Optional, + Protocol, + Sequence, + Tuple, + TypeVar, + Union, +) from .cloudpath import CloudImplementation, CloudPath, implementation_registry from .enums import FileCacheMode from .exceptions import InvalidConfigurationException BoundedCloudPath = TypeVar("BoundedCloudPath", bound=CloudPath) +_UploadPart = Dict[str, Any] + + +class _CloudWriteStream(Protocol): + def write(self, data: bytes) -> int: ... + + def close(self) -> None: ... + + def terminate(self) -> None: ... def register_client_class(key: str) -> Callable: @@ -34,7 +56,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, - ): + ) -> None: self.file_cache_mode = None self._cache_tmp_dir = None self._cloud_meta.validate_completeness() @@ -185,99 +207,64 @@ def _generate_presigned_url( ) -> str: pass - # ====================== STREAMING I/O METHODS ====================== - # Methods to support efficient streaming without local caching. - # Default implementations raise NotImplementedError so that existing Client - # subclasses that don't implement streaming still instantiate normally. - # Providers override as needed. - def _range_download(self, cloud_path: BoundedCloudPath, start: int, end: int) -> bytes: - """Download a byte range from cloud storage. - - Args: - cloud_path: Path to download from - start: Start byte position (inclusive) - end: End byte position (inclusive) - - Returns: - Bytes in the requested range - - Raises: - FileNotFoundError: If object doesn't exist - """ + """Download an inclusive byte range.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_range_download). " "Implement this method or use a non-streaming file_cache_mode." ) def _get_content_length(self, cloud_path: BoundedCloudPath) -> int: - """Get the size of an object without downloading it. - - Args: - cloud_path: Path to query - - Returns: - Size in bytes - - Raises: - FileNotFoundError: If object doesn't exist - """ + """Return object size without downloading it.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_get_content_length)." ) def _initiate_multipart_upload(self, cloud_path: BoundedCloudPath) -> str: - """Start a multipart/chunked upload session. - - Args: - cloud_path: Destination path - - Returns: - Upload session ID/handle (provider-specific, may be empty string) - """ + """Start a multipart upload.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_initiate_multipart_upload)." ) def _upload_part( self, cloud_path: BoundedCloudPath, upload_id: str, part_number: int, data: bytes - ) -> dict: - """Upload a single part/chunk in a multipart upload. - - Args: - cloud_path: Destination path - upload_id: Upload session ID from _initiate_multipart_upload - part_number: Sequential part number (1-indexed) - data: Bytes to upload - - Returns: - Provider-specific metadata needed for finalization - """ + ) -> _UploadPart: + """Upload one part.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_upload_part)." ) def _complete_multipart_upload( - self, cloud_path: BoundedCloudPath, upload_id: str, parts: list + self, cloud_path: BoundedCloudPath, upload_id: str, parts: Sequence[_UploadPart] ) -> None: - """Finalize a multipart upload. - - Args: - cloud_path: Destination path - upload_id: Upload session ID - parts: List of part metadata from _upload_part calls - """ + """Complete a multipart upload.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_complete_multipart_upload)." ) def _abort_multipart_upload(self, cloud_path: BoundedCloudPath, upload_id: str) -> None: - """Cancel a multipart upload and clean up. - - Args: - cloud_path: Destination path - upload_id: Upload session ID - """ + """Abort a multipart upload.""" raise NotImplementedError( f"{type(self).__name__} does not support streaming I/O (_abort_multipart_upload)." ) + + def _open_write_stream(self, cloud_path: BoundedCloudPath) -> _CloudWriteStream: + """Open a provider write stream.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_open_write_stream)." + ) + + def _write_stream(self, stream: _CloudWriteStream, data: bytes) -> int: + return stream.write(data) + + def _close_write_stream(self, stream: _CloudWriteStream) -> None: + stream.close() + + def _abort_write_stream(self, stream: _CloudWriteStream) -> None: + stream.terminate() + + def _put_empty_object(self, cloud_path: BoundedCloudPath) -> None: + """Create an empty object.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_put_empty_object)." + ) diff --git a/cloudpathlib/cloud_io.py b/cloudpathlib/cloud_io.py index 242f48a4..6c1d0893 100644 --- a/cloudpathlib/cloud_io.py +++ b/cloudpathlib/cloud_io.py @@ -1,14 +1,21 @@ -""" -Cloud storage streaming I/O implementations. +"""Buffered cloud I/O without a local cache.""" -Provides BufferedIOBase and TextIOBase compliant file-like objects for cloud storage -that support efficient streaming with range requests and multipart uploads, without -requiring full local caching. -""" +from __future__ import annotations import io from abc import abstractmethod -from typing import Optional, Any, Type, Union, Dict +from types import TracebackType +from typing import TYPE_CHECKING, Any, Optional, Type, Union + +if TYPE_CHECKING: + from _typeshed import ReadableBuffer as _ReadableBuffer + from _typeshed import WriteableBuffer as _WriteableBuffer +else: + _ReadableBuffer = Union[bytes, bytearray, memoryview] + _WriteableBuffer = Union[bytearray, memoryview] + +from .client import Client +from .cloudpath import CloudPath def _validate_file_mode(mode: str) -> None: @@ -25,33 +32,15 @@ def _validate_file_mode(mode: str) -> None: raise ValueError("can't have text and binary mode at once") -# ============================================================================ -# Base Raw I/O Adapter (internal) -# ============================================================================ - - class _CloudStorageRaw(io.RawIOBase): - """ - Internal raw I/O adapter for cloud storage objects. - - Implements efficient range-based reads using cloud provider APIs. - Not exposed to users - internal implementation detail. - """ + """Raw adapter backed by client streaming hooks.""" def __init__( self, - client: Any, - cloud_path: Any, + client: Client, + cloud_path: CloudPath, mode: str = "rb", - ): - """ - Initialize raw cloud storage adapter. - - Args: - client: Cloud provider client (S3Client, AzureBlobClient, etc.) - cloud_path: CloudPath instance - mode: File mode (currently only read modes supported in base) - """ + ) -> None: super().__init__() self._client = client self._cloud_path = cloud_path @@ -76,64 +65,42 @@ def seekable(self) -> bool: """ return self.readable() - def readinto(self, b: bytearray) -> int: # type: ignore[override] - """ - Read bytes into a pre-allocated buffer. - - Args: - b: Buffer to read data into - - Returns: - Number of bytes read (0 at EOF) - """ + def readinto(self, b: _WriteableBuffer, /) -> int: if self._closed: raise ValueError("I/O operation on closed file") if not self.readable(): raise io.UnsupportedOperation("not readable") - if len(b) == 0: + view = memoryview(b).cast("B") + if len(view) == 0: return 0 - # Calculate range to fetch start = self._pos - end = start + len(b) - 1 + end = start + len(view) - 1 - # Clamp end to file size if known (prevents 416 errors) if self._size is None: try: self._size = self._get_size() except Exception: - # If we can't get size, try the request anyway pass if self._size is not None and end >= self._size: - # Clamp to last valid byte end = self._size - 1 if start >= self._size: - # Already at EOF return 0 - # Fetch data from cloud storage try: data = self._range_get(start, end) except Exception as e: - # If we get an error reading beyond EOF, treat as EOF if self._is_eof_error(e): return 0 raise - # Copy data into buffer n = len(data) if n == 0: return 0 - # Ensure we don't write more than the buffer can hold - n = min(n, len(b)) - - try: - b[:n] = data[:n] - except (ValueError, TypeError): - # Fall back to memoryview-based copy for non-contiguous buffer shapes - memoryview(b).cast("B")[:n] = data[:n] + n = min(n, len(view)) + view[:n] = data[:n] self._pos += n return n @@ -180,19 +147,7 @@ def tell(self) -> int: raise ValueError("I/O operation on closed file") return self._pos - def write(self, b: bytes) -> int: # type: ignore[override] - """ - Write bytes to the stream. - - This method is required by RawIOBase for writable streams. - The actual implementation is delegated to subclasses via _upload_chunk. - - Args: - b: Bytes to write - - Returns: - Number of bytes written - """ + def write(self, b: _ReadableBuffer, /) -> int: if self._closed: raise ValueError("I/O operation on closed file") if not self.writable(): @@ -200,19 +155,19 @@ def write(self, b: bytes) -> int: # type: ignore[override] if self._upload_error is not None: raise self._upload_error + data = bytes(b) try: - self._upload_chunk(bytes(b), None) + self._upload_chunk(data) except BaseException as error: self._upload_error = error raise - return len(b) + return len(data) def close(self) -> None: """Close the file.""" if self._closed: return - # Mark as closed FIRST to prevent recursive double-finalize self._closed = True try: @@ -225,7 +180,7 @@ def close(self) -> None: raise self._upload_error if self.writable(): try: - self._finalize_upload(None) + self._finalize_upload() except BaseException: try: self._abort_upload() @@ -233,7 +188,6 @@ def close(self) -> None: pass raise finally: - # Always call parent close() to set the stdlib closed state super().close() def _abort_upload(self) -> None: @@ -241,51 +195,18 @@ def _abort_upload(self) -> None: pass @abstractmethod - def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]]) -> None: - """ - Upload a chunk of data. - - Args: - data: Bytes to upload - upload_state: Upload state dictionary (for multipart uploads) - """ + def _upload_chunk(self, data: bytes) -> None: pass @abstractmethod - def _finalize_upload(self, upload_state: Optional[Dict[str, Any]]) -> None: - """ - Finalize the upload process. - - Args: - upload_state: Upload state dictionary (for multipart uploads) - """ + def _finalize_upload(self) -> None: pass - # Abstract methods to be implemented by subclasses - - @abstractmethod def _range_get(self, start: int, end: int) -> bytes: - """ - Fetch a byte range from cloud storage. - - Args: - start: Start byte position (inclusive) - end: End byte position (inclusive) - - Returns: - Bytes in the requested range - """ - pass + return self._client._range_download(self._cloud_path, start, end) - @abstractmethod def _get_size(self) -> int: - """ - Get the total size of the cloud object. - - Returns: - Size in bytes - """ - pass + return self._client._get_content_length(self._cloud_path) def _is_eof_error(self, error: Exception) -> bool: """ @@ -296,43 +217,91 @@ def _is_eof_error(self, error: Exception) -> bool: return False -# ============================================================================ -# Public Buffered Binary I/O -# ============================================================================ +class _CloudMultipartStorageRaw(_CloudStorageRaw): + """Shared buffered multipart upload lifecycle.""" + _INITIAL_PART_SIZE: int + _MAX_PART_SIZE: int + _MAX_PARTS: int + _PARTS_PER_SIZE_TIER: int + _PROVIDER_NAME: str -class CloudBufferedIO(io.BufferedIOBase): - """ - Buffered binary file-like object for cloud storage. + def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: + super().__init__(client, cloud_path, mode) + self._upload_id: Optional[str] = None + self._parts: list[dict[str, Any]] = [] + self._part_number = 1 + self._write_buffer = bytearray() + + @classmethod + def _part_size_for_number(cls, part_number: int) -> int: + tier = (part_number - 1) // cls._PARTS_PER_SIZE_TIER + return min(cls._INITIAL_PART_SIZE * (2**tier), cls._MAX_PART_SIZE) - Wraps a raw cloud storage adapter with Python's standard buffered I/O classes - (BufferedReader, BufferedWriter, or BufferedRandom) based on the mode. + def _target_part_size(self) -> int: + return self._part_size_for_number(self._part_number) - Example: - >>> from cloudpathlib import S3Client - >>> client = S3Client() - >>> with CloudBufferedIO(client, "s3://bucket/file.bin", mode="rb") as f: - ... data = f.read(1024) - """ + def _check_part_limit(self) -> None: + if self._part_number > self._MAX_PARTS: + raise OSError( + f"{self._PROVIDER_NAME} upload exceeded the {self._MAX_PARTS:,}-part limit" + ) + + def _upload_buffered_part(self, size: int) -> None: + self._check_part_limit() + if self._upload_id is None: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + data = bytes(self._write_buffer[:size]) + part = self._client._upload_part( + self._cloud_path, self._upload_id, self._part_number, data + ) + del self._write_buffer[:size] + self._parts.append(part) + self._part_number += 1 + + def _upload_chunk(self, data: bytes) -> None: + if not data: + return + self._write_buffer.extend(data) + target_size = self._target_part_size() + while len(self._write_buffer) >= target_size: + self._upload_buffered_part(target_size) + target_size = self._target_part_size() + + def _finalize_upload(self) -> None: + if self._write_buffer: + self._upload_buffered_part(len(self._write_buffer)) + if self._upload_id is None: + self._client._put_empty_object(self._cloud_path) + return + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) + self._reset_upload() + + def _abort_upload(self) -> None: + try: + if self._upload_id is not None: + self._client._abort_multipart_upload(self._cloud_path, self._upload_id) + finally: + self._reset_upload() + + def _reset_upload(self) -> None: + self._upload_id = None + self._parts.clear() + self._part_number = 1 + self._write_buffer.clear() + + +class CloudBufferedIO(io.BufferedIOBase): + """Buffered binary I/O backed by a cloud client.""" def __init__( self, raw_io_class: Type[_CloudStorageRaw], - client: Any, - cloud_path: Any, + client: Client, + cloud_path: CloudPath, mode: str = "rb", buffer_size: int = 64 * 1024, - ): - """ - Initialize cloud buffered I/O. - - Args: - raw_io_class: The raw I/O class to use for this provider - client: Cloud provider client instance - cloud_path: CloudPath instance - mode: Streaming file mode ('rb', 'wb', or 'xb') - buffer_size: Size of read/write buffer in bytes (default 64 KiB) - """ + ) -> None: _validate_file_mode(mode) if "b" not in mode: raise ValueError("CloudBufferedIO requires binary mode (must include 'b')") @@ -341,21 +310,14 @@ def __init__( "append and update modes require the local-cache implementation" ) - # Create raw adapter using provided class raw = raw_io_class(client, cloud_path, mode) - # Choose appropriate buffered class based on mode - if "+" in mode: - # Read and write (e.g., 'r+b', 'w+b') - self._buffer: Union[io.BufferedReader, io.BufferedWriter, io.BufferedRandom] = io.BufferedRandom(raw, buffer_size=buffer_size) # type: ignore[arg-type] - elif "r" in mode: - # Read only (e.g., 'rb') + if "r" in mode: + self._buffer: Union[io.BufferedReader, io.BufferedWriter] self._buffer = io.BufferedReader(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] else: - # Write only (e.g., 'wb', 'ab', 'xb') self._buffer = io.BufferedWriter(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] - # Store additional attributes self._cloud_path = cloud_path self._mode = mode self._buffer_size_val = buffer_size @@ -375,35 +337,31 @@ def _buffer_size(self) -> int: """Buffer size for compatibility with tests.""" return self._buffer_size_val - # Delegate all I/O methods to the internal buffer - def read(self, size: Optional[int] = -1) -> bytes: # type: ignore[override] + def read(self, size: Optional[int] = -1, /) -> bytes: return self._buffer.read(size) - def read1(self, size: int = -1) -> bytes: + def read1(self, size: int = -1, /) -> bytes: return self._buffer.read1(size) # type: ignore[attr-defined] - def readinto(self, b): + def readinto(self, b: _WriteableBuffer, /) -> int: return self._buffer.readinto(b) - def readinto1(self, b): + def readinto1(self, b: _WriteableBuffer, /) -> int: return self._buffer.readinto1(b) # type: ignore[attr-defined] - def write(self, b): + def write(self, b: _ReadableBuffer, /) -> int: return self._buffer.write(b) - def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int: return self._buffer.seek(offset, whence) def tell(self) -> int: return self._buffer.tell() - def flush(self): - return self._buffer.flush() + def flush(self) -> None: + self._buffer.flush() - def close(self): - # Delegate entirely to the stdlib buffer: flush → raw.close(). - # _CloudStorageRaw.close() guards against double-finalize via _closed, - # so calling self._buffer.close() is safe and lets exceptions propagate. + def close(self) -> None: if hasattr(self, "_buffer") and not self._buffer.closed: self._buffer.close() @@ -420,58 +378,34 @@ def seekable(self) -> bool: def closed(self) -> bool: return self._buffer.closed - def __enter__(self): + def __enter__(self) -> CloudBufferedIO: return self - def __exit__(self, *args): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + /, + ) -> None: self.close() -# ============================================================================ -# Public Text I/O -# ============================================================================ - - class CloudTextIO(io.TextIOWrapper): - """ - Text file-like object for cloud storage. - - Implements TextIOBase for seamless integration with standard Python I/O - and third-party libraries. Handles encoding/decoding and newline translation. - - Example: - >>> from cloudpathlib import S3Client - >>> client = S3Client() - >>> with CloudTextIO(client, "s3://bucket/file.txt", mode="rt") as f: - ... text = f.read() - """ + """Text I/O backed by a cloud client.""" def __init__( self, raw_io_class: Type[_CloudStorageRaw], - client: Any, - cloud_path: Any, + client: Client, + cloud_path: CloudPath, mode: str = "rt", encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, buffer_size: int = 64 * 1024, line_buffering: bool = False, - ): - """ - Initialize cloud text I/O. - - Args: - raw_io_class: The raw I/O class to use for this provider - client: Cloud provider client instance - cloud_path: CloudPath instance - mode: Streaming file mode ('rt', 'wt', 'xt', or the same without 't') - encoding: Text encoding (default: platform locale, matching ``open``) - errors: Error handling strategy (default: strict) - newline: Newline handling (None, '', '\\n', '\\r', '\\r\\n') - buffer_size: Size of buffer in bytes - line_buffering: Flush text output whenever a newline is written - """ + ) -> None: _validate_file_mode(mode) if "b" in mode: raise ValueError("CloudTextIO requires text mode (no 'b' in mode)") @@ -480,7 +414,6 @@ def __init__( "append and update modes require the local-cache implementation" ) - # Ensure mode has 't' or is text mode if "t" not in mode and "r" in mode: binary_mode = mode.replace("r", "rb", 1) elif "t" not in mode and "w" in mode: @@ -492,12 +425,10 @@ def __init__( else: binary_mode = mode.replace("t", "b") - # Create underlying buffered I/O buffered = CloudBufferedIO( raw_io_class, client, cloud_path, mode=binary_mode, buffer_size=buffer_size ) - # Initialize TextIOWrapper with the buffered stream super().__init__( buffered, encoding=encoding, @@ -506,7 +437,6 @@ def __init__( line_buffering=line_buffering, ) - # Store additional attributes self._cloud_path = cloud_path self._mode = mode diff --git a/cloudpathlib/gs/gs_io.py b/cloudpathlib/gs/gs_io.py index 60d91e10..e53ec968 100644 --- a/cloudpathlib/gs/gs_io.py +++ b/cloudpathlib/gs/gs_io.py @@ -1,69 +1,39 @@ -""" -Google Cloud Storage-specific streaming I/O implementations. +"""Google Cloud Storage streaming I/O.""" -Provides efficient streaming I/O for GCS using range requests and resumable uploads. -Upload state is held on the raw adapter instance (not the shared client) so concurrent -writers to the same client cannot collide. -""" +from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Optional +from ..client import Client, _CloudWriteStream from ..cloud_io import _CloudStorageRaw -from ..cloudpath import register_raw_io_class +from ..cloudpath import CloudPath, register_raw_io_class @register_raw_io_class("gs") class _GSStorageRaw(_CloudStorageRaw): - """ - GCS-specific raw I/O adapter. + """GCS range reads and resumable writes.""" - Implements efficient range-based reads and resumable uploads for GCS. - Write state (_writer) lives on this adapter instance, not on the client, - so concurrent writes to different paths on the same client are safe. - """ - - def __init__(self, client, cloud_path, mode: str = "rb"): + def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: super().__init__(client, cloud_path, mode) - # Open write stream (returned by client._open_write_stream); None until first write - self._writer: Optional[Any] = None - - def _range_get(self, start: int, end: int) -> bytes: - return self._client._range_download(self._cloud_path, start, end) - - def _get_size(self) -> int: - return self._client._get_content_length(self._cloud_path) - - def _is_eof_error(self, error: Exception) -> bool: - error_str = str(error) - if "416" in error_str or "Requested Range Not Satisfiable" in error_str: - return True - if hasattr(error, "code") and error.code == 416: - return True - return False - - # ---- Write support (resumable upload via client._open_write_stream) ---- + self._writer: Optional[_CloudWriteStream] = None - def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: + def _upload_chunk(self, data: bytes) -> None: if not data: return if self._writer is None: self._writer = self._client._open_write_stream(self._cloud_path) - self._writer.write(data) + self._client._write_stream(self._writer, data) - def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: - if self._writer is not None: - self._writer.close() - self._writer = None - else: - # No data was written — create an empty object + def _finalize_upload(self) -> None: + if self._writer is None: self._client._put_empty_object(self._cloud_path) + return + self._client._close_write_stream(self._writer) + self._writer = None def _abort_upload(self) -> None: if self._writer is not None: try: - self._writer.terminate() + self._client._abort_write_stream(self._writer) finally: self._writer = None - - def close(self) -> None: - super().close() diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index 99e1ca6f..baf30a33 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union import warnings -from ..client import Client, register_client_class +from ..client import Client, _CloudWriteStream, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode from .gspath import GSPath @@ -312,13 +312,10 @@ def _generate_presigned_url(self, cloud_path: GSPath, expire_seconds: int = 60 * ) return url - # ====================== STREAMING I/O METHODS ====================== - def _range_download(self, cloud_path: GSPath, start: int, end: int) -> bytes: """Download a byte range from GCS.""" blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) try: - # GCS and our internal API both use an inclusive end offset. return blob.download_as_bytes(start=start, end=end, **self.blob_kwargs) except GCSNotFound: raise FileNotFoundError(f"GCS object not found: {cloud_path}") @@ -339,20 +336,14 @@ def _get_content_length(self, cloud_path: GSPath) -> int: except GCSNotFound: raise FileNotFoundError(f"GCS object not found: {cloud_path}") - def _open_write_stream(self, cloud_path: GSPath): - """Open a GCS resumable upload stream. - - Returns a file-like writer. Data written to it streams incrementally - to GCS rather than being buffered in memory. The caller must close() - the writer to finalize the upload. - """ + def _open_write_stream(self, cloud_path: GSPath) -> _CloudWriteStream: + """Open a GCS resumable upload.""" blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) kwargs: Dict[str, Any] = {} if self.content_type_method is not None: content_type, _ = self.content_type_method(str(cloud_path)) if content_type is not None: kwargs["content_type"] = content_type - # blob_kwargs may carry timeout/retry; pass through where blob.open accepts them for k in ("timeout", "retry"): if k in self.blob_kwargs: kwargs[k] = self.blob_kwargs[k] diff --git a/cloudpathlib/http/http_io.py b/cloudpathlib/http/http_io.py index c5af16ab..77ff0761 100644 --- a/cloudpathlib/http/http_io.py +++ b/cloudpathlib/http/http_io.py @@ -1,64 +1,46 @@ -""" -HTTP-specific streaming I/O implementations. +"""HTTP streaming I/O.""" -Provides streaming I/O for HTTP/HTTPS using range requests and single-PUT uploads. -""" +from __future__ import annotations import tempfile -from typing import Optional, Dict, Any +from typing import Protocol, cast +from ..client import Client from ..cloud_io import _CloudStorageRaw -from ..cloudpath import register_raw_io_class +from ..cloudpath import CloudPath, register_raw_io_class + + +class _HttpStreamingClient(Protocol): + def _put_data( + self, + cloud_path: CloudPath, + data: tempfile.SpooledTemporaryFile[bytes], + content_length: int, + ) -> None: ... @register_raw_io_class("http") @register_raw_io_class("https") class _HttpStorageRaw(_CloudStorageRaw): - """ - HTTP-specific raw I/O adapter. + """HTTP range reads and single-request writes.""" - Implements efficient range-based reads and single-PUT uploads for HTTP/HTTPS. - Write operations require the server to support PUT requests. - Data is buffered in _upload_buffer on the adapter instance (not on the client) - and flushed as a single PUT on close. - """ - - def __init__(self, client, cloud_path, mode: str = "rb"): + def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: super().__init__(client, cloud_path, mode) - self._upload_buffer: Any = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024) - - def _range_get(self, start: int, end: int) -> bytes: - return self._client._range_download(self._cloud_path, start, end) + self._upload_buffer = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024) - def _get_size(self) -> int: - return self._client._get_content_length(self._cloud_path) + def _upload_chunk(self, data: bytes) -> None: + if data: + self._upload_buffer.write(data) - def _is_eof_error(self, error: Exception) -> bool: - error_str = str(error).lower() - return ( - "416" in error_str - or "requested range not satisfiable" in error_str - or "invalid range" in error_str - ) - - # ---- Write support (single PUT on finalize) ---- - - def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: - if not data: - return - self._upload_buffer.write(data) - - def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: + def _finalize_upload(self) -> None: self._upload_buffer.seek(0, 2) content_length = self._upload_buffer.tell() self._upload_buffer.seek(0) try: - self._client._put_data(self._cloud_path, self._upload_buffer, content_length) + client = cast(_HttpStreamingClient, self._client) + client._put_data(self._cloud_path, self._upload_buffer, content_length) finally: self._upload_buffer.close() def _abort_upload(self) -> None: self._upload_buffer.close() - - def close(self) -> None: - super().close() diff --git a/cloudpathlib/http/httpclient.py b/cloudpathlib/http/httpclient.py index 52807b2e..5dcecbb6 100644 --- a/cloudpathlib/http/httpclient.py +++ b/cloudpathlib/http/httpclient.py @@ -203,15 +203,8 @@ def request( # the connection is closed when we exit the context manager. return response, response.read() - # ====================== STREAMING I/O METHODS ====================== - def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes: - """Download a byte range from HTTP. - - Verifies the response is 206 Partial Content. If the server returns 200 - (ignoring the Range header), slices the full body locally so callers - always receive exactly the requested bytes. - """ + """Download an HTTP byte range.""" headers = {"Range": f"bytes={start}-{end}"} request = urllib.request.Request(str(cloud_path), headers=headers) try: @@ -229,7 +222,7 @@ def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes except urllib.error.HTTPError as e: if e.code == 404: raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") - elif e.code == 416: # Range not satisfiable + elif e.code == 416: return b"" raise @@ -247,37 +240,8 @@ def _get_content_length(self, cloud_path: "HttpPath") -> int: raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") raise - def _initiate_multipart_upload(self, cloud_path: "HttpPath") -> str: - """HTTP uploads are single-shot PUT; no session needed.""" - return "" - - def _upload_part( - self, cloud_path: "HttpPath", upload_id: str, part_number: int, data: bytes - ) -> dict: - """HTTP does not support true multipart; use _put_data for a single PUT.""" - raise NotImplementedError( - "HTTP uses a single PUT for uploads; multipart is not supported. " - "Use _put_data instead." - ) - - def _complete_multipart_upload( - self, cloud_path: "HttpPath", upload_id: str, parts: list - ) -> None: - """HTTP does not support true multipart; use _put_data for a single PUT.""" - raise NotImplementedError( - "HTTP uses a single PUT for uploads; multipart is not supported." - ) - - def _abort_multipart_upload(self, cloud_path: "HttpPath", upload_id: str) -> None: - """Nothing to abort for HTTP single-PUT uploads.""" - pass - def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) -> None: - """Upload a file-like body using the client's configured write method. - - Uses self.opener so that any SSL context or auth handlers configured on - this client are applied (important for HttpsClient with self-signed certs). - """ + """Upload a file-like HTTP body.""" url = str(cloud_path) request = urllib.request.Request(url, data=data, method=self.write_file_http_method) content_type = None @@ -293,7 +257,7 @@ def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) f"HTTP PUT failed with status {response.status}: {response.reason}" ) except urllib.error.HTTPError as e: - if e.code == 405: # Method Not Allowed + if e.code == 405: raise NotImplementedError( f"HTTP server does not support {self.write_file_http_method} requests for {url}" ) diff --git a/cloudpathlib/local/localclient.py b/cloudpathlib/local/localclient.py index e1d541bc..edd21c35 100644 --- a/cloudpathlib/local/localclient.py +++ b/cloudpathlib/local/localclient.py @@ -7,10 +7,23 @@ import sys from tempfile import TemporaryDirectory from time import sleep -from typing import Callable, ClassVar, Dict, Iterable, List, Optional, Tuple, Union +from types import TracebackType +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from ..client import Client +from ..client import Client, _CloudWriteStream, _UploadPart from ..enums import FileCacheMode from .localpath import LocalPath @@ -28,14 +41,15 @@ class LocalClient(Client): def __init__( self, - *args, + *args: Any, local_storage_dir: Optional[Union[str, os.PathLike]] = None, file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, - **kwargs, - ): + **kwargs: Any, + ) -> None: self._local_storage_dir = local_storage_dir + self._local_upload_buffers: Dict[str, List[Tuple[int, bytes]]] = {} super().__init__( local_cache_dir=local_cache_dir, @@ -215,9 +229,6 @@ def _generate_presigned_url( query["signature"] = "local" return urlunsplit(parts._replace(query=urlencode(query))) - # ====================== STREAMING I/O METHODS ====================== - # For local clients, streaming uses local file operations. - def _range_download(self, cloud_path: LocalPath, start: int, end: int) -> bytes: """Download a byte range from local storage.""" local_path = self._cloud_path_to_local(cloud_path) @@ -244,23 +255,17 @@ def _initiate_multipart_upload(self, cloud_path: LocalPath) -> str: def _upload_part( self, cloud_path: LocalPath, upload_id: str, part_number: int, data: bytes - ) -> dict: - """Buffer a part keyed by upload_id (not by path) for concurrent-write safety.""" - if not hasattr(self, "_local_upload_buffers"): - self._local_upload_buffers: dict = {} + ) -> _UploadPart: + """Buffer a part by upload ID.""" if upload_id not in self._local_upload_buffers: self._local_upload_buffers[upload_id] = [] self._local_upload_buffers[upload_id].append((part_number, data)) return {"part_number": part_number} def _complete_multipart_upload( - self, cloud_path: LocalPath, upload_id: str, parts: list + self, cloud_path: LocalPath, upload_id: str, parts: Sequence[_UploadPart] ) -> None: - """Complete local file upload by joining all buffered parts.""" - if ( - not hasattr(self, "_local_upload_buffers") - or upload_id not in self._local_upload_buffers - ): + if upload_id not in self._local_upload_buffers: return buffer = self._local_upload_buffers.pop(upload_id, []) @@ -272,12 +277,9 @@ def _complete_multipart_upload( local_path.write_bytes(complete_data) def _abort_multipart_upload(self, cloud_path: LocalPath, upload_id: str) -> None: - """Abort local file upload by cleaning up the buffer.""" - if hasattr(self, "_local_upload_buffers"): - self._local_upload_buffers.pop(upload_id, None) + self._local_upload_buffers.pop(upload_id, None) - def _open_write_stream(self, cloud_path: LocalPath) -> "_LocalWriteStream": - """Return a write stream that buffers data and writes to the local file on close.""" + def _open_write_stream(self, cloud_path: LocalPath) -> _CloudWriteStream: local_path = self._cloud_path_to_local(cloud_path) return _LocalWriteStream(local_path) @@ -289,11 +291,7 @@ def _put_empty_object(self, cloud_path: LocalPath) -> None: class _LocalWriteStream: - """File-like writer that accumulates bytes and flushes to a local path on close. - - Used by LocalClient._open_write_stream so that _GSStorageRaw (and other adapters - that call _open_write_stream) can work correctly against local test clients. - """ + """Buffered local write stream.""" def __init__(self, local_path: Path) -> None: self._local_path = local_path @@ -312,10 +310,19 @@ def close(self) -> None: self._local_path.parent.mkdir(parents=True, exist_ok=True) self._local_path.write_bytes(bytes(self._buf)) - def __enter__(self): + def terminate(self) -> None: + self._closed = True + self._buf.clear() + + def __enter__(self) -> "_LocalWriteStream": return self - def __exit__(self, *args): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: self.close() @@ -323,6 +330,6 @@ def __exit__(self, *args): @atexit.register -def clean_temp_dirs(): +def clean_temp_dirs() -> None: for temp_dir in _temp_dirs_to_clean: temp_dir.cleanup() diff --git a/cloudpathlib/s3/s3_io.py b/cloudpathlib/s3/s3_io.py index 9d7d0509..fb0684b7 100644 --- a/cloudpathlib/s3/s3_io.py +++ b/cloudpathlib/s3/s3_io.py @@ -1,124 +1,17 @@ -""" -S3-specific streaming I/O implementations. +"""S3 streaming I/O.""" -Provides efficient streaming I/O for S3 using range requests and multipart uploads. -""" - -from typing import Optional, Dict, Any - -from ..cloud_io import _CloudStorageRaw +from ..cloud_io import _CloudMultipartStorageRaw from ..cloudpath import register_raw_io_class @register_raw_io_class("s3") -class _S3StorageRaw(_CloudStorageRaw): - """ - S3-specific raw I/O adapter. - - Implements efficient range-based reads and multipart uploads for S3. - S3 requires non-final parts to be at least 5 MiB; this class accumulates - chunks in _write_buffer and only uploads a part once _MIN_PART_SIZE is reached. - """ +class _S3StorageRaw(_CloudMultipartStorageRaw): + """S3 range reads and multipart writes.""" - # S3 minimum part size for non-final parts: 5 MiB - _MIN_PART_SIZE = 5 * 1024 * 1024 + # S3 requires non-final parts of at least 5 MiB. + _INITIAL_PART_SIZE = 5 * 1024 * 1024 + _MIN_PART_SIZE = _INITIAL_PART_SIZE _MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 _MAX_PARTS = 10_000 _PARTS_PER_SIZE_TIER = 1_000 - - def __init__(self, client, cloud_path, mode: str = "rb"): - super().__init__(client, cloud_path, mode) - - # Multipart upload state - self._upload_id: Optional[str] = None - self._parts: list = [] - self._part_number: int = 1 - # Accumulation buffer — we only flush a part when >= _MIN_PART_SIZE bytes - self._write_buffer: bytearray = bytearray() - - def _range_get(self, start: int, end: int) -> bytes: - return self._client._range_download(self._cloud_path, start, end) - - def _get_size(self) -> int: - return self._client._get_content_length(self._cloud_path) - - def _is_eof_error(self, error: Exception) -> bool: - error_str = str(error) - return ( - "InvalidRange" in error_str - or hasattr(error, "__class__") - and "InvalidRange" in error.__class__.__name__ - ) - - @classmethod - def _part_size_for_number(cls, part_number: int) -> int: - tier = (part_number - 1) // cls._PARTS_PER_SIZE_TIER - return min(cls._MIN_PART_SIZE * (2**tier), cls._MAX_PART_SIZE) - - def _target_part_size(self) -> int: - return self._part_size_for_number(self._part_number) - - # ---- Write support (multipart upload with 5 MiB minimum part size) ---- - - def _upload_chunk(self, data: bytes, upload_state: Optional[Dict[str, Any]] = None) -> None: - if not data: - return - - self._write_buffer.extend(data) - - # Upload full-sized parts whenever we have enough buffered data - target_part_size = self._target_part_size() - while len(self._write_buffer) >= target_part_size: - if self._part_number > self._MAX_PARTS: - raise OSError("S3 multipart upload exceeded the 10,000-part limit") - chunk = bytes(self._write_buffer[:target_part_size]) - if self._upload_id is None: - self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) - part_info = self._client._upload_part( - self._cloud_path, self._upload_id, self._part_number, chunk - ) - del self._write_buffer[:target_part_size] - self._parts.append(part_info) - self._part_number += 1 - target_part_size = self._target_part_size() - - def _finalize_upload(self, upload_state: Optional[Dict[str, Any]] = None) -> None: - # Flush remaining buffer as the final part (exempt from 5 MiB floor) - if self._write_buffer: - if self._part_number > self._MAX_PARTS: - raise OSError("S3 multipart upload exceeded the 10,000-part limit") - if self._upload_id is None: - self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) - part_info = self._client._upload_part( - self._cloud_path, - self._upload_id, - self._part_number, - bytes(self._write_buffer), - ) - self._parts.append(part_info) - self._part_number += 1 - self._write_buffer = bytearray() - - if self._upload_id is None: - # No data written — create an empty object directly - self._client._put_empty_object(self._cloud_path) - return - - self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) - self._upload_id = None - self._parts = [] - self._part_number = 1 - self._write_buffer.clear() - - def _abort_upload(self) -> None: - try: - if self._upload_id is not None: - self._client._abort_multipart_upload(self._cloud_path, self._upload_id) - finally: - self._upload_id = None - self._parts = [] - self._part_number = 1 - self._write_buffer.clear() - - def close(self) -> None: - super().close() + _PROVIDER_NAME = "S3 multipart" diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 9cd6730c..3b26c7f2 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -1,9 +1,9 @@ import mimetypes import os from pathlib import Path, PurePosixPath -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union -from ..client import Client, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode from ..exceptions import CloudPathException @@ -400,8 +400,6 @@ def _generate_presigned_url(self, cloud_path: S3Path, expire_seconds: int = 60 * ) return url - # ====================== STREAMING I/O METHODS ====================== - def _range_download(self, cloud_path: S3Path, start: int, end: int) -> bytes: """Download a byte range from S3.""" try: @@ -445,7 +443,7 @@ def _get_content_length(self, cloud_path: S3Path) -> int: raise FileNotFoundError(f"S3 object not found: {cloud_path}") raise - def _streaming_extra_args(self, operation_name: str) -> dict: + def _streaming_extra_args(self, operation_name: str) -> Dict[str, Any]: """Return upload extras accepted by a specific low-level S3 operation.""" try: operation = self.client.meta.service_model.operation_model(operation_name) @@ -502,15 +500,19 @@ def _streaming_extra_args(self, operation_name: str) -> dict: allowed = fallback_allowed[operation_name] return {key: value for key, value in self.boto3_ul_extra_args.items() if key in allowed} - def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: - """Start an S3 multipart upload, threading content-type and upload extra args.""" - extra_args = self._streaming_extra_args("CreateMultipartUpload") + def _streaming_object_args(self, operation_name: str, cloud_path: S3Path) -> Dict[str, Any]: + extra_args = self._streaming_extra_args(operation_name) if self.content_type_method is not None: content_type, content_encoding = self.content_type_method(str(cloud_path)) if content_type is not None: extra_args["ContentType"] = content_type if content_encoding is not None: extra_args["ContentEncoding"] = content_encoding + return extra_args + + def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: + """Start an S3 multipart upload, threading content-type and upload extra args.""" + extra_args = self._streaming_object_args("CreateMultipartUpload", cloud_path) response = self.client.create_multipart_upload( Bucket=cloud_path.bucket, Key=cloud_path.key, @@ -520,7 +522,7 @@ def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: def _upload_part( self, cloud_path: S3Path, upload_id: str, part_number: int, data: bytes - ) -> dict: + ) -> _UploadPart: """Upload a part in an S3 multipart upload.""" response = self.client.upload_part( Bucket=cloud_path.bucket, @@ -534,7 +536,9 @@ def _upload_part( part.update({key: value for key, value in response.items() if key.startswith("Checksum")}) return part - def _complete_multipart_upload(self, cloud_path: S3Path, upload_id: str, parts: list) -> None: + def _complete_multipart_upload( + self, cloud_path: S3Path, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: """Complete an S3 multipart upload.""" self.client.complete_multipart_upload( Bucket=cloud_path.bucket, @@ -552,13 +556,7 @@ def _abort_multipart_upload(self, cloud_path: S3Path, upload_id: str) -> None: def _put_empty_object(self, cloud_path: S3Path) -> None: """Upload a zero-byte object, threading content-type and upload extra args.""" - extra_args = self._streaming_extra_args("PutObject") - if self.content_type_method is not None: - content_type, content_encoding = self.content_type_method(str(cloud_path)) - if content_type is not None: - extra_args["ContentType"] = content_type - if content_encoding is not None: - extra_args["ContentEncoding"] = content_encoding + extra_args = self._streaming_object_args("PutObject", cloud_path) self.client.put_object( Bucket=cloud_path.bucket, Key=cloud_path.key, diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index b4637a69..88c32d4e 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -1162,7 +1162,7 @@ def test_finalize_error_propagates(rig): raw_io_class = path._cloud_meta.raw_io_class class _FailingRaw(raw_io_class): - def _finalize_upload(self, upload_state=None): + def _finalize_upload(self) -> None: raise RuntimeError("simulated upload failure") original_mode = path.client.file_cache_mode From f9b3fb9a35790ed035a57371149ec4f23d537e88 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:11:40 -0700 Subject: [PATCH 08/11] Fix Windows streaming test failures --- cloudpathlib/cloudpath.py | 6 ++---- tests/test_cloud_io.py | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 9024b0fd..a3e0f4d0 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -919,10 +919,8 @@ def _patched_close_upload(*args, **kwargs) -> None: if not self._dirty: return - # original mtime should match what was in the cloud; because of system clocks or rounding - # by the cloud provider, the new version in our cache is "older" than the original version; - # explicitly set the new modified time to be after the original modified time. - if self._local.stat().st_mtime < original_mtime: + # Keep cached writes newer despite timestamp rounding. + if self._local.stat().st_mtime <= original_mtime: new_mtime = original_mtime + 1 os.utime(self._local, times=(new_mtime, new_mtime)) diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index 88c32d4e..e4af11c3 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -76,7 +76,7 @@ def temp_cloud_multiline_file(rig): pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") path = rig.create_cloud_path("test_streaming_multiline.txt") - path.write_text(MULTILINE_TEXT) + path.write_text(MULTILINE_TEXT, encoding="utf-8") # Set client to streaming mode original_mode = path.client.file_cache_mode path.client.file_cache_mode = FileCacheMode.streaming @@ -331,7 +331,7 @@ def test_read_text_mode_without_t(temp_cloud_file): def test_readline(temp_cloud_multiline_file): """Test readline method.""" - with temp_cloud_multiline_file.open(mode="rt") as f: + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: line1 = f.readline() assert line1 == "Line 1\n" @@ -341,7 +341,7 @@ def test_readline(temp_cloud_multiline_file): def test_readlines(temp_cloud_multiline_file): """Test readlines method.""" - with temp_cloud_multiline_file.open(mode="rt") as f: + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: lines = f.readlines() assert len(lines) == 4 assert lines[0] == "Line 1\n" @@ -350,7 +350,7 @@ def test_readlines(temp_cloud_multiline_file): def test_iteration(temp_cloud_multiline_file): """Test iterating over lines.""" - with temp_cloud_multiline_file.open(mode="rt") as f: + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: lines = list(f) assert len(lines) == 4 assert lines[0] == "Line 1\n" From 231c50bd68d3e17e8f1a9d28995adb4434e43d42 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:23:19 -0700 Subject: [PATCH 09/11] Cover streaming extension failure paths --- tests/test_client.py | 46 ++++++++++++++- tests/test_cloud_io.py | 131 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 1d05645c..284bd558 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -7,7 +7,7 @@ import pytest from cloudpathlib import CloudPath -from cloudpathlib.client import register_client_class +from cloudpathlib.client import Client, register_client_class from cloudpathlib.cloudpath import ( implementation_registry, register_path_class, @@ -181,3 +181,47 @@ def test_custom_mys3client_default_client(custom_s3_path): path = CloudPath("mys3://bucket/dir/file.txt") assert isinstance(path.client, CustomClient) assert path.cloud_prefix == "mys3://" + + +@pytest.mark.parametrize( + "call", + [ + lambda client, path: Client._range_download(client, path, 0, 0), + lambda client, path: Client._get_content_length(client, path), + lambda client, path: Client._initiate_multipart_upload(client, path), + lambda client, path: Client._upload_part(client, path, "upload", 1, b"data"), + lambda client, path: Client._complete_multipart_upload(client, path, "upload", []), + lambda client, path: Client._abort_multipart_upload(client, path, "upload"), + lambda client, path: Client._open_write_stream(client, path), + lambda client, path: Client._put_empty_object(client, path), + ], +) +def test_default_streaming_hooks_raise_not_implemented(local_s3_rig, call): + path = local_s3_rig.create_cloud_path("unsupported-stream.bin") + + with pytest.raises(NotImplementedError, match="streaming I/O"): + call(path.client, path) + + +def test_default_write_stream_hooks_delegate(local_s3_rig): + class Stream: + def __init__(self): + self.calls = [] + + def write(self, data): + self.calls.append(("write", data)) + return len(data) + + def close(self): + self.calls.append(("close",)) + + def terminate(self): + self.calls.append(("terminate",)) + + client = local_s3_rig.client_class(**local_s3_rig.required_client_kwargs) + stream = Stream() + + assert Client._write_stream(client, stream, b"data") == 4 + Client._close_write_stream(client, stream) + Client._abort_write_stream(client, stream) + assert stream.calls == [("write", b"data"), ("close",), ("terminate",)] diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index e4af11c3..d4b5904c 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -11,6 +11,7 @@ from cloudpathlib import S3Path, AzureBlobPath, GSPath from cloudpathlib import CloudBufferedIO, CloudTextIO +from cloudpathlib.cloud_io import _CloudStorageRaw from cloudpathlib.enums import FileCacheMode # Sample test data @@ -1489,6 +1490,13 @@ def test_streaming_open_rejects_invalid_modes_without_mutating(local_s3_rig, mod assert path.read_text() == "preserve me" +def test_streaming_open_rejects_non_string_mode(local_s3_rig): + path = local_s3_rig.create_cloud_path("invalid-mode.txt") + + with pytest.raises(TypeError, match="mode must be a string"): + path.open(None) + + @pytest.mark.parametrize( "keyword,value,message", [ @@ -1671,3 +1679,126 @@ def test_s3_invalid_object_state_is_not_eof(s3_rig): path = s3_rig.create_cloud_path("archived.bin") raw = path._cloud_meta.raw_io_class(path.client, path, "rb") assert not raw._is_eof_error(Exception("InvalidObjectState")) + + +def test_raw_read_errors_follow_file_object_semantics(local_s3_rig): + class Raw(_CloudStorageRaw): + def _upload_chunk(self, data): + pass + + def _finalize_upload(self): + pass + + path = local_s3_rig.create_cloud_path("raw-errors.bin") + + unreadable = Raw(path.client, path, "wb") + with pytest.raises(io.UnsupportedOperation, match="not readable"): + unreadable.readinto(bytearray(1)) + + unwritable = Raw(path.client, path, "rb") + with pytest.raises(io.UnsupportedOperation, match="not writable"): + unwritable.write(b"x") + + unwritable.close() + for operation in ( + lambda: unwritable.readinto(bytearray(1)), + lambda: unwritable.seek(0), + unwritable.tell, + lambda: unwritable.write(b"x"), + ): + with pytest.raises(ValueError, match="closed file"): + operation() + + +def test_raw_read_handles_unknown_size_and_provider_eof(local_s3_rig): + class Raw(_CloudStorageRaw): + eof = False + empty = False + + def _get_size(self): + raise OSError("size unavailable") + + def _range_get(self, start, end): + if self.empty: + return b"" + raise OSError("range unavailable") + + def _is_eof_error(self, error): + return self.eof + + def _upload_chunk(self, data): + pass + + def _finalize_upload(self): + pass + + path = local_s3_rig.create_cloud_path("raw-eof.bin") + raw = Raw(path.client, path, "rb") + + with pytest.raises(OSError, match="range unavailable"): + raw.readinto(bytearray(1)) + + raw.eof = True + assert raw.readinto(bytearray(1)) == 0 + + raw.eof = False + raw.empty = True + assert raw.readinto(bytearray(1)) == 0 + + +def test_raw_write_failure_is_sticky_and_aborts_on_close(local_s3_rig): + error = OSError("upload failed") + + class Raw(_CloudStorageRaw): + aborted = False + + def _upload_chunk(self, data): + raise error + + def _finalize_upload(self): + pass + + def _abort_upload(self): + self.aborted = True + + path = local_s3_rig.create_cloud_path("raw-upload-error.bin") + raw = Raw(path.client, path, "wb") + + with pytest.raises(OSError, match="upload failed"): + raw.write(b"first") + with pytest.raises(OSError, match="upload failed"): + raw.write(b"second") + with pytest.raises(OSError, match="upload failed"): + raw.close() + + assert raw.aborted + assert raw.closed + + +@pytest.mark.parametrize("fail_during_write", [True, False]) +def test_raw_abort_failure_does_not_mask_original_error(local_s3_rig, fail_during_write): + original_error = OSError("original failure") + + class Raw(_CloudStorageRaw): + def _upload_chunk(self, data): + if fail_during_write: + raise original_error + + def _finalize_upload(self): + if not fail_during_write: + raise original_error + + def _abort_upload(self): + raise OSError("cleanup failure") + + path = local_s3_rig.create_cloud_path("raw-abort-error.bin") + raw = Raw(path.client, path, "wb") + + if fail_during_write: + with pytest.raises(OSError, match="original failure"): + raw.write(b"data") + else: + raw.write(b"data") + + with pytest.raises(OSError, match="original failure"): + raw.close() From b74fb0621dbbf034013ea61d0b3278c1a66ed6f4 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:14:11 -0700 Subject: [PATCH 10/11] Fix live S3 multipart completion --- cloudpathlib/s3/s3client.py | 6 +++++- tests/test_cloud_io.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 3b26c7f2..8b71e4d1 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -533,7 +533,11 @@ def _upload_part( **self._streaming_extra_args("UploadPart"), ) part = {"PartNumber": part_number, "ETag": response["ETag"]} - part.update({key: value for key, value in response.items() if key.startswith("Checksum")}) + checksum_algorithm = self.boto3_ul_extra_args.get("ChecksumAlgorithm") + if checksum_algorithm is not None: + checksum_key = f"Checksum{checksum_algorithm}" + if checksum_key in response: + part[checksum_key] = response[checksum_key] return part def _complete_multipart_upload( diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index d4b5904c..84b07283 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -1543,6 +1543,9 @@ def test_streaming_binary_mode_supports_unbuffered_io(local_s3_rig): def test_s3_streaming_routes_extra_args_by_operation(s3_rig): + if s3_rig.live_server: + pytest.skip("Synthetic SDK argument-routing test") + path = s3_rig.create_cloud_path("streaming-extra-args.bin") client = path.client original_extra_args = client.boto3_ul_extra_args @@ -1588,7 +1591,32 @@ def record_complete(**kwargs): path.unlink(missing_ok=True) +def test_s3_streaming_ignores_automatic_part_checksums(s3_rig, monkeypatch): + path = s3_rig.create_cloud_path("automatic-checksum.bin") + client = path.client + original_extra_args = client.boto3_ul_extra_args + + monkeypatch.setattr( + client.client, + "upload_part", + lambda **kwargs: {"ETag": '"etag"', "ChecksumCRC32": "checksum"}, + ) + try: + client.boto3_ul_extra_args = {} + part = client._upload_part(path, "upload", 1, b"data") + assert part == {"PartNumber": 1, "ETag": '"etag"'} + + client.boto3_ul_extra_args = {"ChecksumAlgorithm": "CRC32"} + part = client._upload_part(path, "upload", 1, b"data") + assert part["ChecksumCRC32"] == "checksum" + finally: + client.boto3_ul_extra_args = original_extra_args + + def test_gs_streaming_range_is_inclusive_and_forwards_options(gs_rig, monkeypatch): + if gs_rig.live_server: + pytest.skip("Synthetic SDK option-forwarding test") + from tests.mock_clients.mock_gs import MockBlob path = gs_rig.create_cloud_path("range-options.bin") From 71bf2b33d2ab7800dc183a72a893d6f8f7d844a7 Mon Sep 17 00:00:00 2001 From: Peter Bull <1799186+pjbull@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:58:48 -0700 Subject: [PATCH 11/11] Fix review findings, add streaming concurrency, move GS to XML multipart Review fixes (verified by adversarial review of this PR): - write() advances the raw position so tell() is correct on streaming write streams (fixes corrupt output from zipfile and other position-dependent writers) - seek() on write-only raw streams raises io.UnsupportedOperation as documented instead of silently accepting the seek - GS range errors are matched structurally (status 416) instead of by a "416" substring that could swallow unrelated transient errors - Azure block IDs are namespaced by a per-upload session ID so concurrent writers cannot clobber each other's staged blocks - read() to EOF fetches the remainder in one ranged request (readall) instead of one request per 8 KiB - copy()/rename()/replace() work in streaming mode by streaming between clients instead of round-tripping through fspath - streaming writes honor force_overwrite_to_cloud, raising OverwriteNewerCloudError on close instead of clobbering a newer object - Client.__del__ cleans up streaming-mode cache files from the append/update fallback; docs no longer claim streaming never touches disk - streaming error paths raise cloudpathlib exception types; failed size lookups are memoized per stream; negative buffering matches builtins.open; behavior changes recorded in HISTORY.md Coverage and fidelity: - S3 extra-args filtering consults botocore's bundled service model instead of a hand-copied (and already stale) fallback table, with a drift-guard test - mocks now raise realistic SDK errors for missing objects and past-EOF ranges; HTTP test server clamps range ends per RFC 7233; GS mock enforces the 5 MiB non-final part minimum - default streaming buffer raised to 5 MiB; regression tests for all of the above plus parquet-over-streaming compatibility (pyarrow dev dep) Streaming concurrency: - new streaming_max_concurrency client parameter: bounded background part uploads while writing and read-ahead prefetch while reading, sequential by default - GS streaming writes use the XML API multipart upload (via the SDK's transfer-manager machinery) with in-memory parts, replacing the resumable write stream and unifying all multipart providers on one write contract; mock GS implements the XML MPU wire protocol Co-Authored-By: Claude Fable 5 --- HISTORY.md | 11 +- cloudpathlib/azure/azblobclient.py | 25 +- cloudpathlib/client.py | 34 +- cloudpathlib/cloud_io.py | 196 +++++- cloudpathlib/cloudpath.py | 106 ++- cloudpathlib/gs/gs_io.py | 45 +- cloudpathlib/gs/gsclient.py | 136 +++- cloudpathlib/http/httpclient.py | 28 +- cloudpathlib/local/localclient.py | 51 +- cloudpathlib/s3/s3client.py | 73 +-- docs/docs/caching.ipynb | 2 +- docs/docs/streaming_io.md | 152 ++++- requirements-dev.txt | 1 + tests/http_fixtures.py | 6 +- tests/mock_clients/mock_azureblob.py | 10 +- tests/mock_clients/mock_gs.py | 138 ++-- tests/mock_clients/mock_s3.py | 11 + tests/test_caching.py | 51 ++ tests/test_client.py | 25 - tests/test_cloud_io.py | 946 ++++++++++++++++++++++++++- 20 files changed, 1726 insertions(+), 321 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 1d95a873..7c9a77d1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -11,7 +11,16 @@ - Added `CloudTextIO` class implementing `io.TextIOBase` for text streaming operations. - Added provider-specific raw I/O implementations: `_S3StorageRaw`, `_AzureBlobStorageRaw`, `_GSStorageRaw`, `_HttpStorageRaw`. - Added `register_raw_io_class` decorator for registering streaming I/O implementations. - - Added `buffer_size` parameter to `CloudPath.open()` for controlling streaming buffer size. + - Added `buffer_size` parameter to `CloudPath.open()` for controlling streaming buffer size; the default is 5 MiB, matching the block sizes of comparable tools (a full-object `read()` always uses a single ranged request regardless of buffer size). + - Streaming upload extra args for S3 are filtered against botocore's bundled service model (also when the client object does not expose `meta`), so newly added S3 parameters are never silently dropped. + - Added a `streaming_max_concurrency` client parameter (default 1): each open streaming stream may issue up to that many requests in parallel — background part uploads while writing (in-flight memory bounded to concurrency × part size) and read-ahead prefetch of upcoming byte ranges while reading sequentially. + - Google Cloud Storage streaming writes use the XML API multipart upload (via the SDK's transfer-manager machinery) instead of a resumable-upload stream, matching the S3/Azure part mechanism and enabling concurrent part uploads. + - Streaming writes honor `force_overwrite_to_cloud` (and `CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD`), raising `OverwriteNewerCloudError` on close instead of overwriting an object that changed while the stream was open. + - `copy`/`rename`/`replace` work in streaming mode by streaming between clients instead of round-tripping through the local cache (`fspath`). + - Cache files created by the append/update fallback in streaming mode are cleaned up when the client is garbage collected. + - Streaming error paths raise `cloudpathlib.exceptions` types (`CloudPathFileNotFoundError`, `CloudPathNotImplementedError`), which subclass the corresponding builtins. +- Changed `CloudPath.open(mode="a")` on a nonexistent cloud file to create it (matching the stdlib `open` and `pathlib`) instead of raising `CloudPathFileNotFoundError`. **Breaking change for users that relied on the previous error.** (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) +- Changed the cached-write upload tie-break so a save that leaves the cache file's modification time exactly equal to the cloud version's (e.g., same-second writes on coarse-resolution filesystems) uploads instead of raising a spurious `OverwriteNewerCloudError`. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) - Fixed mypy 2.x type errors in `Client` and `CloudPath` that caused CI lint failures (Issue [#563](https://github.com/drivendataorg/cloudpathlib/issues/563), PR [#566](https://github.com/drivendataorg/cloudpathlib/pull/566)) - Changed `S3Client._get_metadata` to read object metadata with `HeadObject` instead of `GetObject`, so `stat`, `etag`, and `size` no longer open the object body. Also fixes a `KeyError` on `ContentLength` against S3-compatible gateways that drop `Content-Length` from `GetObject` responses. (Issue [#564](https://github.com/drivendataorg/cloudpathlib/issues/564), PR [#565](https://github.com/drivendataorg/cloudpathlib/pull/565)) - Added a `lazy` keyword argument to `CloudPath.walk`. By default (`lazy=False`) the existing fast behavior is preserved: the whole subtree is fetched up front with a single recursive listing. Passing `lazy=True` lists each directory on demand so that, when `top_down=True`, callers can prune subdirectories by modifying `dirnames` in-place (à la `os.walk` / `Path.walk`) to skip fetching the contents of those subtrees entirely — dramatically reducing API calls for large, sparsely-traversed trees. (Issue [#518](https://github.com/drivendataorg/cloudpathlib/issues/518)) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 720cc912..a52b5359 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union from itertools import islice +from uuid import uuid4 try: from typing import cast @@ -14,7 +15,7 @@ from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode -from ..exceptions import MissingCredentialsError +from ..exceptions import CloudPathFileNotFoundError, MissingCredentialsError from .azblobpath import AzureBlobPath try: @@ -61,6 +62,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, ): """Class constructor. Sets up a [`BlobServiceClient`]( https://docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobserviceclient?view=azure-python). @@ -108,11 +110,15 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). """ super().__init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) if connection_string is None: @@ -507,7 +513,7 @@ def _range_download(self, cloud_path: AzureBlobPath, start: int, end: int) -> by downloader = blob_client.download_blob(offset=start, length=length) return downloader.readall() except ResourceNotFoundError: - raise FileNotFoundError(f"Azure blob not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"Azure blob not found: {cloud_path}") except HttpResponseError as e: if (e.error and e.error.code == "InvalidRange") or e.status_code == 416: return b"" @@ -522,11 +528,16 @@ def _get_content_length(self, cloud_path: AzureBlobPath) -> int: properties = blob_client.get_blob_properties() return properties.size except ResourceNotFoundError: - raise FileNotFoundError(f"Azure blob not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"Azure blob not found: {cloud_path}") def _initiate_multipart_upload(self, cloud_path: AzureBlobPath) -> str: - """Return the stateless Azure upload ID.""" - return "" + """Return a unique session ID that namespaces this upload's block IDs. + + Azure's uncommitted-block namespace is per-blob, so deterministic block IDs + would let concurrent writers to the same blob overwrite each other's staged + blocks and commit interleaved data. + """ + return uuid4().hex def _upload_part( self, cloud_path: AzureBlobPath, upload_id: str, part_number: int, data: bytes @@ -537,7 +548,9 @@ def _upload_part( blob_client = self.service_client.get_blob_client( container=cloud_path.container, blob=cloud_path.blob ) - block_id = base64.b64encode(f"block-{part_number:06d}".encode()).decode() + # Azure requires all block IDs for a blob to be the same length; uuid4().hex (32) + # plus a fixed-width part number keeps them uniform. + block_id = base64.b64encode(f"{upload_id}-{part_number:06d}".encode()).decode() blob_client.stage_block(block_id=block_id, data=data, length=len(data)) return {"block_id": block_id} diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 86a7a9d9..85621400 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -12,7 +12,6 @@ Generic, Iterable, Optional, - Protocol, Sequence, Tuple, TypeVar, @@ -27,14 +26,6 @@ _UploadPart = Dict[str, Any] -class _CloudWriteStream(Protocol): - def write(self, data: bytes) -> int: ... - - def close(self) -> None: ... - - def terminate(self) -> None: ... - - def register_client_class(key: str) -> Callable: def decorator(cls: type) -> type: if not issubclass(cls, Client): @@ -56,11 +47,18 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, ) -> None: self.file_cache_mode = None self._cache_tmp_dir = None self._cloud_meta.validate_completeness() + if streaming_max_concurrency < 1: + raise ValueError("streaming_max_concurrency must be at least 1") + # concurrent requests per open streaming stream (part uploads / read prefetch); + # 1 means fully sequential I/O + self.streaming_max_concurrency = streaming_max_concurrency + # convert strings passed to enum if isinstance(file_cache_mode, str): file_cache_mode = FileCacheMode(file_cache_mode) @@ -110,6 +108,9 @@ def __del__(self) -> None: FileCacheMode.tmp_dir, FileCacheMode.close_file, FileCacheMode.cloudpath_object, + # streaming avoids the cache except for append/update fallbacks, which + # should not outlive the client + FileCacheMode.streaming, ]: self.clear_cache() @@ -248,21 +249,6 @@ def _abort_multipart_upload(self, cloud_path: BoundedCloudPath, upload_id: str) f"{type(self).__name__} does not support streaming I/O (_abort_multipart_upload)." ) - def _open_write_stream(self, cloud_path: BoundedCloudPath) -> _CloudWriteStream: - """Open a provider write stream.""" - raise NotImplementedError( - f"{type(self).__name__} does not support streaming I/O (_open_write_stream)." - ) - - def _write_stream(self, stream: _CloudWriteStream, data: bytes) -> int: - return stream.write(data) - - def _close_write_stream(self, stream: _CloudWriteStream) -> None: - stream.close() - - def _abort_write_stream(self, stream: _CloudWriteStream) -> None: - stream.terminate() - def _put_empty_object(self, cloud_path: BoundedCloudPath) -> None: """Create an empty object.""" raise NotImplementedError( diff --git a/cloudpathlib/cloud_io.py b/cloudpathlib/cloud_io.py index 6c1d0893..3351ced3 100644 --- a/cloudpathlib/cloud_io.py +++ b/cloudpathlib/cloud_io.py @@ -4,8 +4,9 @@ import io from abc import abstractmethod +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait from types import TracebackType -from typing import TYPE_CHECKING, Any, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, Union if TYPE_CHECKING: from _typeshed import ReadableBuffer as _ReadableBuffer @@ -17,6 +18,12 @@ from .client import Client from .cloudpath import CloudPath +# Bytes fetched/buffered per request for buffered streaming I/O. Sized to match the +# multi-MiB block sizes used by comparable tools (fsspec/s3fs/gcsfs) so per-request +# latency does not dominate sequential throughput; reads never fetch past EOF, so +# small objects only pay for their actual size. +DEFAULT_BUFFER_SIZE = 5 * 1024 * 1024 + def _validate_file_mode(mode: str) -> None: """Validate an ``open`` mode using the same grammar as the stdlib.""" @@ -47,8 +54,15 @@ def __init__( self._mode = mode self._pos = 0 self._size: Optional[int] = None + self._size_fetch_failed = False self._closed = False self._upload_error: Optional[BaseException] = None + # Optional conflict check run just before a write is finalized (set by CloudPath.open) + self._pre_finalize: Optional[Callable[[], None]] = None + # concurrent requests for this stream (read prefetch / background part uploads) + self._max_concurrency = max(1, int(getattr(client, "streaming_max_concurrency", 1))) + self._executor: Optional[ThreadPoolExecutor] = None + self._prefetch: Dict[int, Future] = {} def readable(self) -> bool: """Return whether object was opened for reading.""" @@ -77,19 +91,14 @@ def readinto(self, b: _WriteableBuffer, /) -> int: start = self._pos end = start + len(view) - 1 - if self._size is None: - try: - self._size = self._get_size() - except Exception: - pass - - if self._size is not None and end >= self._size: - end = self._size - 1 - if start >= self._size: + size = self._known_size() + if size is not None and end >= size: + end = size - 1 + if start >= size: return 0 try: - data = self._range_get(start, end) + data = self._fetch_range(start, end) except Exception as e: if self._is_eof_error(e): return 0 @@ -105,6 +114,25 @@ def readinto(self, b: _WriteableBuffer, /) -> int: self._pos += n return n + def readall(self) -> bytes: + """Read from the current position to EOF in a single ranged request when possible.""" + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.readable(): + raise io.UnsupportedOperation("not readable") + + self._discard_prefetch() + size = self._known_size() + if size is None: + # Size unknown: fall back to the default chunked read loop. + return super().readall() + if self._pos >= size: + return b"" + + data = self._range_get(self._pos, size - 1) + self._pos += len(data) + return data + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: """ Change stream position. @@ -118,17 +146,18 @@ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: """ if self._closed: raise ValueError("I/O operation on closed file") + if not self.seekable(): + raise io.UnsupportedOperation("seek") if whence == io.SEEK_SET: new_pos = offset elif whence == io.SEEK_CUR: new_pos = self._pos + offset elif whence == io.SEEK_END: - if self._size is None: - self._size = self._get_size() - if self._size is None: + size = self._known_size() + if size is None: raise OSError("Unable to determine file size for SEEK_END") - new_pos = self._size + offset + new_pos = size + offset else: raise ValueError( f"invalid whence ({whence}, should be {io.SEEK_SET}, " @@ -138,6 +167,8 @@ def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: if new_pos < 0: raise ValueError("negative seek position") + if new_pos != self._pos: + self._discard_prefetch() self._pos = new_pos return self._pos @@ -161,6 +192,7 @@ def write(self, b: _ReadableBuffer, /) -> int: except BaseException as error: self._upload_error = error raise + self._pos += len(data) return len(data) def close(self) -> None: @@ -180,6 +212,8 @@ def close(self) -> None: raise self._upload_error if self.writable(): try: + if self._pre_finalize is not None: + self._pre_finalize() self._finalize_upload() except BaseException: try: @@ -188,6 +222,7 @@ def close(self) -> None: pass raise finally: + self._shutdown_executor() super().close() def _abort_upload(self) -> None: @@ -202,12 +237,76 @@ def _upload_chunk(self, data: bytes) -> None: def _finalize_upload(self) -> None: pass + def _ensure_executor(self) -> Optional[ThreadPoolExecutor]: + """Thread pool for this stream's background requests; None when sequential.""" + if self._max_concurrency <= 1: + return None + if self._executor is None: + self._executor = ThreadPoolExecutor( + max_workers=self._max_concurrency, thread_name_prefix="cloudpathlib-stream" + ) + return self._executor + + def _shutdown_executor(self) -> None: + self._discard_prefetch() + if self._executor is not None: + self._executor.shutdown(wait=True, cancel_futures=True) + self._executor = None + + def _discard_prefetch(self) -> None: + for future in self._prefetch.values(): + future.cancel() + self._prefetch.clear() + + def _fetch_range(self, start: int, end: int) -> bytes: + """Fetch [start, end], serving from and topping up background prefetch when enabled.""" + executor = self._ensure_executor() + if executor is None: + return self._range_get(start, end) + + chunk_len = end - start + 1 + future = self._prefetch.pop(start, None) + if future is not None: + # a short prefetched chunk is a legal short read for RawIOBase consumers + data = future.result() + self._schedule_prefetch(start + max(len(data), 1), chunk_len) + return data + + # position changed or first read: pending prefetches no longer line up + self._discard_prefetch() + data = self._range_get(start, end) + self._schedule_prefetch(end + 1, chunk_len) + return data + + def _schedule_prefetch(self, next_start: int, chunk_len: int) -> None: + """Queue reads ahead of the current position, up to the concurrency window.""" + size = self._known_size() + executor = self._ensure_executor() + if size is None or chunk_len <= 0 or executor is None: + return + start = next_start + while len(self._prefetch) < self._max_concurrency and start < size: + if start not in self._prefetch: + self._prefetch[start] = executor.submit( + self._range_get, start, min(start + chunk_len, size) - 1 + ) + start += chunk_len + def _range_get(self, start: int, end: int) -> bytes: return self._client._range_download(self._cloud_path, start, end) def _get_size(self) -> int: return self._client._get_content_length(self._cloud_path) + def _known_size(self) -> Optional[int]: + """Fetch and memoize the object size, attempting the lookup at most once.""" + if self._size is None and not self._size_fetch_failed: + try: + self._size = self._get_size() + except Exception: + self._size_fetch_failed = True + return self._size + def _is_eof_error(self, error: Exception) -> bool: """ Check if an error indicates EOF/out of range. @@ -229,7 +328,8 @@ class _CloudMultipartStorageRaw(_CloudStorageRaw): def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: super().__init__(client, cloud_path, mode) self._upload_id: Optional[str] = None - self._parts: list[dict[str, Any]] = [] + self._parts: Dict[int, dict[str, Any]] = {} + self._part_futures: Dict[int, Future] = {} self._part_number = 1 self._write_buffer = bytearray() @@ -252,13 +352,43 @@ def _upload_buffered_part(self, size: int) -> None: if self._upload_id is None: self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) data = bytes(self._write_buffer[:size]) - part = self._client._upload_part( - self._cloud_path, self._upload_id, self._part_number, data - ) del self._write_buffer[:size] - self._parts.append(part) + part_number = self._part_number self._part_number += 1 + executor = self._ensure_executor() + if executor is None: + self._parts[part_number] = self._client._upload_part( + self._cloud_path, self._upload_id, part_number, data + ) + return + + # bound in-flight parts (and their buffered bytes) to the concurrency window + self._harvest_part_futures(block=len(self._part_futures) >= self._max_concurrency) + self._part_futures[part_number] = executor.submit( + self._client._upload_part, self._cloud_path, self._upload_id, part_number, data + ) + + def _harvest_part_futures(self, block: bool = False, drain: bool = False) -> None: + """Collect finished background part uploads, re-raising the first failure.""" + if not self._part_futures: + return + if drain: + wait(list(self._part_futures.values())) + elif block: + wait(list(self._part_futures.values()), return_when=FIRST_COMPLETED) + + error: Optional[BaseException] = None + for part_number in [n for n, f in self._part_futures.items() if f.done()]: + future = self._part_futures.pop(part_number) + try: + self._parts[part_number] = future.result() + except BaseException as e: + if error is None: + error = e + if error is not None: + raise error + def _upload_chunk(self, data: bytes) -> None: if not data: return @@ -271,14 +401,19 @@ def _upload_chunk(self, data: bytes) -> None: def _finalize_upload(self) -> None: if self._write_buffer: self._upload_buffered_part(len(self._write_buffer)) + self._harvest_part_futures(drain=True) if self._upload_id is None: self._client._put_empty_object(self._cloud_path) return - self._client._complete_multipart_upload(self._cloud_path, self._upload_id, self._parts) + ordered_parts = [self._parts[number] for number in sorted(self._parts)] + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, ordered_parts) self._reset_upload() def _abort_upload(self) -> None: try: + for future in self._part_futures.values(): + future.cancel() + wait(list(self._part_futures.values())) if self._upload_id is not None: self._client._abort_multipart_upload(self._cloud_path, self._upload_id) finally: @@ -287,6 +422,7 @@ def _abort_upload(self) -> None: def _reset_upload(self) -> None: self._upload_id = None self._parts.clear() + self._part_futures.clear() self._part_number = 1 self._write_buffer.clear() @@ -300,7 +436,8 @@ def __init__( client: Client, cloud_path: CloudPath, mode: str = "rb", - buffer_size: int = 64 * 1024, + buffer_size: int = DEFAULT_BUFFER_SIZE, + pre_finalize: Optional[Callable[[], None]] = None, ) -> None: _validate_file_mode(mode) if "b" not in mode: @@ -311,6 +448,8 @@ def __init__( ) raw = raw_io_class(client, cloud_path, mode) + if pre_finalize is not None: + raw._pre_finalize = pre_finalize if "r" in mode: self._buffer: Union[io.BufferedReader, io.BufferedWriter] @@ -403,8 +542,9 @@ def __init__( encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, - buffer_size: int = 64 * 1024, + buffer_size: int = DEFAULT_BUFFER_SIZE, line_buffering: bool = False, + pre_finalize: Optional[Callable[[], None]] = None, ) -> None: _validate_file_mode(mode) if "b" in mode: @@ -414,19 +554,23 @@ def __init__( "append and update modes require the local-cache implementation" ) + # only r/w/x can reach here: 'b' was rejected above and 'a'/'+' raised earlier if "t" not in mode and "r" in mode: binary_mode = mode.replace("r", "rb", 1) elif "t" not in mode and "w" in mode: binary_mode = mode.replace("w", "wb", 1) - elif "t" not in mode and "a" in mode: - binary_mode = mode.replace("a", "ab", 1) elif "t" not in mode and "x" in mode: binary_mode = mode.replace("x", "xb", 1) else: binary_mode = mode.replace("t", "b") buffered = CloudBufferedIO( - raw_io_class, client, cloud_path, mode=binary_mode, buffer_size=buffer_size + raw_io_class, + client, + cloud_path, + mode=binary_mode, + buffer_size=buffer_size, + pre_finalize=pre_finalize, ) super().__init__( diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index a3e0f4d0..3188d4dc 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -384,8 +384,8 @@ def __fspath__(self) -> str: # Check if streaming mode is enabled if self.client.file_cache_mode == FileCacheMode.streaming: raise CloudPathNotImplementedError( - "fspath is not available in streaming mode. " - "Streaming mode does not create cached files on disk. " + "fspath is not available in streaming mode, which avoids the local file cache " + "(except for append/update modes of `open`, which fall back to it). " "Use CloudPath.open() to read/write data directly." ) @@ -818,8 +818,6 @@ def open( raise ValueError("binary mode doesn't take a newline argument") if not binary_mode and buffering == 0: raise ValueError("can't have unbuffered text I/O") - if buffering < -1: - raise ValueError("invalid buffering size") if buffer_size is not None and buffer_size <= 0: raise ValueError("buffer_size must be greater than zero") @@ -845,24 +843,34 @@ def open( _streaming_unsupported = any(m in mode for m in ("a", "+")) if self.client.file_cache_mode == FileCacheMode.streaming and not _streaming_unsupported: # Import here to keep it localized to streaming functionality - from .cloud_io import CloudBufferedIO, CloudTextIO + from .cloud_io import DEFAULT_BUFFER_SIZE, CloudBufferedIO, CloudTextIO # Get the raw IO class from the cloud implementation raw_io_class = self._cloud_meta.raw_io_class if raw_io_class is None: - raise NotImplementedError( + raise CloudPathNotImplementedError( f"Streaming I/O is not implemented for {self._cloud_meta.name}" ) + # Overwrite protection mirroring the cached path's upload conflict check + pre_finalize = None + if "w" in mode or "x" in mode: + pre_finalize = self._streaming_overwrite_check( + exists_on_cloud, force_overwrite_to_cloud + ) + # Calculate buffer size from buffering or buffer_size parameter if buffer_size is None: if buffering == 0: # A raw provider adapter is the streaming equivalent of FileIO. - return raw_io_class(self.client, self, mode) # type: ignore[return-value] + raw = raw_io_class(self.client, self, mode) + if pre_finalize is not None: + raw._pre_finalize = pre_finalize + return raw # type: ignore[return-value] elif buffering > 0: buffer_size = buffering else: - buffer_size = 64 * 1024 # Default 64 KiB + buffer_size = DEFAULT_BUFFER_SIZE # Return appropriate streaming I/O object if "b" in mode: @@ -872,6 +880,7 @@ def open( cloud_path=self, mode=mode, buffer_size=buffer_size, + pre_finalize=pre_finalize, ) else: return CloudTextIO( # type: ignore[return-value] @@ -884,6 +893,7 @@ def open( newline=newline, buffer_size=buffer_size, line_buffering=buffering == 1, + pre_finalize=pre_finalize, ) # Standard cached mode @@ -952,6 +962,38 @@ def _patched_close_empty_cache(*args, **kwargs): return buffer + def _streaming_overwrite_check( + self, exists_on_cloud: bool, force_overwrite_to_cloud: Optional[bool] + ) -> Optional[Callable[[], None]]: + """Build the pre-upload conflict check for a streaming write, mirroring the cached + path's `OverwriteNewerCloudError` protection in `_upload_file_to_cloud`. Returns None + when overwriting is forced.""" + if force_overwrite_to_cloud is None: + force_overwrite_to_cloud = os.environ.get( + "CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD", "False" + ).lower() in ["1", "true"] + + if force_overwrite_to_cloud: + return None + + original_mtime = self.stat().st_mtime if exists_on_cloud else None + + def check() -> None: + try: + stats = self.stat() + except (NoStatError, CloudPathFileNotFoundError, FileNotFoundError): + # nothing on the cloud to conflict with + return + if original_mtime is None or stats.st_mtime > original_mtime: + raise OverwriteNewerCloudError( + f"Cloud path ({self}) changed while it was open for streaming write, " + f"but is being requested to be overwritten on close. Either (1) pass " + f"`force_overwrite_to_cloud=True` to overwrite; or (2) set env var " + f"CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD=1." + ) + + return check + def replace(self, target: Self) -> Self: if type(self) is not type(target): raise TypeError( @@ -1357,19 +1399,43 @@ def _copy( else: if not destination.exists() or destination.is_file(): - return cast( - Union[Path, Self], - destination.upload_from( - self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud - ), - ) + target_path: CloudPath = destination else: - return cast( - Union[Path, Self], - (destination / self.name).upload_from( - self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud - ), - ) + target_path = destination / self.name + + # streaming mode has no local cache to round-trip through (fspath is + # unavailable), so copy by streaming between the two clients directly + if self.client.file_cache_mode == FileCacheMode.streaming: + if force_overwrite_to_cloud is None: + force_overwrite_to_cloud = os.environ.get( + "CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD", "False" + ).lower() in ["1", "true"] + + if ( + not force_overwrite_to_cloud + and target_path.exists() + and target_path.stat().st_mtime >= self.stat().st_mtime + ): + raise OverwriteNewerCloudError( + f"File ({target_path}) is newer than ({self}). " + f"To overwrite " + f"pass `force_overwrite_to_cloud=True`." + ) + + with ( + self.open("rb") as src_file, + target_path.open("wb", force_overwrite_to_cloud=True) as dst_file, + ): + shutil.copyfileobj(src_file, dst_file) + + return cast(Union[Path, Self], target_path) + + return cast( + Union[Path, Self], + target_path.upload_from( + self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud + ), + ) @overload def copy( diff --git a/cloudpathlib/gs/gs_io.py b/cloudpathlib/gs/gs_io.py index e53ec968..6b9bf298 100644 --- a/cloudpathlib/gs/gs_io.py +++ b/cloudpathlib/gs/gs_io.py @@ -1,39 +1,16 @@ """Google Cloud Storage streaming I/O.""" -from __future__ import annotations - -from typing import Optional - -from ..client import Client, _CloudWriteStream -from ..cloud_io import _CloudStorageRaw -from ..cloudpath import CloudPath, register_raw_io_class +from ..cloud_io import _CloudMultipartStorageRaw +from ..cloudpath import register_raw_io_class @register_raw_io_class("gs") -class _GSStorageRaw(_CloudStorageRaw): - """GCS range reads and resumable writes.""" - - def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: - super().__init__(client, cloud_path, mode) - self._writer: Optional[_CloudWriteStream] = None - - def _upload_chunk(self, data: bytes) -> None: - if not data: - return - if self._writer is None: - self._writer = self._client._open_write_stream(self._cloud_path) - self._client._write_stream(self._writer, data) - - def _finalize_upload(self) -> None: - if self._writer is None: - self._client._put_empty_object(self._cloud_path) - return - self._client._close_write_stream(self._writer) - self._writer = None - - def _abort_upload(self) -> None: - if self._writer is not None: - try: - self._client._abort_write_stream(self._writer) - finally: - self._writer = None +class _GSStorageRaw(_CloudMultipartStorageRaw): + """GCS range reads and XML multipart-upload writes.""" + + # GCS XML multipart uploads require non-final parts of at least 5 MiB. + _INITIAL_PART_SIZE = 5 * 1024 * 1024 + _MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + _MAX_PARTS = 10_000 + _PARTS_PER_SIZE_TIER = 1_000 + _PROVIDER_NAME = "GCS multipart" diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index baf30a33..99795a93 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -1,13 +1,15 @@ from datetime import datetime, timedelta +from functools import lru_cache import mimetypes import os from pathlib import Path, PurePosixPath -from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union, Sequence import warnings -from ..client import Client, _CloudWriteStream, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode +from ..exceptions import CloudPathFileNotFoundError, CloudPathNotImplementedError from .gspath import GSPath try: @@ -18,7 +20,7 @@ from google.api_core.exceptions import NotFound as GCSNotFound from google.auth import default as google_default_auth from google.auth.exceptions import DefaultCredentialsError - from google.cloud.storage import Client as StorageClient + from google.cloud.storage.client import Client as StorageClient except ModuleNotFoundError: implementation_registry["gs"].dependencies_loaded = False @@ -26,9 +28,49 @@ try: - from google.cloud.storage import transfer_manager + import google.cloud.storage.transfer_manager as transfer_manager except ImportError: - transfer_manager = None + transfer_manager = None # type: ignore[assignment] + +try: + from google.cloud.storage._media.requests import XMLMPUContainer, XMLMPUPart +except ImportError: + try: + # older google-cloud-storage versions expose these via google-resumable-media + from google.resumable_media.requests import ( # type: ignore[assignment,no-redef] + XMLMPUContainer, + XMLMPUPart, + ) + except ImportError: + XMLMPUContainer = None # type: ignore[assignment,misc] + XMLMPUPart = None # type: ignore[assignment,misc] + + +@lru_cache(maxsize=1) +def _bytes_mpu_part_class() -> type: + """XMLMPUPart subclass that uploads an in-memory payload instead of a file slice.""" + + class _BytesXMLMPUPart(XMLMPUPart): + def __init__(self, upload_url, upload_id, data, part_number, headers=None): + super().__init__( + upload_url, + upload_id, + filename="", + start=0, + end=len(data), + part_number=part_number, + headers=headers, + checksum=None, + ) + self._data = bytes(data) + + def _prepare_upload_request(self): + if self.finished: + raise ValueError("This part has already been uploaded.") + query = f"?partNumber={self._part_number}&uploadId={self._upload_id}" + return "PUT", self.upload_url + query, self._data, self._headers + + return _BytesXMLMPUPart @register_client_class("gs") @@ -48,6 +90,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, download_chunks_concurrently_kwargs: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, retry: Optional["Retry"] = None, @@ -84,6 +127,9 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). download_chunks_concurrently_kwargs (Optional[Dict[str, Any]]): Keyword arguments to pass to [`download_chunks_concurrently`](https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.transfer_manager#google_cloud_storage_transfer_manager_download_chunks_concurrently) for sliced parallel downloads; Only available in `google-cloud-storage` version 2.7.0 or later, otherwise ignored and a warning is emitted. @@ -124,6 +170,7 @@ def __init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) def _get_metadata(self, cloud_path: GSPath) -> Optional[Dict[str, Any]]: @@ -318,12 +365,14 @@ def _range_download(self, cloud_path: GSPath, start: int, end: int) -> bytes: try: return blob.download_as_bytes(start=start, end=end, **self.blob_kwargs) except GCSNotFound: - raise FileNotFoundError(f"GCS object not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"GCS object not found: {cloud_path}") except Exception as e: - error_str = str(e) - if "416" in error_str or "Requested Range Not Satisfiable" in error_str: - return b"" - if hasattr(e, "code") and e.code == 416: + # match a range-past-EOF error structurally (status 416) rather than by + # substring, so unrelated errors are not silently treated as EOF + status = getattr(e, "code", None) + if status is None: + status = getattr(getattr(e, "response", None), "status_code", None) + if status == 416 or "Requested Range Not Satisfiable" in str(e): return b"" raise @@ -334,20 +383,63 @@ def _get_content_length(self, cloud_path: GSPath) -> int: blob.reload(**self.blob_kwargs) return blob.size except GCSNotFound: - raise FileNotFoundError(f"GCS object not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"GCS object not found: {cloud_path}") - def _open_write_stream(self, cloud_path: GSPath) -> _CloudWriteStream: - """Open a GCS resumable upload.""" - blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) - kwargs: Dict[str, Any] = {} + def _mpu_url(self, cloud_path: GSPath) -> str: + """XML API URL for a multipart upload of this object.""" + from urllib.parse import quote + + connection = self.client._connection + hostname = ( + connection.get_api_base_url_for_mtls() + if hasattr(connection, "get_api_base_url_for_mtls") + else connection.API_BASE_URL + ) + return f"{hostname}/{cloud_path.bucket}/{quote(cloud_path.blob)}" + + def _initiate_multipart_upload(self, cloud_path: GSPath) -> str: + """Start a GCS XML multipart upload, threading content type and encoding.""" + if XMLMPUContainer is None: + raise CloudPathNotImplementedError( + "Streaming writes require google-cloud-storage with XML multipart support." + ) + content_type = None + headers = {} if self.content_type_method is not None: - content_type, _ = self.content_type_method(str(cloud_path)) - if content_type is not None: - kwargs["content_type"] = content_type - for k in ("timeout", "retry"): - if k in self.blob_kwargs: - kwargs[k] = self.blob_kwargs[k] - return blob.open("wb", **kwargs) + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_encoding is not None: + headers["Content-Encoding"] = content_encoding + container = XMLMPUContainer(self._mpu_url(cloud_path), cloud_path.blob, headers=headers) + container.initiate( + transport=self.client._http, content_type=content_type or "application/octet-stream" + ) + return container.upload_id + + def _upload_part( + self, cloud_path: GSPath, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Upload one part of a GCS XML multipart upload.""" + part = _bytes_mpu_part_class()(self._mpu_url(cloud_path), upload_id, data, part_number) + part.upload(self.client._http) + return {"part_number": part_number, "etag": part.etag} + + def _complete_multipart_upload( + self, cloud_path: GSPath, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + """Finalize a GCS XML multipart upload.""" + container = XMLMPUContainer( + self._mpu_url(cloud_path), cloud_path.blob, upload_id=upload_id + ) + for part in parts: + container.register_part(part["part_number"], part["etag"]) + container.finalize(self.client._http) + + def _abort_multipart_upload(self, cloud_path: GSPath, upload_id: str) -> None: + """Cancel a GCS XML multipart upload, discarding uploaded parts.""" + container = XMLMPUContainer( + self._mpu_url(cloud_path), cloud_path.blob, upload_id=upload_id + ) + container.cancel(self.client._http) def _put_empty_object(self, cloud_path: GSPath) -> None: """Upload a zero-byte GCS object.""" diff --git a/cloudpathlib/http/httpclient.py b/cloudpathlib/http/httpclient.py index 5dcecbb6..790a5043 100644 --- a/cloudpathlib/http/httpclient.py +++ b/cloudpathlib/http/httpclient.py @@ -13,6 +13,7 @@ from cloudpathlib.client import Client, register_client_class from cloudpathlib.enums import FileCacheMode +from cloudpathlib.exceptions import CloudPathFileNotFoundError, CloudPathNotImplementedError from .httppath import HttpPath @@ -24,6 +25,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, auth: Optional[urllib.request.BaseHandler] = None, custom_list_page_parser: Optional[Callable[[str], Iterable[str]]] = None, custom_dir_matcher: Optional[Callable[[str], bool]] = None, @@ -41,12 +43,20 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when uploading files. Defaults to `mimetypes.guess_type`. + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). auth (Optional[urllib.request.BaseHandler]): Authentication handler to use for the client. Defaults to None, which will use the default handler. custom_list_page_parser (Optional[Callable[[str], Iterable[str]]]): Function to call to parse pages that list directories. Defaults to looking for `` tags with `href`. custom_dir_matcher (Optional[Callable[[str], bool]]): Function to call to identify a url that is a directory. Defaults to a lambda that checks if the path ends with a `/`. write_file_http_method (Optional[str]): HTTP method to use when writing files. Defaults to "PUT", but some servers may want "POST". """ - super().__init__(file_cache_mode, local_cache_dir, content_type_method) + super().__init__( + file_cache_mode, + local_cache_dir, + content_type_method, + streaming_max_concurrency=streaming_max_concurrency, + ) self.auth = auth if self.auth is None: @@ -105,8 +115,14 @@ def _exists(self, cloud_path: HttpPath) -> bool: raise def _move_file(self, src: HttpPath, dst: HttpPath, remove_src: bool = True) -> HttpPath: - # .fspath will download the file so the local version can be uploaded - self._upload_file(src.fspath, dst) + if self.file_cache_mode == FileCacheMode.streaming: + # streaming mode has no local cache to round-trip through (fspath is + # unavailable), so stream between the two paths directly + with src.open("rb") as src_file, dst.open("wb") as dst_file: + shutil.copyfileobj(src_file, dst_file) + else: + # .fspath will download the file so the local version can be uploaded + self._upload_file(src.fspath, dst) if remove_src: try: self._remove(src) @@ -221,7 +237,7 @@ def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes raise OSError(f"Unexpected status {status} for range request on {cloud_path}") except urllib.error.HTTPError as e: if e.code == 404: - raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"HTTP resource not found: {cloud_path}") elif e.code == 416: return b"" raise @@ -237,7 +253,7 @@ def _get_content_length(self, cloud_path: "HttpPath") -> int: raise ValueError(f"HTTP resource does not provide Content-Length: {cloud_path}") except urllib.error.HTTPError as e: if e.code == 404: - raise FileNotFoundError(f"HTTP resource not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"HTTP resource not found: {cloud_path}") raise def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) -> None: @@ -258,7 +274,7 @@ def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) ) except urllib.error.HTTPError as e: if e.code == 405: - raise NotImplementedError( + raise CloudPathNotImplementedError( f"HTTP server does not support {self.write_file_http_method} requests for {url}" ) raise OSError(f"HTTP upload failed: {e}") diff --git a/cloudpathlib/local/localclient.py b/cloudpathlib/local/localclient.py index edd21c35..fbb1155f 100644 --- a/cloudpathlib/local/localclient.py +++ b/cloudpathlib/local/localclient.py @@ -7,7 +7,6 @@ import sys from tempfile import TemporaryDirectory from time import sleep -from types import TracebackType from typing import ( Any, Callable, @@ -18,13 +17,13 @@ Optional, Sequence, Tuple, - Type, Union, ) from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from ..client import Client, _CloudWriteStream, _UploadPart +from ..client import Client, _UploadPart from ..enums import FileCacheMode +from ..exceptions import CloudPathFileNotFoundError from .localpath import LocalPath @@ -46,6 +45,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, **kwargs: Any, ) -> None: self._local_storage_dir = local_storage_dir @@ -55,6 +55,7 @@ def __init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) @classmethod @@ -233,7 +234,7 @@ def _range_download(self, cloud_path: LocalPath, start: int, end: int) -> bytes: """Download a byte range from local storage.""" local_path = self._cloud_path_to_local(cloud_path) if not local_path.exists(): - raise FileNotFoundError(f"File not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"File not found: {cloud_path}") with open(local_path, "rb") as f: f.seek(start) @@ -244,7 +245,7 @@ def _get_content_length(self, cloud_path: LocalPath) -> int: """Get the size of a local file.""" local_path = self._cloud_path_to_local(cloud_path) if not local_path.exists(): - raise FileNotFoundError(f"File not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"File not found: {cloud_path}") return local_path.stat().st_size def _initiate_multipart_upload(self, cloud_path: LocalPath) -> str: @@ -279,10 +280,6 @@ def _complete_multipart_upload( def _abort_multipart_upload(self, cloud_path: LocalPath, upload_id: str) -> None: self._local_upload_buffers.pop(upload_id, None) - def _open_write_stream(self, cloud_path: LocalPath) -> _CloudWriteStream: - local_path = self._cloud_path_to_local(cloud_path) - return _LocalWriteStream(local_path) - def _put_empty_object(self, cloud_path: LocalPath) -> None: """Create a zero-byte local file.""" local_path = self._cloud_path_to_local(cloud_path) @@ -290,42 +287,6 @@ def _put_empty_object(self, cloud_path: LocalPath) -> None: local_path.write_bytes(b"") -class _LocalWriteStream: - """Buffered local write stream.""" - - def __init__(self, local_path: Path) -> None: - self._local_path = local_path - self._buf: bytearray = bytearray() - self._closed: bool = False - - def write(self, data: bytes) -> int: - if self._closed: - raise ValueError("I/O operation on closed stream") - self._buf.extend(data) - return len(data) - - def close(self) -> None: - if not self._closed: - self._closed = True - self._local_path.parent.mkdir(parents=True, exist_ok=True) - self._local_path.write_bytes(bytes(self._buf)) - - def terminate(self) -> None: - self._closed = True - self._buf.clear() - - def __enter__(self) -> "_LocalWriteStream": - return self - - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: - self.close() - - _temp_dirs_to_clean: List[TemporaryDirectory] = [] diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 8b71e4d1..3785e04d 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -1,3 +1,4 @@ +from functools import lru_cache import mimetypes import os from pathlib import Path, PurePosixPath @@ -6,7 +7,7 @@ from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode -from ..exceptions import CloudPathException +from ..exceptions import CloudPathException, CloudPathFileNotFoundError from .s3path import S3Path try: @@ -19,6 +20,12 @@ implementation_registry["s3"].dependencies_loaded = False +@lru_cache(maxsize=None) +def _botocore_s3_operation_model(operation_name: str): + """Operation model from the installed botocore's bundled S3 service data (no network).""" + return botocore.session.get_session().get_service_model("s3").operation_model(operation_name) + + @register_client_class("s3") class S3Client(Client): """Client class for AWS S3 which handles authentication with AWS for [`S3Path`](../s3path/) @@ -40,6 +47,7 @@ def __init__( addressing_style: Optional[str] = None, boto3_transfer_config: Optional["TransferConfig"] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, extra_args: Optional[dict] = None, ): """Class constructor. Sets up a boto3 [`Session`]( @@ -77,6 +85,9 @@ def __init__( [s3 transfers](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig) content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). extra_args (Optional[dict]): A dictionary of extra args passed to download, upload, copy, and list functions as relevant. You can include any keys supported by upload, download, or copy operations, and we will pass on only the relevant args. To see the @@ -132,6 +143,7 @@ def __init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) def _get_boto3_config(self, signature_version: Optional[str] = None): @@ -416,7 +428,7 @@ def _range_download(self, cloud_path: S3Path, start: int, end: int) -> bytes: except ClientError as e: code = e.response["Error"]["Code"] if code in ("404", "NoSuchKey"): - raise FileNotFoundError(f"S3 object not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"S3 object not found: {cloud_path}") if code in ("InvalidRange", "416"): return b"" raise @@ -440,64 +452,19 @@ def _get_content_length(self, cloud_path: S3Path) -> int: except ClientError as e: code = e.response["Error"]["Code"] if code in ("404", "NoSuchKey"): - raise FileNotFoundError(f"S3 object not found: {cloud_path}") + raise CloudPathFileNotFoundError(f"S3 object not found: {cloud_path}") raise def _streaming_extra_args(self, operation_name: str) -> Dict[str, Any]: """Return upload extras accepted by a specific low-level S3 operation.""" try: operation = self.client.meta.service_model.operation_model(operation_name) - allowed = set(operation.input_shape.members) except AttributeError: - # The test client intentionally implements only a small boto3 surface. - fallback_allowed = { - "CreateMultipartUpload": { - "ACL", - "CacheControl", - "ChecksumAlgorithm", - "ContentDisposition", - "ContentEncoding", - "ContentLanguage", - "ContentType", - "ExpectedBucketOwner", - "Expires", - "Metadata", - "ObjectLockLegalHoldStatus", - "ObjectLockMode", - "ObjectLockRetainUntilDate", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5", - "SSEKMSEncryptionContext", - "SSEKMSKeyId", - "ServerSideEncryption", - "StorageClass", - "Tagging", - "WebsiteRedirectLocation", - }, - "UploadPart": { - "ChecksumAlgorithm", - "ExpectedBucketOwner", - "RequestPayer", - "SSECustomerAlgorithm", - "SSECustomerKey", - "SSECustomerKeyMD5", - }, - "CompleteMultipartUpload": { - "ChecksumCRC32", - "ChecksumCRC32C", - "ChecksumCRC64NVME", - "ChecksumSHA1", - "ChecksumSHA256", - "ChecksumType", - "ExpectedBucketOwner", - "MpuObjectSize", - "RequestPayer", - }, - "PutObject": set(self.boto3_ul_extra_args), - } - allowed = fallback_allowed[operation_name] + # client objects that do not expose botocore's meta (e.g. test doubles): + # consult the installed botocore service model directly so filtering + # behaves identically to a real boto3 client + operation = _botocore_s3_operation_model(operation_name) + allowed = set(operation.input_shape.members) return {key: value for key, value in self.boto3_ul_extra_args.items() if key in allowed} def _streaming_object_args(self, operation_name: str, cloud_path: S3Path) -> Dict[str, Any]: diff --git a/docs/docs/caching.ipynb b/docs/docs/caching.ipynb index c94099e8..fc96c66c 100644 --- a/docs/docs/caching.ipynb +++ b/docs/docs/caching.ipynb @@ -451,7 +451,7 @@ " - `\"tmp_dir\"` (_default_) - Cached files are saved using Python's [`TemporaryDirectory`](https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryDirectory). This provides three potential avenues for the cache to get cleared. First, cached files are removed by `cloudpathlib` when the `*Client` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called. Second, Python clears a temporary directory if all references to that directory leave scope. Finally since the folder is in an operating system temp directory, it will be cleared by the OS (which, depending on the OS, may not happen until system restart).\n", " - `\"cloudpath_object\"` - cached files are removed when the `CloudPath` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called.\n", " - `\"close_file\"` - since we only download a file to the cache on read/write, we can ensure the cache is empty by removing the cached file as soon as the read/write is finished. Reading/writing the same `CloudPath` multiple times will result in re-downloading the file from the cloud. Note: For this to work, `cloudpath` needs to be in control of the reading/writing of files. This means your code base should use the `CloudPath.write_*`, `CloudPath.read_*`, and `CloudPath.open` methods. Using `CloudPath.fspath` (or passing the `CloudPath` as a `PathLike` object to another library) will not clear the cache on file close since it was not opened by `cloudpathlib`.\n", - " - `\"streaming\"` - files are never written to disk. Data is streamed directly from/to cloud storage using range requests for reads and multipart/block uploads for writes. This mode uses only small in-memory buffers and is ideal for large files or memory-constrained environments. Note: `.fspath` and similar properties are not available in streaming mode. Currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n", + " - `\"streaming\"` - supported read/write modes stream directly from/to cloud storage using range requests for reads and multipart/block uploads for writes, without writing cache files to disk (append and update modes are the exception: they fall back to the local cache, which is cleaned up when the client is garbage collected). This mode uses only small in-memory buffers and is ideal for large files or memory-constrained environments. Note: `.fspath` and similar properties are not available in streaming mode. Currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n", "\n", "Note: Although we use it in the examples below, for `\"cloudpath_object\"` and `\"tmp_dir\"` you normally shouldn't need to explicitly call `del`. Letting Python garbage collection run on its own once all references to the object leave scope should be sufficient. See details [in the Python docs](https://docs.python.org/3/reference/datamodel.html?highlight=__del__#object.__del__)).\n" ] diff --git a/docs/docs/streaming_io.md b/docs/docs/streaming_io.md index e0fcddf6..d0e67730 100644 --- a/docs/docs/streaming_io.md +++ b/docs/docs/streaming_io.md @@ -132,7 +132,7 @@ CloudPath.open( - `encoding`: Text encoding (default: platform locale, text mode only) - `errors`: Error handling strategy (default: `"strict"`, text mode only) - `newline`: Newline handling (text mode only) -- `buffer_size`: Size of read/write buffer in bytes (default: 64 KiB) +- `buffer_size`: Size of read/write buffer in bytes (default: 5 MiB) **Returns:** @@ -140,6 +140,25 @@ CloudPath.open( - `CloudTextIO` for text modes (when streaming) - Standard file object (when not streaming) +### Compatibility with standard interfaces + +Streaming file objects subclass the standard `io` base classes, so code written +against Python's file protocol works without modification: + +- `CloudBufferedIO` is an `io.BufferedIOBase` (and `io.IOBase`); `CloudTextIO` + is an `io.TextIOWrapper`. `isinstance` checks against the `io` ABCs pass. +- `CloudPath.open()` keeps the same signature and mode grammar as + `pathlib.Path.open()`, so call sites that accept either a `Path` or a + `CloudPath` behave the same in both cases. +- Any library that accepts an open file object works with streaming streams: + `json`, `csv`, `pickle`, `zipfile`, `tarfile`, `pandas`, `PIL.Image.open`, + `pyarrow`, etc. Read streams are seekable, so formats that require random + access (zip archives, parquet) also work. +- The one intentional gap is `os.PathLike`: `os.fspath(path)` / + `path.fspath` raise `CloudPathNotImplementedError` in streaming mode because + there is no local file to point at. Pass the open file object instead of the + path to libraries that require a filesystem path. + ### `CloudBufferedIO` Binary file-like object implementing `io.BufferedIOBase`. @@ -292,6 +311,36 @@ with write_path.open("wt") as f: df.to_csv(f, index=False) ``` + +### Using with parquet + +Streaming read streams are seekable, which is exactly what columnar formats +need: `pyarrow` seeks to the parquet footer to read the file metadata, then +fetches only the byte ranges for the row groups and columns you ask for — the +rest of the object is never downloaded. + +```python +import pyarrow.parquet as pq +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.parquet", client=client) + +with path.open("rb") as f: + parquet_file = pq.ParquetFile(f) + + # metadata comes from the footer alone + print(parquet_file.metadata.num_rows, parquet_file.schema_arrow) + + # reads only the column chunks for "user_id" + table = parquet_file.read(columns=["user_id"]) +``` + +For column-slicing workloads, a smaller `buffer_size` (e.g. 64 KiB - 1 MiB) +reduces over-fetch around the footer and column chunk boundaries; for reading +most of the file, keep the default. + ### Using with PIL/Pillow ```python @@ -321,14 +370,14 @@ from cloudpathlib.enums import FileCacheMode client = S3Client(file_cache_mode=FileCacheMode.streaming) -# Use larger buffer for better throughput on fast connections +# Use a larger buffer for better throughput on fast connections path = S3Path("s3://bucket/large-file.bin", client=client) -with path.open("rb", buffer_size=1024*1024) as f: +with path.open("rb", buffer_size=16 * 1024 * 1024) as f: data = f.read() -# Use smaller buffer for memory-constrained environments +# Use a smaller buffer for memory-constrained environments path = S3Path("s3://bucket/file.txt", client=client) -with path.open("rt", buffer_size=8192) as f: +with path.open("rt", buffer_size=64 * 1024) as f: for line in f: process(line) ``` @@ -339,9 +388,23 @@ with path.open("rt", buffer_size=8192) as f: The `buffer_size` parameter controls how much data is fetched from/written to cloud storage in each request: -- **Larger buffers** (256 KiB - 1 MiB): Better throughput, fewer requests, more memory -- **Smaller buffers** (8 KiB - 64 KiB): Lower memory usage, more requests, lower throughput -- **Default** (64 KiB): Good balance for most use cases +- **Default** (5 MiB): Matches the multi-MiB block sizes used by comparable + tools (fsspec/s3fs/gcsfs) so per-request latency does not dominate sequential + throughput. Reads never fetch past EOF, so small objects only pay for their + actual size. +- **Smaller buffers** (64 KiB - 1 MiB): Less memory per open stream and less + over-fetch when reading small slices of large objects, at the cost of more + requests for sequential scans. +- **Larger buffers**: Fewer, bigger requests; memory per open stream grows to + match. + +Two request-count notes: + +- A full-object `read()` is always satisfied with a single ranged request + regardless of `buffer_size`. +- Each buffered refill costs one ranged request, so a sequential scan of a + 1 GiB object makes ~205 requests at the 5 MiB default versus ~16,000 at + 64 KiB. ### Read Patterns @@ -349,6 +412,47 @@ The `buffer_size` parameter controls how much data is fetched from/written to cl - **Random seeks**: Each seek may trigger a new range request - less efficient - **Small random reads**: Consider downloading the file to cache instead +### Concurrency + +Pass `streaming_max_concurrency=N` when constructing a client to let each open +streaming stream issue up to `N` requests in parallel (the default of `1` is +fully sequential): + +```python +client = S3Client( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=4, +) +``` + +- **Writes**: completed parts upload in a background thread pool while your + code keeps writing; the stream blocks only when `N` parts are already in + flight, so buffered memory is bounded by roughly `N x` part size. Works for + S3, Azure, and GCS (all use order-independent multipart/block uploads); + HTTP writes are a single request and are unaffected. +- **Reads**: the next `N` byte ranges are prefetched in the background while + you consume the current one, pipelining sequential scans. Seeking outside + the prefetched window discards it. +- Failures in background requests surface on the next `write()`/`close()` + (aborting the upload) or the next `read()`, exactly like sequential errors. + +Semantics that no configuration changes: + +- **Streams are not thread-safe.** Like ordinary Python file objects, a single + `CloudBufferedIO`/`CloudTextIO` instance must not be shared between threads + without external locking. Open one stream per thread instead. +- **Concurrent readers are safe.** Any number of streams (across threads or + processes) can read the same object simultaneously; each issues independent + range requests and holds independent positions. +- **Concurrent writers to the same object are last-committer-wins.** Each + writer's upload is isolated (S3 multipart upload IDs; per-session Azure block + IDs), so writers cannot corrupt each other's data — whichever stream closes + last determines the final object. +- **Conflict detection**: open a write stream with + `force_overwrite_to_cloud=False` and closing raises `OverwriteNewerCloudError` + instead of overwriting a version of the object that was uploaded while the + stream was open. + ### Write Contract Object storage is fundamentally a **sequential, write-once** medium. @@ -359,10 +463,27 @@ Streaming mode reflects that contract: | `wb`, `w`, `xb`, `x` | True streaming — data is forwarded to the provider as it arrives | | `ab`, `a`, `r+b`, `r+`, `w+b`, `w+` | **Falls back to cache** — the object is downloaded, mutated locally, then re-uploaded on close. Semantics are correct; performance matches the cached path. | -Attempting to seek backward on a write-only streaming stream raises +!!! warning "There is no true streaming append" + Object stores cannot append to (or modify a byte range of) an existing + object — every write creates a whole new object. Anything that opens an + existing file for appending or in-place update (`a`, `a+`, `r+`, `w+`) + therefore cannot stream: cloudpathlib downloads the entire object to the + local cache, applies the writes there, and re-uploads the entire object on + close. That is correct but costs a full download plus a full upload (and + temporary disk space) proportional to the object's size — for a + log-appending workload, prefer writing many small objects or a + provider-native mechanism instead. + +Attempting to seek on a write-only streaming stream raises `io.UnsupportedOperation` because the provider has already accepted the earlier bytes. +Streaming writes honor `force_overwrite_to_cloud` (and the +`CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD` environment variable): when it +resolves to `False`, closing the stream raises `OverwriteNewerCloudError` +instead of clobbering a version of the object that was uploaded while the +stream was open. + ### Multipart/Block Uploads For write operations, the streaming I/O system automatically handles: @@ -372,8 +493,10 @@ For write operations, the streaming I/O system automatically handles: The final part may be smaller than 5 MiB. - **Azure**: Block blob staging — blocks grow adaptively during very large uploads and are committed on close. -- **GCS**: Resumable upload (`blob.open("wb")`) — data streams - incrementally to GCS without in-memory buffering. +- **GCS**: XML API multipart upload — parts are buffered to the provider + minimum of **5 MiB** and assembled on close, mirroring the S3 mechanism + (set an `AbortIncompleteMultipartUpload` bucket lifecycle rule to expire + uploads orphaned by hardware failure). ## Provider-Specific Behavior @@ -414,7 +537,8 @@ with path.open("rt") as f: ### Google Cloud Storage - Uses GCS SDK `download_as_bytes()` with start/end for reads -- Uses a resumable `blob.open("wb")` stream for writes +- Uses the XML API multipart upload for writes (via the SDK's transfer-manager + machinery), which supports concurrent part uploads - Supports GCS-specific features through client configuration ```python @@ -511,7 +635,7 @@ f.close() # Easy to forget! ### Streaming Mode Limitations -When using `FileCacheMode.streaming`, certain CloudPath features are not available because streaming mode doesn't create cached files on disk: +When using `FileCacheMode.streaming`, certain CloudPath features are not available because streaming mode avoids the local file cache (the append/update modes of `open` are the exception — they fall back to the cache, which is cleaned up when the client is garbage collected): **Not Available:** - `.fspath` property - Raises `CloudPathNotImplementedError` @@ -601,7 +725,7 @@ client = S3Client(file_cache_mode=FileCacheMode.streaming) path = S3Path("s3://bucket/file.txt", client=client) # Larger buffer for faster networks -with path.open("rb", buffer_size=1024*1024) as f: +with path.open("rb", buffer_size=16 * 1024 * 1024) as f: data = f.read() ``` diff --git a/requirements-dev.txt b/requirements-dev.txt index dc9e3560..2f6ca636 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,6 +20,7 @@ nbautoexport pandas pillow psutil +pyarrow pydantic pytest<9.1 pytest-cases>=3.9.1 diff --git a/tests/http_fixtures.py b/tests/http_fixtures.py index dec3479a..dbf86f43 100644 --- a/tests/http_fixtures.py +++ b/tests/http_fixtures.py @@ -105,12 +105,14 @@ def _handle_range_request(self, range_header): else: end = file_size - 1 - # Validate range - if start < 0 or end >= file_size or start > end: + # Validate range (RFC 7233: a range is unsatisfiable only when start is past + # EOF; an end past EOF is clamped to the final byte, as real servers do) + if start < 0 or start >= file_size or start > end: self.send_error(416, "Requested Range Not Satisfiable") self.send_header("Content-Range", f"bytes */{file_size}") self.end_headers() return + end = min(end, file_size - 1) # Read the requested range with path.open("rb") as f: diff --git a/tests/mock_clients/mock_azureblob.py b/tests/mock_clients/mock_azureblob.py index 8370d05f..f110c168 100644 --- a/tests/mock_clients/mock_azureblob.py +++ b/tests/mock_clients/mock_azureblob.py @@ -8,7 +8,7 @@ from azure.storage.blob import BlobProperties from azure.storage.blob._list_blobs_helper import BlobPrefix from azure.storage.blob._shared.authentication import SharedKeyCredentialPolicy -from azure.core.exceptions import ResourceNotFoundError +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from .utils import delete_empty_parents_up_to_root @@ -135,6 +135,14 @@ def get_blob_properties(self): raise ResourceNotFoundError def download_blob(self, offset=None, length=None): + path = self.root / self.key + if not (path.exists() and path.is_file()): + raise ResourceNotFoundError + if offset is not None and offset >= path.stat().st_size: + # real Azure rejects ranges starting past EOF (an end past EOF is clamped) + error = HttpResponseError("The range specified is invalid for the current size") + error.status_code = 416 + raise error return MockStorageStreamDownloader(self.root, self.key, offset=offset, length=length) def set_blob_metadata(self, metadata): diff --git a/tests/mock_clients/mock_gs.py b/tests/mock_clients/mock_gs.py index 165034ba..a2d67dc9 100644 --- a/tests/mock_clients/mock_gs.py +++ b/tests/mock_clients/mock_gs.py @@ -4,7 +4,12 @@ import shutil from tempfile import TemporaryDirectory -from google.api_core.exceptions import NotFound +import urllib.parse +from uuid import uuid4 +from xml.etree import ElementTree + +from google.api_core.exceptions import NotFound, RequestRangeNotSatisfiable +import requests from .utils import delete_empty_parents_up_to_root @@ -21,6 +26,8 @@ def __init__(self, *args, **kwargs): shutil.copytree(TEST_ASSETS, self.tmp_path / test_dir) self.metadata_cache = {} + self._connection = _MockConnection() + self._http = MockMPUTransport(self) @classmethod def create_anonymous_client(cls): @@ -83,8 +90,14 @@ def download_as_bytes(self, start=None, end=None, timeout=None, retry=None): retry.mocked_retries = 1 from_path = self.bucket / self.name + if not (from_path.exists() and from_path.is_file()): + raise NotFound(f"blob not found: {self.name}") data = from_path.read_bytes() + # real GCS rejects ranges starting past EOF (an end past EOF is clamped) + if start is not None and start >= len(data): + raise RequestRangeNotSatisfiable("The requested range is not satisfiable") + # Handle byte range if specified if start is not None: if end is not None: @@ -110,7 +123,9 @@ def reload( timeout=None, retry=None, ): - pass + path = self.bucket / self.name + if not (path.exists() and path.is_file()): + raise NotFound(f"blob not found: {self.name}") def upload_from_filename(self, filename, content_type=None, timeout=None, retry=None): # if timeout is not None, assume that the test wants a timeout and throw it @@ -176,42 +191,6 @@ def public_url(self) -> str: def generate_signed_url(self, version: str, expiration: timedelta, method: str): return f"https://storage.googleapis.com{self.bucket}/{self.name}?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=TEST&X-Goog-Date=20240131T185515Z&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=TEST" - def open(self, mode="rb", **kwargs): - """Return a file-like writer/reader for resumable uploads (mock implementation).""" - if mode == "wb": - return _MockBlobWriter(self) - raise NotImplementedError(f"Mock blob.open() only supports 'wb', not {mode!r}") - - -class _MockBlobWriter: - """Simulates a GCS resumable upload stream (blob.open('wb')).""" - - def __init__(self, blob: "MockBlob") -> None: - self._blob = blob - self._buf: bytearray = bytearray() - self._closed: bool = False - - def write(self, data: bytes) -> int: - if self._closed: - raise ValueError("I/O operation on closed stream") - self._buf.extend(data) - return len(data) - - def close(self) -> None: - if not self._closed: - self._closed = True - self._blob.upload_from_string(bytes(self._buf)) - - def terminate(self) -> None: - self._closed = True - self._buf.clear() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - class MockBucket: def __init__(self, name, bucket_name, client=None): @@ -308,3 +287,86 @@ def download_chunks_concurrently( def mock_default_auth(): return "fake-credentials", "fake-default-project" + + +class _MockConnection: + """Just enough of google.cloud.storage._http.Connection for the XML MPU URL.""" + + API_BASE_URL = "https://storage.googleapis.com" + + +def _mpu_response(status, headers=None, body=b""): + response = requests.Response() + response.status_code = status + response.headers.update(headers or {}) + response._content = body if isinstance(body, bytes) else body.encode() + return response + + +class MockMPUTransport: + """Fake authorized session implementing the GCS XML multipart-upload API.""" + + _XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/" + + def __init__(self, client): + self.client = client + self.uploads = {} + + def request(self, method, url, data=None, headers=None, **kwargs): + parsed = urllib.parse.urlsplit(url) + query = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + # url path is /{bucket}/{blob}; the mock stores blobs under tmp_path directly + _, _, blob = parsed.path.lstrip("/").partition("/") + blob = urllib.parse.unquote(blob) + + if method == "POST" and "uploads" in query: + upload_id = uuid4().hex + self.uploads[upload_id] = { + "blob": blob, + "parts": {}, + "content_type": (headers or {}).get("content-type"), + } + body = ( + f'' + f"{upload_id}" + ) + return _mpu_response(200, body=body) + + upload_id = query.get("uploadId") + if upload_id not in self.uploads: + return _mpu_response(404, body="NoSuchUpload") + + if method == "PUT" and "partNumber" in query: + part_number = int(query["partNumber"]) + etag = f'"mock-etag-{part_number}"' + self.uploads[upload_id]["parts"][part_number] = (etag, bytes(data)) + return _mpu_response(200, headers={"etag": etag}) + + if method == "POST": + upload = self.uploads.pop(upload_id) + root = ElementTree.fromstring(data) + parts = [] + for part_element in root.findall("Part"): + part_number = int(part_element.find("PartNumber").text) + etag = part_element.find("ETag").text + stored_etag, part_data = upload["parts"][part_number] + assert etag == stored_etag, "ETag mismatch in CompleteMultipartUpload" + parts.append(part_data) + # real GCS enforces a 5 MiB minimum for all non-final parts at finalize + if any(len(part_data) < 5 * 1024 * 1024 for part_data in parts[:-1]): + return _mpu_response(400, body="EntityTooSmall") + content = b"".join(parts) + target = self.client.tmp_path / upload["blob"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + if upload["content_type"] is not None: + self.client.metadata_cache[self.client.tmp_path / upload["blob"]] = upload[ + "content_type" + ] + return _mpu_response(200, body="") + + if method == "DELETE": + self.uploads.pop(upload_id, None) + return _mpu_response(204) + + return _mpu_response(400, body="Unsupported mock MPU request") diff --git a/tests/mock_clients/mock_s3.py b/tests/mock_clients/mock_s3.py index da181703..c049fbee 100644 --- a/tests/mock_clients/mock_s3.py +++ b/tests/mock_clients/mock_s3.py @@ -278,6 +278,17 @@ def get_object(self, Bucket, Key, Range=None, **kwargs): match = re.match(r"bytes=(\d+)-(\d+)", Range) if match: start, end = int(match.group(1)), int(match.group(2)) + if start >= len(data): + # real S3 rejects ranges starting past EOF (an end past EOF is clamped) + raise ClientError( + { + "Error": { + "Code": "InvalidRange", + "Message": "The requested range is not satisfiable", + } + }, + {}, + ) data = data[start : end + 1] else: raise ClientError( diff --git a/tests/test_caching.py b/tests/test_caching.py index 4fce4f6f..108b58c6 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -540,3 +540,54 @@ def test_reuse_cache_after_manual_cache_clear(rig: CloudProviderTestRig): _ = f.read() assert cp._local.exists() + + +def test_write_mtime_tie_does_not_raise(rig: CloudProviderTestRig): + """A save that leaves the cache file's mtime exactly equal to the cloud version's + (coarse-resolution filesystems, no-op writes) must bump the mtime and upload rather + than raise OverwriteNewerCloudError.""" + client = rig.client_class(**rig.required_client_kwargs) + cp = rig.create_cloud_path("dir_0/file0_0.txt", client=client) + + cp.write_text("v1") + _sync_filesystem() + + # re-sync the cache from the cloud so the cache file's mtime equals the cloud mtime + cp.clear_cache() + cp.read_text() + cloud_mtime = cp.stat().st_mtime + + with cp.open("w") as f: + f.write("v2") + f.flush() + # simulate a write that leaves the mtime unchanged (e.g. same-second write on a + # coarse-resolution filesystem) + os.utime(cp._local, times=(cloud_mtime, cloud_mtime)) + + assert cp.read_text() == "v2" + + +def test_streaming_append_fallback_cache_cleaned_up(rig: CloudProviderTestRig): + """Append/update modes fall back to the cache in streaming mode; those cache files + must be cleaned up when the client is garbage collected, like other cache modes.""" + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, **rig.required_client_kwargs + ) + cp = rig.create_cloud_path("dir_0/file0_0.txt", client=client) + + with cp.open("a") as f: + f.write("appended") + + # the fallback created a real cache file + assert cp._local.exists() + + cache_path = cp._local + client_cache_dir = client._local_cache_dir + del f # the with-statement target outlives the block and holds the path + del cp + del client + # the patched close handle forms a reference cycle, so collection is not immediate + gc.collect() + + assert not cache_path.exists() + assert not client_cache_dir.exists() diff --git a/tests/test_client.py b/tests/test_client.py index 284bd558..8160b246 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -192,7 +192,6 @@ def test_custom_mys3client_default_client(custom_s3_path): lambda client, path: Client._upload_part(client, path, "upload", 1, b"data"), lambda client, path: Client._complete_multipart_upload(client, path, "upload", []), lambda client, path: Client._abort_multipart_upload(client, path, "upload"), - lambda client, path: Client._open_write_stream(client, path), lambda client, path: Client._put_empty_object(client, path), ], ) @@ -201,27 +200,3 @@ def test_default_streaming_hooks_raise_not_implemented(local_s3_rig, call): with pytest.raises(NotImplementedError, match="streaming I/O"): call(path.client, path) - - -def test_default_write_stream_hooks_delegate(local_s3_rig): - class Stream: - def __init__(self): - self.calls = [] - - def write(self, data): - self.calls.append(("write", data)) - return len(data) - - def close(self): - self.calls.append(("close",)) - - def terminate(self): - self.calls.append(("terminate",)) - - client = local_s3_rig.client_class(**local_s3_rig.required_client_kwargs) - stream = Stream() - - assert Client._write_stream(client, stream, b"data") == 4 - Client._close_write_stream(client, stream) - Client._abort_write_stream(client, stream) - assert stream.calls == [("write", b"data"), ("close",), ("terminate",)] diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py index 84b07283..6bea2f5b 100644 --- a/tests/test_cloud_io.py +++ b/tests/test_cloud_io.py @@ -7,12 +7,19 @@ import io import threading +import time +import zipfile import pytest from cloudpathlib import S3Path, AzureBlobPath, GSPath from cloudpathlib import CloudBufferedIO, CloudTextIO from cloudpathlib.cloud_io import _CloudStorageRaw from cloudpathlib.enums import FileCacheMode +from cloudpathlib.exceptions import ( + CloudPathFileNotFoundError, + CloudPathNotImplementedError, + OverwriteNewerCloudError, +) # Sample test data BINARY_DATA = b"Hello, World! This is binary data.\n" * 100 @@ -835,8 +842,8 @@ def test_azure_block_upload(rig): pass -def test_gs_resumable_upload(rig): - """Test that GCS upload works.""" +def test_gs_multipart_streaming_upload(rig): + """Test that GCS streaming upload works (XML multipart under the hood).""" if rig.path_class.cloud_prefix != "gs://": pytest.skip("Not testing GCS") @@ -1446,7 +1453,7 @@ def test_custom_client_without_raw_io_class_instantiates(local_s3_rig, monkeypat assert path.read_text() == "cached" path.client.file_cache_mode = FileCacheMode.streaming - with pytest.raises(NotImplementedError, match="Streaming I/O is not implemented"): + with pytest.raises(CloudPathNotImplementedError, match="Streaming I/O is not implemented"): path.open("r") @@ -1830,3 +1837,936 @@ def _abort_upload(self): with pytest.raises(OSError, match="original failure"): raw.close() + + +# ============================================================================ +# Regression tests — PR #535 review fixes +# ============================================================================ + +_STREAMING_PREFIXES = ("s3://", "az://", "gs://", "http://", "https://") + + +def _skip_if_no_streaming(rig): + if rig.path_class.cloud_prefix not in _STREAMING_PREFIXES: + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + +@pytest.fixture +def streaming_rig(rig): + """The rig with its default client switched to streaming mode for the test.""" + _skip_if_no_streaming(rig) + client = rig.client_class._default_client + original_mode = client.file_cache_mode + client.file_cache_mode = FileCacheMode.streaming + yield rig + client.file_cache_mode = original_mode + + +def test_write_tell_tracks_position(streaming_rig): + """tell() on streaming write streams must report total bytes written, not just + the bytes pending in the buffer (write() previously never advanced the raw position).""" + path = streaming_rig.create_cloud_path("test_write_tell.bin") + + try: + # small buffer so most bytes reach the raw layer instead of sitting in the buffer + with path.open("wb", buffer_size=64 * 1024) as f: + assert f.tell() == 0 + f.write(b"x" * 200_000) # larger than the buffer + assert f.tell() == 200_000 + f.write(b"y" * 100) # small write held in the buffer + assert f.tell() == 200_100 + assert path.read_bytes() == b"x" * 200_000 + b"y" * 100 + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_zipfile_write_roundtrip(streaming_rig): + """Position-dependent writers like zipfile rely on tell(); a streaming write + stream must produce a valid archive.""" + path = streaming_rig.create_cloud_path("test_streaming_archive.zip") + big_member = b"data" * 50_000 # > the 64 KiB buffer below so bytes reach the raw layer + + try: + with path.open("wb", buffer_size=64 * 1024) as f: + with zipfile.ZipFile(f, "w") as zf: + zf.writestr("a.txt", b"hello world") + zf.writestr("b.bin", big_member) + + with zipfile.ZipFile(io.BytesIO(path.read_bytes())) as zf: + assert zf.testzip() is None + assert zf.read("a.txt") == b"hello world" + assert zf.read("b.bin") == big_member + finally: + try: + path.unlink() + except Exception: + pass + + +def test_raw_write_stream_rejects_seek(streaming_rig): + """seek() on a write-only raw stream must raise io.UnsupportedOperation instead of + silently succeeding while writes keep appending.""" + path = streaming_rig.create_cloud_path("test_raw_seek_write.bin") + + try: + with path.open("wb", buffering=0) as f: + assert not f.seekable() + f.write(b"data") + with pytest.raises(io.UnsupportedOperation): + f.seek(0) + assert path.read_bytes() == b"data" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_full_read_uses_single_range_request(streaming_rig, monkeypatch): + """read() to EOF must fetch the remaining bytes in one ranged request (readall), + not fall back to one request per 8 KiB default chunk.""" + rig = streaming_rig + content = bytes(range(256)) * 2048 # 512 KiB + + # write in cached mode, then stream the read + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_readall.bin") + path.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + calls = [] + original_range_download = type(client)._range_download + + def counting_range_download(self, cloud_path, start, end): + calls.append((start, end)) + return original_range_download(self, cloud_path, start, end) + + monkeypatch.setattr(type(client), "_range_download", counting_range_download) + + try: + with path.open("rb") as f: + data = f.read() + assert data == content + assert len(calls) <= 2, f"expected a single ranged request, got {calls}" + finally: + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_streaming_size_fetch_failure_is_memoized(streaming_rig, monkeypatch): + """A failing content-length lookup must be attempted at most once per stream, + not re-issued before every chunk read.""" + rig = streaming_rig + content = b"z" * (200 * 1024) + + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_size_memo.bin") + path.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + calls = {"n": 0} + + def failing_get_content_length(self, cloud_path): + calls["n"] += 1 + raise OSError("no size available") + + monkeypatch.setattr(type(client), "_get_content_length", failing_get_content_length) + + try: + with path.open("rb", buffer_size=16 * 1024) as f: + data = f.read() + assert data == content + assert calls["n"] == 1 + finally: + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_gs_range_download_transient_error_not_treated_as_eof(gs_rig, monkeypatch): + """Errors that merely contain '416' in their message (request IDs, generation + numbers) must propagate; only true 416 range errors read as EOF.""" + path = gs_rig.create_cloud_path("test_416_matching.bin") + + class FakeServiceUnavailable(Exception): + code = 503 + + class FakeRangeError(Exception): + code = 416 + + def make_stub(error): + class StubBlob: + def download_as_bytes(self, start=None, end=None, **kwargs): + raise error + + class StubBucket: + def blob(self, name): + return StubBlob() + + return lambda name: StubBucket() + + # transient error whose message contains "416" must raise, not return EOF + monkeypatch.setattr( + path.client.client, + "bucket", + make_stub(FakeServiceUnavailable("503 GET /o/file?generation=1234164 backend error")), + ) + with pytest.raises(FakeServiceUnavailable): + path.client._range_download(path, 0, 9) + + # structured 416 still reads as EOF + monkeypatch.setattr( + path.client.client, "bucket", make_stub(FakeRangeError("range not satisfiable")) + ) + assert path.client._range_download(path, 0, 9) == b"" + + # exact reason phrase still reads as EOF (some layers do not expose a code) + monkeypatch.setattr( + path.client.client, "bucket", make_stub(Exception("Requested Range Not Satisfiable")) + ) + assert path.client._range_download(path, 0, 9) == b"" + + +def test_azure_block_ids_namespaced_per_upload(azure_rig): + """Concurrent streaming writers to the same blob must stage blocks under distinct + IDs so they cannot clobber each other's uncommitted blocks.""" + path = azure_rig.create_cloud_path("test_block_ids.bin") + + upload_a = path.client._initiate_multipart_upload(path) + upload_b = path.client._initiate_multipart_upload(path) + assert upload_a and upload_b and upload_a != upload_b + + try: + part_a = path.client._upload_part(path, upload_a, 1, b"A" * 16) + part_b = path.client._upload_part(path, upload_b, 1, b"B" * 16) + assert part_a["block_id"] != part_b["block_id"] + + # committing B yields exactly B's data even though A staged the same part number + path.client._complete_multipart_upload(path, upload_b, [part_b]) + assert path.read_bytes() == b"B" * 16 + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_cross_client_copy(streaming_rig): + """copy() between paths on different client instances must work in streaming mode + (the cached implementation round-trips through fspath, which streaming forbids).""" + rig = streaming_rig + content = b"copy me" * 100 + + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + src = rig.create_cloud_path("test_copy_src.bin") + src.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + other_client = rig.client_class(**rig.required_client_kwargs) + dst = other_client.CloudPath(str(rig.create_cloud_path("test_copy_dst.bin"))) + assert src.client is not dst.client + + try: + result = src.copy(dst) + assert result.read_bytes() == content + finally: + for p in (src, dst): + try: + p.unlink() + except Exception: + pass + + +def test_http_rename_in_streaming_mode(rig): + """rename()/replace() on HTTP paths must work in streaming mode without fspath.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("HTTP/HTTPS-specific test") + + path = rig.create_cloud_path("test_rename_src.bin") + path.write_bytes(b"move me") + target = rig.create_cloud_path("test_rename_dst.bin") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + result = path.rename(target) + assert result.read_bytes() == b"move me" + assert not path.exists() + finally: + path.client.file_cache_mode = original_mode + for p in (path, target): + try: + p.unlink() + except Exception: + pass + + +def test_streaming_write_conflict_raises(streaming_rig): + """A streaming write with force_overwrite_to_cloud=False must not clobber a + version uploaded while the stream was open.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_conflict.bin") + path.write_bytes(b"original") + + try: + f = path.open("wb", force_overwrite_to_cloud=False) + f.write(b"mine") + + # a concurrent writer replaces the object with a strictly newer version + time.sleep(1.1) # some providers report modification times in whole seconds + path.write_bytes(b"concurrent") + + with pytest.raises(OverwriteNewerCloudError): + f.close() + + assert path.read_bytes() == b"concurrent" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_write_conflict_force_overwrites(streaming_rig): + """force_overwrite_to_cloud=True skips the conflict check and wins.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_conflict_force.bin") + path.write_bytes(b"original") + + try: + f = path.open("wb", force_overwrite_to_cloud=True) + f.write(b"mine") + path.write_bytes(b"concurrent") + f.close() + + assert path.read_bytes() == b"mine" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_write_overwrite_unchanged_cloud_succeeds(streaming_rig): + """Overwriting an object that did not change while the stream was open is a + normal write and must not raise.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_overwrite_ok.bin") + path.write_bytes(b"v1") + + try: + with path.open("wb", force_overwrite_to_cloud=False) as f: + f.write(b"v2") + assert path.read_bytes() == b"v2" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_negative_buffering_accepted(rig): + """Any negative buffering value means 'use the default', matching builtins.open.""" + path = rig.create_cloud_path("test_neg_buffering.txt") + + try: + # cached mode passes buffering through to the local filesystem open + with path.open("w", buffering=-2) as f: + f.write("cached") + assert path.read_text() == "cached" + + if rig.path_class.cloud_prefix in _STREAMING_PREFIXES: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("rb", buffering=-2) as f: + assert f.read() == b"cached" + finally: + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_not_found_errors_are_cloudpathlib_exceptions(rig): + """Streaming hooks must raise cloudpathlib's exception types so callers catching + CloudPathException (or CloudPathFileNotFoundError) see streaming errors too.""" + _skip_if_no_streaming(rig) + missing = rig.create_cloud_path("definitely_missing_for_streaming.bin") + + with pytest.raises(CloudPathFileNotFoundError): + missing.client._range_download(missing, 0, 9) + + with pytest.raises(CloudPathFileNotFoundError): + missing.client._get_content_length(missing) + + +# ============================================================================ +# Coverage-gap tests — raw-stream edge semantics, provider EOF mapping, guards +# ============================================================================ + + +def test_raw_stream_edge_semantics(local_s3_rig): + """Raw adapter edge cases follow file-object semantics.""" + path = local_s3_rig.create_cloud_path("raw-edges.bin") + path.write_bytes(b"0123456789") + raw = path._cloud_meta.raw_io_class(path.client, path, "rb") + + # empty destination buffer reads zero bytes + assert raw.readinto(bytearray(0)) == 0 + + # relative seek and whence validation at the raw layer + raw.seek(4) + assert raw.seek(2, io.SEEK_CUR) == 6 + with pytest.raises(ValueError, match="invalid whence"): + raw.seek(0, 42) + + # readall from a mid-stream position, then again at EOF + assert raw.readall() == b"6789" + assert raw.readall() == b"" + raw.close() + + # write-only streams cannot readall; closed streams cannot readall + writer = path._cloud_meta.raw_io_class(path.client, path, "wb") + with pytest.raises(io.UnsupportedOperation): + writer.readall() + writer.write(b"replaced") + writer.close() + writer.close() # double close is a no-op + with pytest.raises(ValueError, match="closed file"): + writer.readall() + assert path.read_bytes() == b"replaced" + + +def test_multipart_part_limit_enforced(local_s3_rig): + """Exceeding the provider's maximum part count raises a clear OSError.""" + path = local_s3_rig.create_cloud_path("part-limit.bin") + + class TinyParts(path._cloud_meta.raw_io_class): + _INITIAL_PART_SIZE = 4 + _MAX_PART_SIZE = 4 + _MAX_PARTS = 2 + _PARTS_PER_SIZE_TIER = 1_000 + + raw = TinyParts(path.client, path, "wb") + raw.write(b"x" * 8) # exactly two full parts — at the limit + with pytest.raises(OSError, match="part limit"): + raw.write(b"x" * 4) + with pytest.raises(OSError, match="part limit"): + raw.close() # the write failure is sticky and aborts the upload + + +def test_buffered_io_direct_construction_guards(local_s3_rig): + """Direct construction validates modes that the open() path never forwards.""" + path = local_s3_rig.create_cloud_path("direct-construction.bin") + path.write_bytes(b"0123456789") + raw_cls = path._cloud_meta.raw_io_class + + with pytest.raises(io.UnsupportedOperation, match="append and update"): + CloudBufferedIO(raw_cls, path.client, path, mode="ab") + with pytest.raises(io.UnsupportedOperation, match="append and update"): + CloudTextIO(raw_cls, path.client, path, mode="a") + + # readinto1 delegates to the buffered reader + with CloudBufferedIO(raw_cls, path.client, path, mode="rb") as f: + buf = bytearray(4) + assert f.readinto1(buf) == 4 + assert bytes(buf) == b"0123" + + +def test_streaming_text_exclusive_create(streaming_rig): + """mode='x' in text form creates a new object via streaming and rejects existing ones.""" + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = streaming_rig.create_cloud_path("test_x_create.txt") + try: + with path.open(mode="x") as f: + f.write("created") + assert path.read_text() == "created" + with pytest.raises(CloudPathFileExistsError): + path.open(mode="x") + finally: + try: + path.unlink() + except Exception: + pass + + +def test_range_download_past_eof_returns_empty(streaming_rig): + """A range starting past EOF maps to EOF (empty bytes) on every provider.""" + path = streaming_rig.create_cloud_path("past_eof.bin") + path.write_bytes(b"0123456789") + try: + assert path.client._range_download(path, 100, 199) == b"" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_raw_empty_write_is_noop(streaming_rig): + """Writing b'' at the raw layer uploads nothing but still finalizes correctly.""" + path = streaming_rig.create_cloud_path("empty_chunk.bin") + try: + with path.open("wb", buffering=0) as f: + assert f.write(b"") == 0 + f.write(b"payload") + assert f.write(b"") == 0 + assert path.read_bytes() == b"payload" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_s3_streaming_extra_args_uses_service_model(s3_rig): + """When the boto3 client exposes a service model, allowed params come from it, + and unknown params are dropped.""" + from types import SimpleNamespace + + client = s3_rig.client_class(**s3_rig.required_client_kwargs) + client.boto3_ul_extra_args = {"StorageClass": "STANDARD_IA", "NotARealParam": "x"} + + shape = SimpleNamespace(members={"StorageClass": None}) + operation_model = SimpleNamespace(input_shape=shape) + service_model = SimpleNamespace(operation_model=lambda name: operation_model) + client.client = SimpleNamespace(meta=SimpleNamespace(service_model=service_model)) + + assert client._streaming_extra_args("CreateMultipartUpload") == {"StorageClass": "STANDARD_IA"} + + +def test_s3_streaming_extra_args_fallback_matches_botocore(s3_rig): + """The hard-coded fallback table must filter identically to botocore's real + service model, so it cannot silently drop newly added parameters.""" + from types import SimpleNamespace + + botocore_session = pytest.importorskip("botocore.session") + service_model = botocore_session.get_session().get_service_model("s3") + + for operation in ( + "CreateMultipartUpload", + "UploadPart", + "CompleteMultipartUpload", + "PutObject", + ): + members = set(service_model.operation_model(operation).input_shape.members) + extra_args = {name: "value" for name in sorted(members)} + + fallback_client = s3_rig.client_class(**s3_rig.required_client_kwargs) + fallback_client.boto3_ul_extra_args = extra_args + fallback_client.client = SimpleNamespace() # no .meta -> fallback table + + real_client = s3_rig.client_class(**s3_rig.required_client_kwargs) + real_client.boto3_ul_extra_args = extra_args + real_client.client = SimpleNamespace(meta=SimpleNamespace(service_model=service_model)) + + assert fallback_client._streaming_extra_args( + operation + ) == real_client._streaming_extra_args( + operation + ), f"fallback table diverges from botocore for {operation}" + + +def test_s3_streaming_content_encoding_threaded(s3_rig): + """Encodings from content_type_method are added to streaming upload args.""" + import mimetypes + + client = s3_rig.client_class( + content_type_method=mimetypes.guess_type, **s3_rig.required_client_kwargs + ) + path = s3_rig.create_cloud_path("encoded.txt.gz", client=client) + + args = client._streaming_object_args("CreateMultipartUpload", path) + assert args.get("ContentType") == "text/plain" + assert args.get("ContentEncoding") == "gzip" + + +def test_azure_streaming_content_settings_branches(azure_rig): + """Content settings resolve for absent, empty, and populated content type methods.""" + client_none = azure_rig.client_class( + content_type_method=None, **azure_rig.required_client_kwargs + ) + path = azure_rig.create_cloud_path("content-settings.bin", client=client_none) + assert client_none._streaming_content_settings(path) is None + + client_empty = azure_rig.client_class( + content_type_method=lambda name: (None, None), **azure_rig.required_client_kwargs + ) + assert client_empty._streaming_content_settings(path) is None + + client_full = azure_rig.client_class( + content_type_method=lambda name: ("text/plain", "gzip"), + **azure_rig.required_client_kwargs, + ) + settings = client_full._streaming_content_settings(path) + assert settings.content_type == "text/plain" + assert settings.content_encoding == "gzip" + + # abort is a documented no-op: uncommitted blocks simply expire server-side + client_none._abort_multipart_upload(path, "upload-id") + + +def test_gs_multipart_upload_hooks(gs_rig): + """GS streaming writes use the XML multipart API: unique upload IDs, ordered + assembly of size-compliant parts, and cancellable uploads.""" + client = gs_rig.client_class(**gs_rig.required_client_kwargs) + path = gs_rig.create_cloud_path("mpu-hooks.bin", client=client) + + upload_a = client._initiate_multipart_upload(path) + upload_b = client._initiate_multipart_upload(path) + assert upload_a and upload_b and upload_a != upload_b + + head = b"A" * (5 * 1024 * 1024) # non-final parts must be at least 5 MiB + tail = b"B" * 16 + part_1 = client._upload_part(path, upload_a, 1, head) + part_2 = client._upload_part(path, upload_a, 2, tail) + client._complete_multipart_upload(path, upload_a, [part_1, part_2]) + client._abort_multipart_upload(path, upload_b) + + try: + assert path.read_bytes() == head + tail + finally: + try: + path.unlink() + except Exception: + pass + + +def test_gs_mpu_initiate_threads_content_type_and_encoding(gs_rig): + """Initiate carries the content type and encoding from content_type_method.""" + from types import SimpleNamespace + + client = gs_rig.client_class( + content_type_method=lambda name: ("text/plain", "gzip"), + **gs_rig.required_client_kwargs, + ) + path = gs_rig.create_cloud_path("mpu-headers.txt.gz", client=client) + + captured = {} + + class StubTransport: + def request(self, method, url, data=None, headers=None, **kwargs): + import requests + + captured["method"] = method + captured["url"] = url + captured["headers"] = {key.lower(): value for key, value in (headers or {}).items()} + response = requests.Response() + response.status_code = 200 + response._content = ( + b'' + b"stub-upload" + ) + return response + + client.client = SimpleNamespace( + _connection=SimpleNamespace(API_BASE_URL="https://storage.googleapis.com"), + _http=StubTransport(), + ) + + upload_id = client._initiate_multipart_upload(path) + assert upload_id == "stub-upload" + assert captured["method"] == "POST" + assert captured["url"].endswith("?uploads") + assert captured["headers"]["content-type"] == "text/plain" + assert captured["headers"]["content-encoding"] == "gzip" + + +def test_http_streaming_error_paths(http_rig, monkeypatch): + """HTTP: unexpected range status, missing Content-Length, failing PUT status, + and unsupported PUT method all map to clear errors.""" + import urllib.error + + path = http_rig.create_cloud_path("http_errors.bin") + path.write_bytes(b"0123456789") + + class FakeResponse: + def __init__(self, status, headers=None): + self.status = status + self.headers = headers if headers is not None else {} + self.reason = "stub" + + def read(self, *args): + return b"" + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + # a non-206/200 success status for a range request is unexpected + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(204)) + with pytest.raises(OSError, match="Unexpected status"): + path.client._range_download(path, 0, 3) + + # HEAD without Content-Length cannot size the stream + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(200)) + with pytest.raises(ValueError, match="Content-Length"): + path.client._get_content_length(path) + + # a failing PUT status raises OSError + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(500)) + with pytest.raises(OSError, match="HTTP PUT failed"): + path.client._put_data(path, io.BytesIO(b"x"), 1) + + # non-404 HTTP errors propagate from reads and size checks + def raise_403(req): + raise urllib.error.HTTPError("url", 403, "Forbidden", {}, None) + + monkeypatch.setattr(path.client.opener, "open", raise_403) + with pytest.raises(urllib.error.HTTPError): + path.client._range_download(path, 0, 3) + with pytest.raises(urllib.error.HTTPError): + path.client._get_content_length(path) + + # servers that reject the write method surface CloudPathNotImplementedError + def raise_405(req): + raise urllib.error.HTTPError("url", 405, "Method Not Allowed", {}, None) + + monkeypatch.setattr(path.client.opener, "open", raise_405) + with pytest.raises(CloudPathNotImplementedError): + path.client._put_data(path, io.BytesIO(b"x"), 1) + + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_open_buffer_size_must_be_positive(streaming_rig): + """buffer_size=0 is rejected up front.""" + path = streaming_rig.create_cloud_path("bad_buffer.bin") + with pytest.raises(ValueError, match="buffer_size"): + path.open("wb", buffer_size=0) + + +def test_open_directory_raises(local_s3_rig): + """Opening a directory raises CloudPathIsADirectoryError in any cache mode.""" + from cloudpathlib.exceptions import CloudPathIsADirectoryError + + file_path = local_s3_rig.create_cloud_path("adir/inner.txt") + file_path.write_text("x") + dir_path = local_s3_rig.create_cloud_path("adir") + + with pytest.raises(CloudPathIsADirectoryError): + dir_path.open("rb") + + +def test_streaming_parquet_metadata_and_column_read(streaming_rig): + """Parquet readers work over a seekable streaming stream: the footer and a + single column can be read without downloading the whole object.""" + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + + rig = streaming_rig + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_columns.parquet") + table = pa.table({"a": list(range(10_000)), "b": ["x" * 20] * 10_000}) + sink = io.BytesIO() + pq.write_table(table, sink) + path.write_bytes(sink.getvalue()) + client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("rb", buffer_size=64 * 1024) as f: + parquet_file = pq.ParquetFile(f) + assert parquet_file.metadata.num_rows == 10_000 + column = parquet_file.read(columns=["a"]) + assert column.column("a").to_pylist()[:3] == [0, 1, 2] + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# streaming_max_concurrency — concurrent part uploads and read prefetch +# ============================================================================ + + +def test_streaming_max_concurrency_validation(local_s3_rig): + """The concurrency knob must be a positive integer.""" + with pytest.raises(ValueError, match="streaming_max_concurrency"): + local_s3_rig.client_class( + streaming_max_concurrency=0, **local_s3_rig.required_client_kwargs + ) + + +def test_concurrent_multipart_write_correctness(streaming_rig): + """A multi-part streaming write with concurrency > 1 produces identical content, + even when an early part finishes after later ones.""" + from cloudpathlib.cloud_io import _CloudMultipartStorageRaw + + rig = streaming_rig + if not issubclass(rig.raw_io_class, _CloudMultipartStorageRaw): + pytest.skip("provider does not use multipart streaming writes") + + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=4, + **rig.required_client_kwargs, + ) + path = rig.create_cloud_path("test_concurrent_parts.bin", client=client) + data = bytes(range(256)) * (48 * 1024) # 12 MiB -> two 5 MiB parts + final part + + part_numbers = [] + real_upload_part = client._upload_part + + def delaying_upload_part(cloud_path, upload_id, part_number, part_data): + if part_number == 1: + time.sleep(0.2) # force part 1 to finish after later parts + part_numbers.append(part_number) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + client._upload_part = delaying_upload_part + + try: + with path.open("wb") as f: + f.write(data) + assert sorted(part_numbers) == list(range(1, len(part_numbers) + 1)) + assert len(part_numbers) >= 2 + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_concurrent_multipart_write_parallelism_observed(local_s3_rig): + """With concurrency 2, two part uploads genuinely run at the same time.""" + path = local_s3_rig.create_cloud_path("test_parallel_parts.bin") + client = local_s3_rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=2, + **local_s3_rig.required_client_kwargs, + ) + path = local_s3_rig.create_cloud_path("test_parallel_parts.bin", client=client) + + barrier = threading.Barrier(2, timeout=30) + overlapped = [] + real_upload_part = client._upload_part + + def rendezvous_upload_part(cloud_path, upload_id, part_number, part_data): + if part_number <= 2: + barrier.wait() # only passes if both uploads are in flight simultaneously + overlapped.append(part_number) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + client._upload_part = rendezvous_upload_part + + data = b"Z" * (12 * 1024 * 1024) # two 5 MiB parts + final part + try: + with path.open("wb") as f: + f.write(data) + assert sorted(overlapped) == [1, 2] + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_concurrent_multipart_write_failure_is_sticky_and_aborts(local_s3_rig): + """A failing background part upload surfaces on a later write/close and aborts.""" + client = local_s3_rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=2, + **local_s3_rig.required_client_kwargs, + ) + path = local_s3_rig.create_cloud_path("test_failing_part.bin", client=client) + + aborted = [] + real_abort = client._abort_multipart_upload + + def spy_abort(cloud_path, upload_id): + aborted.append(upload_id) + return real_abort(cloud_path, upload_id) + + def failing_upload_part(cloud_path, upload_id, part_number, part_data): + raise OSError("part upload failed") + + client._upload_part = failing_upload_part + client._abort_multipart_upload = spy_abort + + f = path.open("wb") + with pytest.raises(OSError, match="part upload failed"): + # keep writing until the background failure is harvested + for _ in range(10): + f.write(b"Q" * (6 * 1024 * 1024)) + f.close() + with pytest.raises(OSError, match="part upload failed"): + f.close() + + assert aborted, "failed upload was not aborted" + assert not path.exists() + + +def test_read_prefetch_correctness_and_no_wasted_requests(streaming_rig): + """Sequential reads with prefetch fetch each byte range exactly once and + return identical data; seeking invalidates the prefetch window correctly.""" + rig = streaming_rig + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=3, + **rig.required_client_kwargs, + ) + path = rig.create_cloud_path("test_prefetch.bin", client=client) + chunk = 128 * 1024 + data = bytes(range(256)) * (4 * 1024) # 1 MiB -> 8 chunks + + path.write_bytes(data) + + calls = [] + real_range_download = client._range_download + + def counting_range_download(cloud_path, start, end): + calls.append((start, end)) + return real_range_download(cloud_path, start, end) + + client._range_download = counting_range_download + + try: + with path.open("rb", buffer_size=chunk) as f: + read_back = b"" + while True: + piece = f.read1(chunk) + if not piece: + break + read_back = read_back + piece + assert read_back == data + starts = sorted(start for start, _ in calls) + assert starts == list(range(0, len(data), chunk)), f"unexpected requests: {calls}" + + # seeking back re-reads correctly even though prefetched chunks are discarded + with path.open("rb", buffer_size=chunk) as f: + f.read1(chunk) + f.seek(3 * chunk) + assert f.read1(chunk) == data[3 * chunk : 4 * chunk] + f.seek(0) + assert f.read1(chunk) == data[:chunk] + finally: + try: + path.unlink() + except Exception: + pass