diff --git a/HISTORY.md b/HISTORY.md index 64e08d5b..7c9a77d1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -5,6 +5,22 @@ - Added `AGENTS.md` with repository-specific guidance for coding agents covering contributor workflow, compatibility expectations, test rig and mock usage, live backend validation, and PR hygiene. +- Added streaming I/O support for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS via `FileCacheMode.streaming`. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) + - Added `FileCacheMode.streaming` enum value to enable direct streaming I/O without local caching. + - Added `CloudBufferedIO` class implementing `io.BufferedIOBase` for binary streaming operations. + - Added `CloudTextIO` class implementing `io.TextIOBase` for text streaming operations. + - Added provider-specific raw I/O implementations: `_S3StorageRaw`, `_AzureBlobStorageRaw`, `_GSStorageRaw`, `_HttpStorageRaw`. + - Added `register_raw_io_class` decorator for registering streaming I/O implementations. + - Added `buffer_size` parameter to `CloudPath.open()` for controlling streaming buffer size; the default is 5 MiB, matching the block sizes of comparable tools (a full-object `read()` always uses a single ranged request regardless of buffer size). + - Streaming upload extra args for S3 are filtered against botocore's bundled service model (also when the client object does not expose `meta`), so newly added S3 parameters are never silently dropped. + - Added a `streaming_max_concurrency` client parameter (default 1): each open streaming stream may issue up to that many requests in parallel — background part uploads while writing (in-flight memory bounded to concurrency × part size) and read-ahead prefetch of upcoming byte ranges while reading sequentially. + - Google Cloud Storage streaming writes use the XML API multipart upload (via the SDK's transfer-manager machinery) instead of a resumable-upload stream, matching the S3/Azure part mechanism and enabling concurrent part uploads. + - Streaming writes honor `force_overwrite_to_cloud` (and `CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD`), raising `OverwriteNewerCloudError` on close instead of overwriting an object that changed while the stream was open. + - `copy`/`rename`/`replace` work in streaming mode by streaming between clients instead of round-tripping through the local cache (`fspath`). + - Cache files created by the append/update fallback in streaming mode are cleaned up when the client is garbage collected. + - Streaming error paths raise `cloudpathlib.exceptions` types (`CloudPathFileNotFoundError`, `CloudPathNotImplementedError`), which subclass the corresponding builtins. +- Changed `CloudPath.open(mode="a")` on a nonexistent cloud file to create it (matching the stdlib `open` and `pathlib`) instead of raising `CloudPathFileNotFoundError`. **Breaking change for users that relied on the previous error.** (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) +- Changed the cached-write upload tie-break so a save that leaves the cache file's modification time exactly equal to the cloud version's (e.g., same-second writes on coarse-resolution filesystems) uploads instead of raising a spurious `OverwriteNewerCloudError`. (PR [#535](https://github.com/drivendataorg/cloudpathlib/pull/535)) - Fixed mypy 2.x type errors in `Client` and `CloudPath` that caused CI lint failures (Issue [#563](https://github.com/drivendataorg/cloudpathlib/issues/563), PR [#566](https://github.com/drivendataorg/cloudpathlib/pull/566)) - Changed `S3Client._get_metadata` to read object metadata with `HeadObject` instead of `GetObject`, so `stat`, `etag`, and `size` no longer open the object body. Also fixes a `KeyError` on `ContentLength` against S3-compatible gateways that drop `Content-Length` from `GetObject` responses. (Issue [#564](https://github.com/drivendataorg/cloudpathlib/issues/564), PR [#565](https://github.com/drivendataorg/cloudpathlib/pull/565)) - Added a `lazy` keyword argument to `CloudPath.walk`. By default (`lazy=False`) the existing fast behavior is preserved: the whole subtree is fetched up front with a single recursive listing. Passing `lazy=True` lists each directory on demand so that, when `top_down=True`, callers can prune subdirectories by modifying `dirnames` in-place (à la `os.walk` / `Path.walk`) to skip fetching the contents of those subtrees entirely — dramatically reducing API calls for large, sparsely-traversed trees. (Issue [#518](https://github.com/drivendataorg/cloudpathlib/issues/518)) diff --git a/cloudpathlib/__init__.py b/cloudpathlib/__init__.py index 37a3b764..cc23e215 100644 --- a/cloudpathlib/__init__.py +++ b/cloudpathlib/__init__.py @@ -4,6 +4,7 @@ from .anypath import AnyPath from .azure.azblobclient import AzureBlobClient from .azure.azblobpath import AzureBlobPath +from .cloud_io import CloudBufferedIO, CloudTextIO from .cloudpath import CloudPath, implementation_registry from .patches import patch_open, patch_os_functions, patch_glob, patch_all_builtins from .gs.gsclient import GSClient @@ -26,7 +27,9 @@ "AnyPath", "AzureBlobClient", "AzureBlobPath", + "CloudBufferedIO", "CloudPath", + "CloudTextIO", "implementation_registry", "GSClient", "GSPath", diff --git a/cloudpathlib/azure/__init__.py b/cloudpathlib/azure/__init__.py index e29ada6a..1647fe96 100644 --- a/cloudpathlib/azure/__init__.py +++ b/cloudpathlib/azure/__init__.py @@ -1,5 +1,6 @@ from .azblobclient import AzureBlobClient from .azblobpath import AzureBlobPath +from .azure_io import _AzureBlobStorageRaw # noqa: F401 - imported for registration __all__ = [ "AzureBlobClient", diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 60bd01d3..a52b5359 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -3,18 +3,19 @@ import os from http import HTTPStatus from pathlib import Path -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union from itertools import islice +from uuid import uuid4 try: from typing import cast except ImportError: from typing_extensions import cast -from ..client import Client, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode -from ..exceptions import MissingCredentialsError +from ..exceptions import CloudPathFileNotFoundError, MissingCredentialsError from .azblobpath import AzureBlobPath try: @@ -61,6 +62,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, ): """Class constructor. Sets up a [`BlobServiceClient`]( https://docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobserviceclient?view=azure-python). @@ -108,11 +110,15 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). """ super().__init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) if connection_string is None: @@ -497,6 +503,92 @@ def _generate_presigned_url( url = f"{self._get_public_url(cloud_path)}?{sas_token}" return url + def _range_download(self, cloud_path: AzureBlobPath, start: int, end: int) -> bytes: + """Download a byte range from Azure Blob Storage.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + try: + length = end - start + 1 + downloader = blob_client.download_blob(offset=start, length=length) + return downloader.readall() + except ResourceNotFoundError: + raise CloudPathFileNotFoundError(f"Azure blob not found: {cloud_path}") + except HttpResponseError as e: + if (e.error and e.error.code == "InvalidRange") or e.status_code == 416: + return b"" + raise + + def _get_content_length(self, cloud_path: AzureBlobPath) -> int: + """Get the size of an Azure blob.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + try: + properties = blob_client.get_blob_properties() + return properties.size + except ResourceNotFoundError: + raise CloudPathFileNotFoundError(f"Azure blob not found: {cloud_path}") + + def _initiate_multipart_upload(self, cloud_path: AzureBlobPath) -> str: + """Return a unique session ID that namespaces this upload's block IDs. + + Azure's uncommitted-block namespace is per-blob, so deterministic block IDs + would let concurrent writers to the same blob overwrite each other's staged + blocks and commit interleaved data. + """ + return uuid4().hex + + def _upload_part( + self, cloud_path: AzureBlobPath, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Upload a block in an Azure block blob upload.""" + import base64 + + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + # Azure requires all block IDs for a blob to be the same length; uuid4().hex (32) + # plus a fixed-width part number keeps them uniform. + block_id = base64.b64encode(f"{upload_id}-{part_number:06d}".encode()).decode() + blob_client.stage_block(block_id=block_id, data=data, length=len(data)) + return {"block_id": block_id} + + def _complete_multipart_upload( + self, cloud_path: AzureBlobPath, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + """Commit an Azure block blob upload, threading content-type.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + block_ids = [part["block_id"] for part in parts] + blob_client.commit_block_list( + block_ids, content_settings=self._streaming_content_settings(cloud_path) + ) + + def _streaming_content_settings( + self, cloud_path: AzureBlobPath + ) -> Optional["ContentSettings"]: + if self.content_type_method is None: + return None + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if not content_type and not content_encoding: + return None + return ContentSettings(content_type=content_type, content_encoding=content_encoding) + + def _abort_multipart_upload(self, cloud_path: AzureBlobPath, upload_id: str) -> None: + """Let Azure expire uncommitted blocks.""" + pass + + def _put_empty_object(self, cloud_path: AzureBlobPath) -> None: + """Upload a zero-byte Azure blob, threading content-type.""" + blob_client = self.service_client.get_blob_client( + container=cloud_path.container, blob=cloud_path.blob + ) + blob_client.upload_blob( + b"", overwrite=True, content_settings=self._streaming_content_settings(cloud_path) + ) + def _hns_rmtree(data_lake_client, container, directory): """Stateless implementation so can be used in test suite cleanup as well. diff --git a/cloudpathlib/azure/azure_io.py b/cloudpathlib/azure/azure_io.py new file mode 100644 index 00000000..ee68fe3a --- /dev/null +++ b/cloudpathlib/azure/azure_io.py @@ -0,0 +1,23 @@ +"""Azure Blob Storage streaming I/O.""" + +from ..cloud_io import _CloudMultipartStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("azure") +class _AzureBlobStorageRaw(_CloudMultipartStorageRaw): + """Azure range reads and block writes.""" + + # Azure permits at most 50,000 committed blocks. + _INITIAL_PART_SIZE = 4 * 1024 * 1024 + _BLOCK_SIZE = _INITIAL_PART_SIZE + _MAX_PART_SIZE = 4_000 * 1024 * 1024 + _MAX_BLOCK_SIZE = _MAX_PART_SIZE + _MAX_PARTS = 50_000 + _BLOCKS_PER_SIZE_TIER = 1_000 + _PARTS_PER_SIZE_TIER = _BLOCKS_PER_SIZE_TIER + _PROVIDER_NAME = "Azure block" + + @classmethod + def _block_size_for_number(cls, block_number: int) -> int: + return cls._part_size_for_number(block_number) diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index d1c36fd5..85621400 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -4,13 +4,26 @@ from pathlib import Path import shutil from tempfile import TemporaryDirectory -from typing import ClassVar, Generic, Callable, Iterable, Optional, Tuple, TypeVar, Union +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Generic, + Iterable, + Optional, + Sequence, + Tuple, + TypeVar, + Union, +) from .cloudpath import CloudImplementation, CloudPath, implementation_registry from .enums import FileCacheMode from .exceptions import InvalidConfigurationException BoundedCloudPath = TypeVar("BoundedCloudPath", bound=CloudPath) +_UploadPart = Dict[str, Any] def register_client_class(key: str) -> Callable: @@ -34,11 +47,18 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, - ): + streaming_max_concurrency: int = 1, + ) -> None: self.file_cache_mode = None self._cache_tmp_dir = None self._cloud_meta.validate_completeness() + if streaming_max_concurrency < 1: + raise ValueError("streaming_max_concurrency must be at least 1") + # concurrent requests per open streaming stream (part uploads / read prefetch); + # 1 means fully sequential I/O + self.streaming_max_concurrency = streaming_max_concurrency + # convert strings passed to enum if isinstance(file_cache_mode, str): file_cache_mode = FileCacheMode(file_cache_mode) @@ -88,6 +108,9 @@ def __del__(self) -> None: FileCacheMode.tmp_dir, FileCacheMode.close_file, FileCacheMode.cloudpath_object, + # streaming avoids the cache except for append/update fallbacks, which + # should not outlive the client + FileCacheMode.streaming, ]: self.clear_cache() @@ -184,3 +207,50 @@ def _generate_presigned_url( self, cloud_path: BoundedCloudPath, expire_seconds: int = 60 * 60 ) -> str: pass + + def _range_download(self, cloud_path: BoundedCloudPath, start: int, end: int) -> bytes: + """Download an inclusive byte range.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_range_download). " + "Implement this method or use a non-streaming file_cache_mode." + ) + + def _get_content_length(self, cloud_path: BoundedCloudPath) -> int: + """Return object size without downloading it.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_get_content_length)." + ) + + def _initiate_multipart_upload(self, cloud_path: BoundedCloudPath) -> str: + """Start a multipart upload.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_initiate_multipart_upload)." + ) + + def _upload_part( + self, cloud_path: BoundedCloudPath, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Upload one part.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_upload_part)." + ) + + def _complete_multipart_upload( + self, cloud_path: BoundedCloudPath, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + """Complete a multipart upload.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_complete_multipart_upload)." + ) + + def _abort_multipart_upload(self, cloud_path: BoundedCloudPath, upload_id: str) -> None: + """Abort a multipart upload.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_abort_multipart_upload)." + ) + + def _put_empty_object(self, cloud_path: BoundedCloudPath) -> None: + """Create an empty object.""" + raise NotImplementedError( + f"{type(self).__name__} does not support streaming I/O (_put_empty_object)." + ) diff --git a/cloudpathlib/cloud_io.py b/cloudpathlib/cloud_io.py new file mode 100644 index 00000000..3351ced3 --- /dev/null +++ b/cloudpathlib/cloud_io.py @@ -0,0 +1,595 @@ +"""Buffered cloud I/O without a local cache.""" + +from __future__ import annotations + +import io +from abc import abstractmethod +from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from types import TracebackType +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, Union + +if TYPE_CHECKING: + from _typeshed import ReadableBuffer as _ReadableBuffer + from _typeshed import WriteableBuffer as _WriteableBuffer +else: + _ReadableBuffer = Union[bytes, bytearray, memoryview] + _WriteableBuffer = Union[bytearray, memoryview] + +from .client import Client +from .cloudpath import CloudPath + +# Bytes fetched/buffered per request for buffered streaming I/O. Sized to match the +# multi-MiB block sizes used by comparable tools (fsspec/s3fs/gcsfs) so per-request +# latency does not dominate sequential throughput; reads never fetch past EOF, so +# small objects only pay for their actual size. +DEFAULT_BUFFER_SIZE = 5 * 1024 * 1024 + + +def _validate_file_mode(mode: str) -> None: + """Validate an ``open`` mode using the same grammar as the stdlib.""" + if not isinstance(mode, str): + raise TypeError(f"mode must be a string, not {type(mode).__name__}") + if not mode or any(character not in "rwaxbt+" for character in mode): + raise ValueError(f"invalid mode: {mode!r}") + if sum(mode.count(character) for character in "rwax") != 1: + raise ValueError("must have exactly one of create/read/write/append mode") + if mode.count("+") > 1 or mode.count("b") > 1 or mode.count("t") > 1: + raise ValueError(f"invalid mode: {mode!r}") + if "b" in mode and "t" in mode: + raise ValueError("can't have text and binary mode at once") + + +class _CloudStorageRaw(io.RawIOBase): + """Raw adapter backed by client streaming hooks.""" + + def __init__( + self, + client: Client, + cloud_path: CloudPath, + mode: str = "rb", + ) -> None: + super().__init__() + self._client = client + self._cloud_path = cloud_path + self._mode = mode + self._pos = 0 + self._size: Optional[int] = None + self._size_fetch_failed = False + self._closed = False + self._upload_error: Optional[BaseException] = None + # Optional conflict check run just before a write is finalized (set by CloudPath.open) + self._pre_finalize: Optional[Callable[[], None]] = None + # concurrent requests for this stream (read prefetch / background part uploads) + self._max_concurrency = max(1, int(getattr(client, "streaming_max_concurrency", 1))) + self._executor: Optional[ThreadPoolExecutor] = None + self._prefetch: Dict[int, Future] = {} + + def readable(self) -> bool: + """Return whether object was opened for reading.""" + return "r" in self._mode or "+" in self._mode + + def writable(self) -> bool: + """Return whether object was opened for writing.""" + return "w" in self._mode or "a" in self._mode or "+" in self._mode or "x" in self._mode + + def seekable(self) -> bool: + """Return whether object supports random access. + + Streaming writes are sequential-only; seeking is only valid for readable streams. + """ + return self.readable() + + def readinto(self, b: _WriteableBuffer, /) -> int: + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.readable(): + raise io.UnsupportedOperation("not readable") + view = memoryview(b).cast("B") + if len(view) == 0: + return 0 + + start = self._pos + end = start + len(view) - 1 + + size = self._known_size() + if size is not None and end >= size: + end = size - 1 + if start >= size: + return 0 + + try: + data = self._fetch_range(start, end) + except Exception as e: + if self._is_eof_error(e): + return 0 + raise + + n = len(data) + if n == 0: + return 0 + + n = min(n, len(view)) + view[:n] = data[:n] + + self._pos += n + return n + + def readall(self) -> bytes: + """Read from the current position to EOF in a single ranged request when possible.""" + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.readable(): + raise io.UnsupportedOperation("not readable") + + self._discard_prefetch() + size = self._known_size() + if size is None: + # Size unknown: fall back to the default chunked read loop. + return super().readall() + if self._pos >= size: + return b"" + + data = self._range_get(self._pos, size - 1) + self._pos += len(data) + return data + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + """ + Change stream position. + + Args: + offset: Offset in bytes + whence: Position to seek from (SEEK_SET, SEEK_CUR, SEEK_END) + + Returns: + New absolute position + """ + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.seekable(): + raise io.UnsupportedOperation("seek") + + if whence == io.SEEK_SET: + new_pos = offset + elif whence == io.SEEK_CUR: + new_pos = self._pos + offset + elif whence == io.SEEK_END: + size = self._known_size() + if size is None: + raise OSError("Unable to determine file size for SEEK_END") + new_pos = size + offset + else: + raise ValueError( + f"invalid whence ({whence}, should be {io.SEEK_SET}, " + f"{io.SEEK_CUR}, or {io.SEEK_END})" + ) + + if new_pos < 0: + raise ValueError("negative seek position") + + if new_pos != self._pos: + self._discard_prefetch() + self._pos = new_pos + return self._pos + + def tell(self) -> int: + """Return current stream position.""" + if self._closed: + raise ValueError("I/O operation on closed file") + return self._pos + + def write(self, b: _ReadableBuffer, /) -> int: + if self._closed: + raise ValueError("I/O operation on closed file") + if not self.writable(): + raise io.UnsupportedOperation("not writable") + if self._upload_error is not None: + raise self._upload_error + + data = bytes(b) + try: + self._upload_chunk(data) + except BaseException as error: + self._upload_error = error + raise + self._pos += len(data) + return len(data) + + def close(self) -> None: + """Close the file.""" + if self._closed: + return + + self._closed = True + + try: + if self.writable() and self._upload_error is not None: + try: + self._abort_upload() + except Exception: + pass + finally: + raise self._upload_error + if self.writable(): + try: + if self._pre_finalize is not None: + self._pre_finalize() + self._finalize_upload() + except BaseException: + try: + self._abort_upload() + except Exception: + pass + raise + finally: + self._shutdown_executor() + super().close() + + def _abort_upload(self) -> None: + """Best-effort cleanup after a write or finalization failure.""" + pass + + @abstractmethod + def _upload_chunk(self, data: bytes) -> None: + pass + + @abstractmethod + def _finalize_upload(self) -> None: + pass + + def _ensure_executor(self) -> Optional[ThreadPoolExecutor]: + """Thread pool for this stream's background requests; None when sequential.""" + if self._max_concurrency <= 1: + return None + if self._executor is None: + self._executor = ThreadPoolExecutor( + max_workers=self._max_concurrency, thread_name_prefix="cloudpathlib-stream" + ) + return self._executor + + def _shutdown_executor(self) -> None: + self._discard_prefetch() + if self._executor is not None: + self._executor.shutdown(wait=True, cancel_futures=True) + self._executor = None + + def _discard_prefetch(self) -> None: + for future in self._prefetch.values(): + future.cancel() + self._prefetch.clear() + + def _fetch_range(self, start: int, end: int) -> bytes: + """Fetch [start, end], serving from and topping up background prefetch when enabled.""" + executor = self._ensure_executor() + if executor is None: + return self._range_get(start, end) + + chunk_len = end - start + 1 + future = self._prefetch.pop(start, None) + if future is not None: + # a short prefetched chunk is a legal short read for RawIOBase consumers + data = future.result() + self._schedule_prefetch(start + max(len(data), 1), chunk_len) + return data + + # position changed or first read: pending prefetches no longer line up + self._discard_prefetch() + data = self._range_get(start, end) + self._schedule_prefetch(end + 1, chunk_len) + return data + + def _schedule_prefetch(self, next_start: int, chunk_len: int) -> None: + """Queue reads ahead of the current position, up to the concurrency window.""" + size = self._known_size() + executor = self._ensure_executor() + if size is None or chunk_len <= 0 or executor is None: + return + start = next_start + while len(self._prefetch) < self._max_concurrency and start < size: + if start not in self._prefetch: + self._prefetch[start] = executor.submit( + self._range_get, start, min(start + chunk_len, size) - 1 + ) + start += chunk_len + + def _range_get(self, start: int, end: int) -> bytes: + return self._client._range_download(self._cloud_path, start, end) + + def _get_size(self) -> int: + return self._client._get_content_length(self._cloud_path) + + def _known_size(self) -> Optional[int]: + """Fetch and memoize the object size, attempting the lookup at most once.""" + if self._size is None and not self._size_fetch_failed: + try: + self._size = self._get_size() + except Exception: + self._size_fetch_failed = True + return self._size + + def _is_eof_error(self, error: Exception) -> bool: + """ + Check if an error indicates EOF/out of range. + + Override in subclasses for provider-specific error handling. + """ + return False + + +class _CloudMultipartStorageRaw(_CloudStorageRaw): + """Shared buffered multipart upload lifecycle.""" + + _INITIAL_PART_SIZE: int + _MAX_PART_SIZE: int + _MAX_PARTS: int + _PARTS_PER_SIZE_TIER: int + _PROVIDER_NAME: str + + def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: + super().__init__(client, cloud_path, mode) + self._upload_id: Optional[str] = None + self._parts: Dict[int, dict[str, Any]] = {} + self._part_futures: Dict[int, Future] = {} + self._part_number = 1 + self._write_buffer = bytearray() + + @classmethod + def _part_size_for_number(cls, part_number: int) -> int: + tier = (part_number - 1) // cls._PARTS_PER_SIZE_TIER + return min(cls._INITIAL_PART_SIZE * (2**tier), cls._MAX_PART_SIZE) + + def _target_part_size(self) -> int: + return self._part_size_for_number(self._part_number) + + def _check_part_limit(self) -> None: + if self._part_number > self._MAX_PARTS: + raise OSError( + f"{self._PROVIDER_NAME} upload exceeded the {self._MAX_PARTS:,}-part limit" + ) + + def _upload_buffered_part(self, size: int) -> None: + self._check_part_limit() + if self._upload_id is None: + self._upload_id = self._client._initiate_multipart_upload(self._cloud_path) + data = bytes(self._write_buffer[:size]) + del self._write_buffer[:size] + part_number = self._part_number + self._part_number += 1 + + executor = self._ensure_executor() + if executor is None: + self._parts[part_number] = self._client._upload_part( + self._cloud_path, self._upload_id, part_number, data + ) + return + + # bound in-flight parts (and their buffered bytes) to the concurrency window + self._harvest_part_futures(block=len(self._part_futures) >= self._max_concurrency) + self._part_futures[part_number] = executor.submit( + self._client._upload_part, self._cloud_path, self._upload_id, part_number, data + ) + + def _harvest_part_futures(self, block: bool = False, drain: bool = False) -> None: + """Collect finished background part uploads, re-raising the first failure.""" + if not self._part_futures: + return + if drain: + wait(list(self._part_futures.values())) + elif block: + wait(list(self._part_futures.values()), return_when=FIRST_COMPLETED) + + error: Optional[BaseException] = None + for part_number in [n for n, f in self._part_futures.items() if f.done()]: + future = self._part_futures.pop(part_number) + try: + self._parts[part_number] = future.result() + except BaseException as e: + if error is None: + error = e + if error is not None: + raise error + + def _upload_chunk(self, data: bytes) -> None: + if not data: + return + self._write_buffer.extend(data) + target_size = self._target_part_size() + while len(self._write_buffer) >= target_size: + self._upload_buffered_part(target_size) + target_size = self._target_part_size() + + def _finalize_upload(self) -> None: + if self._write_buffer: + self._upload_buffered_part(len(self._write_buffer)) + self._harvest_part_futures(drain=True) + if self._upload_id is None: + self._client._put_empty_object(self._cloud_path) + return + ordered_parts = [self._parts[number] for number in sorted(self._parts)] + self._client._complete_multipart_upload(self._cloud_path, self._upload_id, ordered_parts) + self._reset_upload() + + def _abort_upload(self) -> None: + try: + for future in self._part_futures.values(): + future.cancel() + wait(list(self._part_futures.values())) + if self._upload_id is not None: + self._client._abort_multipart_upload(self._cloud_path, self._upload_id) + finally: + self._reset_upload() + + def _reset_upload(self) -> None: + self._upload_id = None + self._parts.clear() + self._part_futures.clear() + self._part_number = 1 + self._write_buffer.clear() + + +class CloudBufferedIO(io.BufferedIOBase): + """Buffered binary I/O backed by a cloud client.""" + + def __init__( + self, + raw_io_class: Type[_CloudStorageRaw], + client: Client, + cloud_path: CloudPath, + mode: str = "rb", + buffer_size: int = DEFAULT_BUFFER_SIZE, + pre_finalize: Optional[Callable[[], None]] = None, + ) -> None: + _validate_file_mode(mode) + if "b" not in mode: + raise ValueError("CloudBufferedIO requires binary mode (must include 'b')") + if "a" in mode or "+" in mode: + raise io.UnsupportedOperation( + "append and update modes require the local-cache implementation" + ) + + raw = raw_io_class(client, cloud_path, mode) + if pre_finalize is not None: + raw._pre_finalize = pre_finalize + + if "r" in mode: + self._buffer: Union[io.BufferedReader, io.BufferedWriter] + self._buffer = io.BufferedReader(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] + else: + self._buffer = io.BufferedWriter(raw, buffer_size=buffer_size) # type: ignore[arg-type,assignment] + + self._cloud_path = cloud_path + self._mode = mode + self._buffer_size_val = buffer_size + + @property + def name(self) -> str: + """File name (the cloud URL).""" + return str(self._cloud_path) + + @property + def mode(self) -> str: + """File mode.""" + return self._mode + + @property + def _buffer_size(self) -> int: + """Buffer size for compatibility with tests.""" + return self._buffer_size_val + + def read(self, size: Optional[int] = -1, /) -> bytes: + return self._buffer.read(size) + + def read1(self, size: int = -1, /) -> bytes: + return self._buffer.read1(size) # type: ignore[attr-defined] + + def readinto(self, b: _WriteableBuffer, /) -> int: + return self._buffer.readinto(b) + + def readinto1(self, b: _WriteableBuffer, /) -> int: + return self._buffer.readinto1(b) # type: ignore[attr-defined] + + def write(self, b: _ReadableBuffer, /) -> int: + return self._buffer.write(b) + + def seek(self, offset: int, whence: int = io.SEEK_SET, /) -> int: + return self._buffer.seek(offset, whence) + + def tell(self) -> int: + return self._buffer.tell() + + def flush(self) -> None: + self._buffer.flush() + + def close(self) -> None: + if hasattr(self, "_buffer") and not self._buffer.closed: + self._buffer.close() + + def readable(self) -> bool: + return self._buffer.readable() + + def writable(self) -> bool: + return self._buffer.writable() + + def seekable(self) -> bool: + return self._buffer.seekable() + + @property + def closed(self) -> bool: + return self._buffer.closed + + def __enter__(self) -> CloudBufferedIO: + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + /, + ) -> None: + self.close() + + +class CloudTextIO(io.TextIOWrapper): + """Text I/O backed by a cloud client.""" + + def __init__( + self, + raw_io_class: Type[_CloudStorageRaw], + client: Client, + cloud_path: CloudPath, + mode: str = "rt", + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + buffer_size: int = DEFAULT_BUFFER_SIZE, + line_buffering: bool = False, + pre_finalize: Optional[Callable[[], None]] = None, + ) -> None: + _validate_file_mode(mode) + if "b" in mode: + raise ValueError("CloudTextIO requires text mode (no 'b' in mode)") + if "a" in mode or "+" in mode: + raise io.UnsupportedOperation( + "append and update modes require the local-cache implementation" + ) + + # only r/w/x can reach here: 'b' was rejected above and 'a'/'+' raised earlier + if "t" not in mode and "r" in mode: + binary_mode = mode.replace("r", "rb", 1) + elif "t" not in mode and "w" in mode: + binary_mode = mode.replace("w", "wb", 1) + elif "t" not in mode and "x" in mode: + binary_mode = mode.replace("x", "xb", 1) + else: + binary_mode = mode.replace("t", "b") + + buffered = CloudBufferedIO( + raw_io_class, + client, + cloud_path, + mode=binary_mode, + buffer_size=buffer_size, + pre_finalize=pre_finalize, + ) + + super().__init__( + buffered, + encoding=encoding, + errors=errors, + newline=newline, + line_buffering=line_buffering, + ) + + self._cloud_path = cloud_path + self._mode = mode + + @property + def name(self) -> str: + """File name (the cloud URL).""" + return str(self._cloud_path) + + @property + def mode(self) -> str: + """File mode.""" + return self._mode diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 01306fa7..3188d4dc 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -108,12 +108,14 @@ def _make_selector(pattern_parts, _flavour, case_sensitive=True): # noqa: F811 class CloudImplementation: - name: str + name: Optional[str] = None dependencies_loaded: bool = True _client_class: Type["Client"] _path_class: Type["CloudPath"] + _raw_io_class: Optional[Type] = None def validate_completeness(self) -> None: + # raw_io_class is optional; streaming raises NotImplementedError when absent expected = ["client_class", "path_class"] missing = [cls for cls in expected if getattr(self, f"_{cls}") is None] if missing: @@ -121,9 +123,11 @@ def validate_completeness(self) -> None: f"Implementation is missing registered components: {missing}" ) if not self.dependencies_loaded: + # Use name if available, otherwise fall back to client class name + pkg_name = self.name if self.name else self._client_class.__name__.lower() raise MissingDependenciesError( f"Missing dependencies for {self._client_class.__name__}. You can install them " - f"with 'pip install cloudpathlib[{self.name}]'." + f"with 'pip install cloudpathlib[{pkg_name}]'." ) @property @@ -136,6 +140,11 @@ def path_class(self) -> Type["CloudPath"]: self.validate_completeness() return self._path_class + @property + def raw_io_class(self) -> Optional[Type]: + self.validate_completeness() + return self._raw_io_class + implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation) @@ -155,6 +164,23 @@ def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: return decorator +def register_raw_io_class(key: str) -> Callable[[Type[T]], Type[T]]: + """Decorator to register a raw I/O class for a cloud provider. + + Args: + key: The cloud provider key (e.g., 's3', 'azure', 'gs') + + Returns: + Decorator function + """ + + def decorator(cls: Type[T]) -> Type[T]: + implementation_registry[key]._raw_io_class = cls + return cls + + return decorator + + class CloudPathMeta(abc.ABCMeta): @overload def __call__( @@ -355,6 +381,14 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, type(self)) and str(self) == str(other) def __fspath__(self) -> str: + # Check if streaming mode is enabled + if self.client.file_cache_mode == FileCacheMode.streaming: + raise CloudPathNotImplementedError( + "fspath is not available in streaming mode, which avoids the local file cache " + "(except for append/update modes of `open`, which fall back to it). " + "Use CloudPath.open() to read/write data directly." + ) + if self.is_file(): self._refresh_cache() return str(self._local) @@ -680,6 +714,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "TextIOWrapper": ... @overload @@ -692,6 +727,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "FileIO": ... @overload @@ -704,6 +740,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedRandom": ... @overload @@ -716,6 +753,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedWriter": ... @overload @@ -728,6 +766,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BufferedReader": ... @overload @@ -740,6 +779,7 @@ def open( newline: None = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "BinaryIO": ... @overload @@ -752,6 +792,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, force_overwrite_to_cloud: Optional[bool] = None, + buffer_size: Optional[int] = None, ) -> "IO[Any]": ... def open( @@ -763,7 +804,23 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: Optional[bool] = None, # extra kwarg not in pathlib force_overwrite_to_cloud: Optional[bool] = None, # extra kwarg not in pathlib + buffer_size: Optional[int] = None, # extra kwarg for streaming mode ) -> "IO[Any]": + from .cloud_io import _validate_file_mode + + _validate_file_mode(mode) + binary_mode = "b" in mode + if binary_mode and encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + if binary_mode and errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + if binary_mode and newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + if not binary_mode and buffering == 0: + raise ValueError("can't have unbuffered text I/O") + if buffer_size is not None and buffer_size <= 0: + raise ValueError("buffer_size must be greater than zero") + # if trying to call open on a directory that exists exists_on_cloud = self.exists() @@ -772,15 +829,74 @@ def open( f"Cannot open directory, only files. Tried to open ({self})" ) - if not exists_on_cloud and any(m in mode for m in ("r", "a")): + if not exists_on_cloud and "r" in mode: raise CloudPathFileNotFoundError( - f"File opened for read or append, but it does not exist on cloud: {self}" + f"File opened for read, but it does not exist on cloud: {self}" ) - if mode == "x" and self.exists(): + if "x" in mode and exists_on_cloud: raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.") - # TODO: consider streaming from client rather than DLing entire file to cache + # Use streaming I/O if file_cache_mode is streaming AND the mode is supported. + # Append (a) and update/random (+) modes cannot be done correctly as pure streaming + # over object storage; fall through to the cached path for correct semantics. + _streaming_unsupported = any(m in mode for m in ("a", "+")) + if self.client.file_cache_mode == FileCacheMode.streaming and not _streaming_unsupported: + # Import here to keep it localized to streaming functionality + from .cloud_io import DEFAULT_BUFFER_SIZE, CloudBufferedIO, CloudTextIO + + # Get the raw IO class from the cloud implementation + raw_io_class = self._cloud_meta.raw_io_class + if raw_io_class is None: + raise CloudPathNotImplementedError( + f"Streaming I/O is not implemented for {self._cloud_meta.name}" + ) + + # Overwrite protection mirroring the cached path's upload conflict check + pre_finalize = None + if "w" in mode or "x" in mode: + pre_finalize = self._streaming_overwrite_check( + exists_on_cloud, force_overwrite_to_cloud + ) + + # Calculate buffer size from buffering or buffer_size parameter + if buffer_size is None: + if buffering == 0: + # A raw provider adapter is the streaming equivalent of FileIO. + raw = raw_io_class(self.client, self, mode) + if pre_finalize is not None: + raw._pre_finalize = pre_finalize + return raw # type: ignore[return-value] + elif buffering > 0: + buffer_size = buffering + else: + buffer_size = DEFAULT_BUFFER_SIZE + + # Return appropriate streaming I/O object + if "b" in mode: + return CloudBufferedIO( # type: ignore[return-value] + raw_io_class=raw_io_class, + client=self.client, + cloud_path=self, + mode=mode, + buffer_size=buffer_size, + pre_finalize=pre_finalize, + ) + else: + return CloudTextIO( # type: ignore[return-value] + raw_io_class=raw_io_class, + client=self.client, + cloud_path=self, + mode=mode, + encoding=encoding, + errors=errors, + newline=newline, + buffer_size=buffer_size, + line_buffering=buffering == 1, + pre_finalize=pre_finalize, + ) + + # Standard cached mode self._refresh_cache(force_overwrite_from_cloud=force_overwrite_from_cloud) # create any directories that may be needed if the file is new @@ -813,10 +929,8 @@ def _patched_close_upload(*args, **kwargs) -> None: if not self._dirty: return - # original mtime should match what was in the cloud; because of system clocks or rounding - # by the cloud provider, the new version in our cache is "older" than the original version; - # explicitly set the new modified time to be after the original modified time. - if self._local.stat().st_mtime < original_mtime: + # Keep cached writes newer despite timestamp rounding. + if self._local.stat().st_mtime <= original_mtime: new_mtime = original_mtime + 1 os.utime(self._local, times=(new_mtime, new_mtime)) @@ -848,6 +962,38 @@ def _patched_close_empty_cache(*args, **kwargs): return buffer + def _streaming_overwrite_check( + self, exists_on_cloud: bool, force_overwrite_to_cloud: Optional[bool] + ) -> Optional[Callable[[], None]]: + """Build the pre-upload conflict check for a streaming write, mirroring the cached + path's `OverwriteNewerCloudError` protection in `_upload_file_to_cloud`. Returns None + when overwriting is forced.""" + if force_overwrite_to_cloud is None: + force_overwrite_to_cloud = os.environ.get( + "CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD", "False" + ).lower() in ["1", "true"] + + if force_overwrite_to_cloud: + return None + + original_mtime = self.stat().st_mtime if exists_on_cloud else None + + def check() -> None: + try: + stats = self.stat() + except (NoStatError, CloudPathFileNotFoundError, FileNotFoundError): + # nothing on the cloud to conflict with + return + if original_mtime is None or stats.st_mtime > original_mtime: + raise OverwriteNewerCloudError( + f"Cloud path ({self}) changed while it was open for streaming write, " + f"but is being requested to be overwritten on close. Either (1) pass " + f"`force_overwrite_to_cloud=True` to overwrite; or (2) set env var " + f"CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD=1." + ) + + return check + def replace(self, target: Self) -> Self: if type(self) is not type(target): raise TypeError( @@ -1253,19 +1399,43 @@ def _copy( else: if not destination.exists() or destination.is_file(): - return cast( - Union[Path, Self], - destination.upload_from( - self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud - ), - ) + target_path: CloudPath = destination else: - return cast( - Union[Path, Self], - (destination / self.name).upload_from( - self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud - ), - ) + target_path = destination / self.name + + # streaming mode has no local cache to round-trip through (fspath is + # unavailable), so copy by streaming between the two clients directly + if self.client.file_cache_mode == FileCacheMode.streaming: + if force_overwrite_to_cloud is None: + force_overwrite_to_cloud = os.environ.get( + "CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD", "False" + ).lower() in ["1", "true"] + + if ( + not force_overwrite_to_cloud + and target_path.exists() + and target_path.stat().st_mtime >= self.stat().st_mtime + ): + raise OverwriteNewerCloudError( + f"File ({target_path}) is newer than ({self}). " + f"To overwrite " + f"pass `force_overwrite_to_cloud=True`." + ) + + with ( + self.open("rb") as src_file, + target_path.open("wb", force_overwrite_to_cloud=True) as dst_file, + ): + shutil.copyfileobj(src_file, dst_file) + + return cast(Union[Path, Self], target_path) + + return cast( + Union[Path, Self], + target_path.upload_from( + self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud + ), + ) @overload def copy( diff --git a/cloudpathlib/enums.py b/cloudpathlib/enums.py index 55260c17..b34fd36a 100644 --- a/cloudpathlib/enums.py +++ b/cloudpathlib/enums.py @@ -15,6 +15,8 @@ class FileCacheMode(str, Enum): called by Python garbage collection. close_file (str): Cache for a `CloudPath` file is removed as soon as the file is closed. Note: you must use `CloudPath.open` whenever opening the file for this method to function. + streaming (str): No caching is used; files are streamed directly from/to cloud storage using + efficient range requests and multipart uploads. Modes can be set by passing them to the Client or by setting the `CLOUDPATHLIB_FILE_CACHE_MODE` environment variable. @@ -26,6 +28,7 @@ class FileCacheMode(str, Enum): tmp_dir = "tmp_dir" # DEFAULT: handled by deleting client, Python, or OS (usually on machine restart) cloudpath_object = "cloudpath_object" # __del__ called on the CloudPath object close_file = "close_file" # cache is cleared when file is closed + streaming = "streaming" # no caching, direct streaming I/O @classmethod def from_environment(cls) -> Optional["FileCacheMode"]: diff --git a/cloudpathlib/gs/__init__.py b/cloudpathlib/gs/__init__.py index d68a41c4..d4c5e24a 100644 --- a/cloudpathlib/gs/__init__.py +++ b/cloudpathlib/gs/__init__.py @@ -1,5 +1,6 @@ from .gsclient import GSClient from .gspath import GSPath +from .gs_io import _GSStorageRaw # noqa: F401 - imported for registration __all__ = [ "GSClient", diff --git a/cloudpathlib/gs/gs_io.py b/cloudpathlib/gs/gs_io.py new file mode 100644 index 00000000..6b9bf298 --- /dev/null +++ b/cloudpathlib/gs/gs_io.py @@ -0,0 +1,16 @@ +"""Google Cloud Storage streaming I/O.""" + +from ..cloud_io import _CloudMultipartStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("gs") +class _GSStorageRaw(_CloudMultipartStorageRaw): + """GCS range reads and XML multipart-upload writes.""" + + # GCS XML multipart uploads require non-final parts of at least 5 MiB. + _INITIAL_PART_SIZE = 5 * 1024 * 1024 + _MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + _MAX_PARTS = 10_000 + _PARTS_PER_SIZE_TIER = 1_000 + _PROVIDER_NAME = "GCS multipart" diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index 96705ece..99795a93 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -1,13 +1,15 @@ from datetime import datetime, timedelta +from functools import lru_cache import mimetypes import os from pathlib import Path, PurePosixPath -from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union, Sequence import warnings -from ..client import Client, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode +from ..exceptions import CloudPathFileNotFoundError, CloudPathNotImplementedError from .gspath import GSPath try: @@ -15,18 +17,60 @@ from google.auth.credentials import Credentials from google.api_core.retry import Retry + from google.api_core.exceptions import NotFound as GCSNotFound from google.auth import default as google_default_auth from google.auth.exceptions import DefaultCredentialsError - from google.cloud.storage import Client as StorageClient + from google.cloud.storage.client import Client as StorageClient except ModuleNotFoundError: implementation_registry["gs"].dependencies_loaded = False + GCSNotFound = Exception # type: ignore[misc, assignment] # fallback so name is always defined try: - from google.cloud.storage import transfer_manager + import google.cloud.storage.transfer_manager as transfer_manager except ImportError: - transfer_manager = None + transfer_manager = None # type: ignore[assignment] + +try: + from google.cloud.storage._media.requests import XMLMPUContainer, XMLMPUPart +except ImportError: + try: + # older google-cloud-storage versions expose these via google-resumable-media + from google.resumable_media.requests import ( # type: ignore[assignment,no-redef] + XMLMPUContainer, + XMLMPUPart, + ) + except ImportError: + XMLMPUContainer = None # type: ignore[assignment,misc] + XMLMPUPart = None # type: ignore[assignment,misc] + + +@lru_cache(maxsize=1) +def _bytes_mpu_part_class() -> type: + """XMLMPUPart subclass that uploads an in-memory payload instead of a file slice.""" + + class _BytesXMLMPUPart(XMLMPUPart): + def __init__(self, upload_url, upload_id, data, part_number, headers=None): + super().__init__( + upload_url, + upload_id, + filename="", + start=0, + end=len(data), + part_number=part_number, + headers=headers, + checksum=None, + ) + self._data = bytes(data) + + def _prepare_upload_request(self): + if self.finished: + raise ValueError("This part has already been uploaded.") + query = f"?partNumber={self._part_number}&uploadId={self._upload_id}" + return "PUT", self.upload_url + query, self._data, self._headers + + return _BytesXMLMPUPart @register_client_class("gs") @@ -46,6 +90,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, download_chunks_concurrently_kwargs: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, retry: Optional["Retry"] = None, @@ -82,6 +127,9 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). download_chunks_concurrently_kwargs (Optional[Dict[str, Any]]): Keyword arguments to pass to [`download_chunks_concurrently`](https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.transfer_manager#google_cloud_storage_transfer_manager_download_chunks_concurrently) for sliced parallel downloads; Only available in `google-cloud-storage` version 2.7.0 or later, otherwise ignored and a warning is emitted. @@ -122,6 +170,7 @@ def __init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) def _get_metadata(self, cloud_path: GSPath) -> Optional[Dict[str, Any]]: @@ -310,5 +359,92 @@ def _generate_presigned_url(self, cloud_path: GSPath, expire_seconds: int = 60 * ) return url + def _range_download(self, cloud_path: GSPath, start: int, end: int) -> bytes: + """Download a byte range from GCS.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + try: + return blob.download_as_bytes(start=start, end=end, **self.blob_kwargs) + except GCSNotFound: + raise CloudPathFileNotFoundError(f"GCS object not found: {cloud_path}") + except Exception as e: + # match a range-past-EOF error structurally (status 416) rather than by + # substring, so unrelated errors are not silently treated as EOF + status = getattr(e, "code", None) + if status is None: + status = getattr(getattr(e, "response", None), "status_code", None) + if status == 416 or "Requested Range Not Satisfiable" in str(e): + return b"" + raise + + def _get_content_length(self, cloud_path: GSPath) -> int: + """Get the size of a GCS object.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + try: + blob.reload(**self.blob_kwargs) + return blob.size + except GCSNotFound: + raise CloudPathFileNotFoundError(f"GCS object not found: {cloud_path}") + + def _mpu_url(self, cloud_path: GSPath) -> str: + """XML API URL for a multipart upload of this object.""" + from urllib.parse import quote + + connection = self.client._connection + hostname = ( + connection.get_api_base_url_for_mtls() + if hasattr(connection, "get_api_base_url_for_mtls") + else connection.API_BASE_URL + ) + return f"{hostname}/{cloud_path.bucket}/{quote(cloud_path.blob)}" + + def _initiate_multipart_upload(self, cloud_path: GSPath) -> str: + """Start a GCS XML multipart upload, threading content type and encoding.""" + if XMLMPUContainer is None: + raise CloudPathNotImplementedError( + "Streaming writes require google-cloud-storage with XML multipart support." + ) + content_type = None + headers = {} + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_encoding is not None: + headers["Content-Encoding"] = content_encoding + container = XMLMPUContainer(self._mpu_url(cloud_path), cloud_path.blob, headers=headers) + container.initiate( + transport=self.client._http, content_type=content_type or "application/octet-stream" + ) + return container.upload_id + + def _upload_part( + self, cloud_path: GSPath, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Upload one part of a GCS XML multipart upload.""" + part = _bytes_mpu_part_class()(self._mpu_url(cloud_path), upload_id, data, part_number) + part.upload(self.client._http) + return {"part_number": part_number, "etag": part.etag} + + def _complete_multipart_upload( + self, cloud_path: GSPath, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + """Finalize a GCS XML multipart upload.""" + container = XMLMPUContainer( + self._mpu_url(cloud_path), cloud_path.blob, upload_id=upload_id + ) + for part in parts: + container.register_part(part["part_number"], part["etag"]) + container.finalize(self.client._http) + + def _abort_multipart_upload(self, cloud_path: GSPath, upload_id: str) -> None: + """Cancel a GCS XML multipart upload, discarding uploaded parts.""" + container = XMLMPUContainer( + self._mpu_url(cloud_path), cloud_path.blob, upload_id=upload_id + ) + container.cancel(self.client._http) + + def _put_empty_object(self, cloud_path: GSPath) -> None: + """Upload a zero-byte GCS object.""" + blob = self.client.bucket(cloud_path.bucket).blob(cloud_path.blob) + blob.upload_from_string(b"", **self.blob_kwargs) + GSClient.GSPath = GSClient.CloudPath # type: ignore diff --git a/cloudpathlib/http/__init__.py b/cloudpathlib/http/__init__.py index ccf7452e..00085d9c 100644 --- a/cloudpathlib/http/__init__.py +++ b/cloudpathlib/http/__init__.py @@ -1,5 +1,6 @@ from .httpclient import HttpClient, HttpsClient from .httppath import HttpPath, HttpsPath +from .http_io import _HttpStorageRaw # noqa: F401 __all__ = [ "HttpClient", diff --git a/cloudpathlib/http/http_io.py b/cloudpathlib/http/http_io.py new file mode 100644 index 00000000..77ff0761 --- /dev/null +++ b/cloudpathlib/http/http_io.py @@ -0,0 +1,46 @@ +"""HTTP streaming I/O.""" + +from __future__ import annotations + +import tempfile +from typing import Protocol, cast + +from ..client import Client +from ..cloud_io import _CloudStorageRaw +from ..cloudpath import CloudPath, register_raw_io_class + + +class _HttpStreamingClient(Protocol): + def _put_data( + self, + cloud_path: CloudPath, + data: tempfile.SpooledTemporaryFile[bytes], + content_length: int, + ) -> None: ... + + +@register_raw_io_class("http") +@register_raw_io_class("https") +class _HttpStorageRaw(_CloudStorageRaw): + """HTTP range reads and single-request writes.""" + + def __init__(self, client: Client, cloud_path: CloudPath, mode: str = "rb") -> None: + super().__init__(client, cloud_path, mode) + self._upload_buffer = tempfile.SpooledTemporaryFile(max_size=8 * 1024 * 1024) + + def _upload_chunk(self, data: bytes) -> None: + if data: + self._upload_buffer.write(data) + + def _finalize_upload(self) -> None: + self._upload_buffer.seek(0, 2) + content_length = self._upload_buffer.tell() + self._upload_buffer.seek(0) + try: + client = cast(_HttpStreamingClient, self._client) + client._put_data(self._cloud_path, self._upload_buffer, content_length) + finally: + self._upload_buffer.close() + + def _abort_upload(self) -> None: + self._upload_buffer.close() diff --git a/cloudpathlib/http/httpclient.py b/cloudpathlib/http/httpclient.py index a67690ea..790a5043 100644 --- a/cloudpathlib/http/httpclient.py +++ b/cloudpathlib/http/httpclient.py @@ -6,13 +6,14 @@ import urllib.parse import urllib.error from pathlib import Path -from typing import Iterable, Optional, Tuple, Union, Callable +from typing import BinaryIO, Iterable, Optional, Tuple, Union, Callable import shutil import mimetypes import warnings from cloudpathlib.client import Client, register_client_class from cloudpathlib.enums import FileCacheMode +from cloudpathlib.exceptions import CloudPathFileNotFoundError, CloudPathNotImplementedError from .httppath import HttpPath @@ -24,6 +25,7 @@ def __init__( file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, auth: Optional[urllib.request.BaseHandler] = None, custom_list_page_parser: Optional[Callable[[str], Iterable[str]]] = None, custom_dir_matcher: Optional[Callable[[str], bool]] = None, @@ -41,12 +43,20 @@ def __init__( the `CLOUDPATHLIB_LOCAL_CACHE_DIR` environment variable. content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when uploading files. Defaults to `mimetypes.guess_type`. + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). auth (Optional[urllib.request.BaseHandler]): Authentication handler to use for the client. Defaults to None, which will use the default handler. custom_list_page_parser (Optional[Callable[[str], Iterable[str]]]): Function to call to parse pages that list directories. Defaults to looking for `` tags with `href`. custom_dir_matcher (Optional[Callable[[str], bool]]): Function to call to identify a url that is a directory. Defaults to a lambda that checks if the path ends with a `/`. write_file_http_method (Optional[str]): HTTP method to use when writing files. Defaults to "PUT", but some servers may want "POST". """ - super().__init__(file_cache_mode, local_cache_dir, content_type_method) + super().__init__( + file_cache_mode, + local_cache_dir, + content_type_method, + streaming_max_concurrency=streaming_max_concurrency, + ) self.auth = auth if self.auth is None: @@ -105,8 +115,14 @@ def _exists(self, cloud_path: HttpPath) -> bool: raise def _move_file(self, src: HttpPath, dst: HttpPath, remove_src: bool = True) -> HttpPath: - # .fspath will download the file so the local version can be uploaded - self._upload_file(src.fspath, dst) + if self.file_cache_mode == FileCacheMode.streaming: + # streaming mode has no local cache to round-trip through (fspath is + # unavailable), so stream between the two paths directly + with src.open("rb") as src_file, dst.open("wb") as dst_file: + shutil.copyfileobj(src_file, dst_file) + else: + # .fspath will download the file so the local version can be uploaded + self._upload_file(src.fspath, dst) if remove_src: try: self._remove(src) @@ -203,6 +219,66 @@ def request( # the connection is closed when we exit the context manager. return response, response.read() + def _range_download(self, cloud_path: "HttpPath", start: int, end: int) -> bytes: + """Download an HTTP byte range.""" + headers = {"Range": f"bytes={start}-{end}"} + request = urllib.request.Request(str(cloud_path), headers=headers) + try: + with self.opener.open(request) as response: + status = response.status + if status == 206: + return response.read(end - start + 1) + elif status == 200: + raise OSError( + f"HTTP server ignored the Range header for {cloud_path}; " + "streaming reads require byte-range support" + ) + else: + raise OSError(f"Unexpected status {status} for range request on {cloud_path}") + except urllib.error.HTTPError as e: + if e.code == 404: + raise CloudPathFileNotFoundError(f"HTTP resource not found: {cloud_path}") + elif e.code == 416: + return b"" + raise + + def _get_content_length(self, cloud_path: "HttpPath") -> int: + """Get the size of an HTTP resource.""" + request = urllib.request.Request(str(cloud_path), method="HEAD") + try: + with self.opener.open(request) as response: + content_length = response.headers.get("Content-Length") + if content_length: + return int(content_length) + raise ValueError(f"HTTP resource does not provide Content-Length: {cloud_path}") + except urllib.error.HTTPError as e: + if e.code == 404: + raise CloudPathFileNotFoundError(f"HTTP resource not found: {cloud_path}") + raise + + def _put_data(self, cloud_path: "HttpPath", data: BinaryIO, content_length: int) -> None: + """Upload a file-like HTTP body.""" + url = str(cloud_path) + request = urllib.request.Request(url, data=data, method=self.write_file_http_method) + content_type = None + if self.content_type_method is not None: + content_type, _ = self.content_type_method(str(cloud_path)) + request.add_header("Content-Type", content_type or "application/octet-stream") + request.add_header("Content-Length", str(content_length)) + + try: + with self.opener.open(request) as response: + if response.status not in (200, 201, 204): + raise OSError( + f"HTTP PUT failed with status {response.status}: {response.reason}" + ) + except urllib.error.HTTPError as e: + if e.code == 405: + raise CloudPathNotImplementedError( + f"HTTP server does not support {self.write_file_http_method} requests for {url}" + ) + raise OSError(f"HTTP upload failed: {e}") + HttpClient.HttpPath = HttpClient.CloudPath # type: ignore diff --git a/cloudpathlib/local/implementations/azure.py b/cloudpathlib/local/implementations/azure.py index 2b44814f..c273341b 100644 --- a/cloudpathlib/local/implementations/azure.py +++ b/cloudpathlib/local/implementations/azure.py @@ -6,6 +6,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...azure.azure_io import _AzureBlobStorageRaw # noqa: F401 + local_azure_blob_implementation = CloudImplementation() """Replacement for "azure" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -81,3 +84,4 @@ def md5(self) -> str: local_azure_blob_implementation.name = "azure" local_azure_blob_implementation._client_class = LocalAzureBlobClient local_azure_blob_implementation._path_class = LocalAzureBlobPath +local_azure_blob_implementation._raw_io_class = _AzureBlobStorageRaw diff --git a/cloudpathlib/local/implementations/gs.py b/cloudpathlib/local/implementations/gs.py index 8633ff4e..4ca78873 100644 --- a/cloudpathlib/local/implementations/gs.py +++ b/cloudpathlib/local/implementations/gs.py @@ -4,6 +4,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...gs.gs_io import _GSStorageRaw # noqa: F401 + local_gs_implementation = CloudImplementation() """Replacement for "gs" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -64,3 +67,4 @@ def md5(self) -> str: local_gs_implementation.name = "gs" local_gs_implementation._client_class = LocalGSClient local_gs_implementation._path_class = LocalGSPath +local_gs_implementation._raw_io_class = _GSStorageRaw diff --git a/cloudpathlib/local/implementations/s3.py b/cloudpathlib/local/implementations/s3.py index edc8c67e..e74c72fe 100644 --- a/cloudpathlib/local/implementations/s3.py +++ b/cloudpathlib/local/implementations/s3.py @@ -4,6 +4,9 @@ from ..localclient import LocalClient from ..localpath import LocalPath +# Import raw I/O class to ensure it's registered +from ...s3.s3_io import _S3StorageRaw # noqa: F401 + local_s3_implementation = CloudImplementation() """Replacement for "s3" CloudImplementation meta object in cloudpathlib.implementation_registry""" @@ -60,3 +63,4 @@ def etag(self): local_s3_implementation.name = "s3" local_s3_implementation._client_class = LocalS3Client local_s3_implementation._path_class = LocalS3Path +local_s3_implementation._raw_io_class = _S3StorageRaw diff --git a/cloudpathlib/local/localclient.py b/cloudpathlib/local/localclient.py index b3f73932..fbb1155f 100644 --- a/cloudpathlib/local/localclient.py +++ b/cloudpathlib/local/localclient.py @@ -7,11 +7,23 @@ import sys from tempfile import TemporaryDirectory from time import sleep -from typing import Callable, ClassVar, Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, +) from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from ..client import Client +from ..client import Client, _UploadPart from ..enums import FileCacheMode +from ..exceptions import CloudPathFileNotFoundError from .localpath import LocalPath @@ -28,19 +40,22 @@ class LocalClient(Client): def __init__( self, - *args, + *args: Any, local_storage_dir: Optional[Union[str, os.PathLike]] = None, file_cache_mode: Optional[Union[str, FileCacheMode]] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, - **kwargs, - ): + streaming_max_concurrency: int = 1, + **kwargs: Any, + ) -> None: self._local_storage_dir = local_storage_dir + self._local_upload_buffers: Dict[str, List[Tuple[int, bytes]]] = {} super().__init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) @classmethod @@ -215,11 +230,67 @@ def _generate_presigned_url( query["signature"] = "local" return urlunsplit(parts._replace(query=urlencode(query))) + def _range_download(self, cloud_path: LocalPath, start: int, end: int) -> bytes: + """Download a byte range from local storage.""" + local_path = self._cloud_path_to_local(cloud_path) + if not local_path.exists(): + raise CloudPathFileNotFoundError(f"File not found: {cloud_path}") + + with open(local_path, "rb") as f: + f.seek(start) + length = end - start + 1 + return f.read(length) + + def _get_content_length(self, cloud_path: LocalPath) -> int: + """Get the size of a local file.""" + local_path = self._cloud_path_to_local(cloud_path) + if not local_path.exists(): + raise CloudPathFileNotFoundError(f"File not found: {cloud_path}") + return local_path.stat().st_size + + def _initiate_multipart_upload(self, cloud_path: LocalPath) -> str: + """Return a unique upload ID so concurrent uploads don't share a buffer.""" + import uuid + + return str(uuid.uuid4()) + + def _upload_part( + self, cloud_path: LocalPath, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Buffer a part by upload ID.""" + if upload_id not in self._local_upload_buffers: + self._local_upload_buffers[upload_id] = [] + self._local_upload_buffers[upload_id].append((part_number, data)) + return {"part_number": part_number} + + def _complete_multipart_upload( + self, cloud_path: LocalPath, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + if upload_id not in self._local_upload_buffers: + return + + buffer = self._local_upload_buffers.pop(upload_id, []) + buffer.sort(key=lambda x: x[0]) + complete_data = b"".join([data for _, data in buffer]) + + local_path = self._cloud_path_to_local(cloud_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.write_bytes(complete_data) + + def _abort_multipart_upload(self, cloud_path: LocalPath, upload_id: str) -> None: + self._local_upload_buffers.pop(upload_id, None) + + def _put_empty_object(self, cloud_path: LocalPath) -> None: + """Create a zero-byte local file.""" + local_path = self._cloud_path_to_local(cloud_path) + local_path.parent.mkdir(parents=True, exist_ok=True) + local_path.write_bytes(b"") + _temp_dirs_to_clean: List[TemporaryDirectory] = [] @atexit.register -def clean_temp_dirs(): +def clean_temp_dirs() -> None: for temp_dir in _temp_dirs_to_clean: temp_dir.cleanup() diff --git a/cloudpathlib/s3/__init__.py b/cloudpathlib/s3/__init__.py index 77d27176..cb71b813 100644 --- a/cloudpathlib/s3/__init__.py +++ b/cloudpathlib/s3/__init__.py @@ -1,5 +1,6 @@ from .s3client import S3Client from .s3path import S3Path +from .s3_io import _S3StorageRaw # noqa: F401 - imported for registration __all__ = [ "S3Client", diff --git a/cloudpathlib/s3/s3_io.py b/cloudpathlib/s3/s3_io.py new file mode 100644 index 00000000..fb0684b7 --- /dev/null +++ b/cloudpathlib/s3/s3_io.py @@ -0,0 +1,17 @@ +"""S3 streaming I/O.""" + +from ..cloud_io import _CloudMultipartStorageRaw +from ..cloudpath import register_raw_io_class + + +@register_raw_io_class("s3") +class _S3StorageRaw(_CloudMultipartStorageRaw): + """S3 range reads and multipart writes.""" + + # S3 requires non-final parts of at least 5 MiB. + _INITIAL_PART_SIZE = 5 * 1024 * 1024 + _MIN_PART_SIZE = _INITIAL_PART_SIZE + _MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + _MAX_PARTS = 10_000 + _PARTS_PER_SIZE_TIER = 1_000 + _PROVIDER_NAME = "S3 multipart" diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index 9fa4bd75..3785e04d 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -1,12 +1,13 @@ +from functools import lru_cache import mimetypes import os from pathlib import Path, PurePosixPath -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union -from ..client import Client, register_client_class +from ..client import Client, _UploadPart, register_client_class from ..cloudpath import implementation_registry from ..enums import FileCacheMode -from ..exceptions import CloudPathException +from ..exceptions import CloudPathException, CloudPathFileNotFoundError from .s3path import S3Path try: @@ -19,6 +20,12 @@ implementation_registry["s3"].dependencies_loaded = False +@lru_cache(maxsize=None) +def _botocore_s3_operation_model(operation_name: str): + """Operation model from the installed botocore's bundled S3 service data (no network).""" + return botocore.session.get_session().get_service_model("s3").operation_model(operation_name) + + @register_client_class("s3") class S3Client(Client): """Client class for AWS S3 which handles authentication with AWS for [`S3Path`](../s3path/) @@ -40,6 +47,7 @@ def __init__( addressing_style: Optional[str] = None, boto3_transfer_config: Optional["TransferConfig"] = None, content_type_method: Optional[Callable] = mimetypes.guess_type, + streaming_max_concurrency: int = 1, extra_args: Optional[dict] = None, ): """Class constructor. Sets up a boto3 [`Session`]( @@ -77,6 +85,9 @@ def __init__( [s3 transfers](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig) content_type_method (Optional[Callable]): Function to call to guess media type (mimetype) when writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding). + streaming_max_concurrency (int): Maximum concurrent requests per open streaming + stream (background part uploads and read prefetch) when using + `FileCacheMode.streaming`; defaults to 1 (sequential). extra_args (Optional[dict]): A dictionary of extra args passed to download, upload, copy, and list functions as relevant. You can include any keys supported by upload, download, or copy operations, and we will pass on only the relevant args. To see the @@ -132,6 +143,7 @@ def __init__( local_cache_dir=local_cache_dir, content_type_method=content_type_method, file_cache_mode=file_cache_mode, + streaming_max_concurrency=streaming_max_concurrency, ) def _get_boto3_config(self, signature_version: Optional[str] = None): @@ -400,5 +412,128 @@ def _generate_presigned_url(self, cloud_path: S3Path, expire_seconds: int = 60 * ) return url + def _range_download(self, cloud_path: S3Path, start: int, end: int) -> bytes: + """Download a byte range from S3.""" + try: + response = self.client.get_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + Range=f"bytes={start}-{end}", + **self.boto3_dl_extra_args, + ) + body = response["Body"] + data = body.read() + body.close() + return data + except ClientError as e: + code = e.response["Error"]["Code"] + if code in ("404", "NoSuchKey"): + raise CloudPathFileNotFoundError(f"S3 object not found: {cloud_path}") + if code in ("InvalidRange", "416"): + return b"" + raise + except Exception as e: + if "InvalidRange" in str(e): + return b"" + raise + + def _get_content_length(self, cloud_path: S3Path) -> int: + """Get the size of an S3 object. + + head_object raises ClientError with code 404, not NoSuchKey. + """ + try: + response = self.client.head_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + **self.boto3_dl_extra_args, + ) + return response["ContentLength"] + except ClientError as e: + code = e.response["Error"]["Code"] + if code in ("404", "NoSuchKey"): + raise CloudPathFileNotFoundError(f"S3 object not found: {cloud_path}") + raise + + def _streaming_extra_args(self, operation_name: str) -> Dict[str, Any]: + """Return upload extras accepted by a specific low-level S3 operation.""" + try: + operation = self.client.meta.service_model.operation_model(operation_name) + except AttributeError: + # client objects that do not expose botocore's meta (e.g. test doubles): + # consult the installed botocore service model directly so filtering + # behaves identically to a real boto3 client + operation = _botocore_s3_operation_model(operation_name) + allowed = set(operation.input_shape.members) + return {key: value for key, value in self.boto3_ul_extra_args.items() if key in allowed} + + def _streaming_object_args(self, operation_name: str, cloud_path: S3Path) -> Dict[str, Any]: + extra_args = self._streaming_extra_args(operation_name) + if self.content_type_method is not None: + content_type, content_encoding = self.content_type_method(str(cloud_path)) + if content_type is not None: + extra_args["ContentType"] = content_type + if content_encoding is not None: + extra_args["ContentEncoding"] = content_encoding + return extra_args + + def _initiate_multipart_upload(self, cloud_path: S3Path) -> str: + """Start an S3 multipart upload, threading content-type and upload extra args.""" + extra_args = self._streaming_object_args("CreateMultipartUpload", cloud_path) + response = self.client.create_multipart_upload( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + **extra_args, + ) + return response["UploadId"] + + def _upload_part( + self, cloud_path: S3Path, upload_id: str, part_number: int, data: bytes + ) -> _UploadPart: + """Upload a part in an S3 multipart upload.""" + response = self.client.upload_part( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + UploadId=upload_id, + PartNumber=part_number, + Body=data, + **self._streaming_extra_args("UploadPart"), + ) + part = {"PartNumber": part_number, "ETag": response["ETag"]} + checksum_algorithm = self.boto3_ul_extra_args.get("ChecksumAlgorithm") + if checksum_algorithm is not None: + checksum_key = f"Checksum{checksum_algorithm}" + if checksum_key in response: + part[checksum_key] = response[checksum_key] + return part + + def _complete_multipart_upload( + self, cloud_path: S3Path, upload_id: str, parts: Sequence[_UploadPart] + ) -> None: + """Complete an S3 multipart upload.""" + self.client.complete_multipart_upload( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + UploadId=upload_id, + MultipartUpload={"Parts": parts}, + **self._streaming_extra_args("CompleteMultipartUpload"), + ) + + def _abort_multipart_upload(self, cloud_path: S3Path, upload_id: str) -> None: + """Abort an S3 multipart upload.""" + self.client.abort_multipart_upload( + Bucket=cloud_path.bucket, Key=cloud_path.key, UploadId=upload_id + ) + + def _put_empty_object(self, cloud_path: S3Path) -> None: + """Upload a zero-byte object, threading content-type and upload extra args.""" + extra_args = self._streaming_object_args("PutObject", cloud_path) + self.client.put_object( + Bucket=cloud_path.bucket, + Key=cloud_path.key, + Body=b"", + **extra_args, + ) + S3Client.S3Path = S3Client.CloudPath # type: ignore diff --git a/docs/docs/caching.ipynb b/docs/docs/caching.ipynb index f2be92c5..fc96c66c 100644 --- a/docs/docs/caching.ipynb +++ b/docs/docs/caching.ipynb @@ -443,16 +443,15 @@ "\n", "### Automatically\n", "\n", - "We provide a number of different ways for the cache to get cleared automatically for you depending on your use case. These range from no cache clearing done by `cloudpathlib` (`\"persistent\"`), to the most aggressive (`\"close_file\"`), which deletes a file from the cache as soon as the file handle is closed and the file is uploaded to the cloud, if it was changed).\n", + "We provide a number of different ways for the cache to get cleared automatically for you depending on your use case. These range from no cache clearing done by `cloudpathlib` (`\"persistent\"`), to the most aggressive (`\"close_file\"`), which deletes a file from the cache as soon as the file handle is closed and the file is uploaded to the cloud, if it was changed). There is also a `\"streaming\"` mode that bypasses caching entirely for direct I/O.\n", "\n", "The modes are defined in the `FileCacheMode` enum, which you can use directly or you can use the corresponding string value. Examples of both methods are included below.\n", "\n", - "Note: There is not currently a cache mode that _never_ writes a file to disk and only keeps it in memory.\n", - "\n", " - `\"persistent\"` - `cloudpathlib` does not clear the cache at all. In this case, you must also pass a `local_cache_dir` when you instantiate the client.\n", " - `\"tmp_dir\"` (_default_) - Cached files are saved using Python's [`TemporaryDirectory`](https://docs.python.org/3/library/tempfile.html#tempfile.TemporaryDirectory). This provides three potential avenues for the cache to get cleared. First, cached files are removed by `cloudpathlib` when the `*Client` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called. Second, Python clears a temporary directory if all references to that directory leave scope. Finally since the folder is in an operating system temp directory, it will be cleared by the OS (which, depending on the OS, may not happen until system restart).\n", " - `\"cloudpath_object\"` - cached files are removed when the `CloudPath` object is garbage collected. This happens on the next garbage collection run after the object leaves scope or `del` is called.\n", " - `\"close_file\"` - since we only download a file to the cache on read/write, we can ensure the cache is empty by removing the cached file as soon as the read/write is finished. Reading/writing the same `CloudPath` multiple times will result in re-downloading the file from the cloud. Note: For this to work, `cloudpath` needs to be in control of the reading/writing of files. This means your code base should use the `CloudPath.write_*`, `CloudPath.read_*`, and `CloudPath.open` methods. Using `CloudPath.fspath` (or passing the `CloudPath` as a `PathLike` object to another library) will not clear the cache on file close since it was not opened by `cloudpathlib`.\n", + " - `\"streaming\"` - supported read/write modes stream directly from/to cloud storage using range requests for reads and multipart/block uploads for writes, without writing cache files to disk (append and update modes are the exception: they fall back to the local cache, which is cleaned up when the client is garbage collected). This mode uses only small in-memory buffers and is ideal for large files or memory-constrained environments. Note: `.fspath` and similar properties are not available in streaming mode. Currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n", "\n", "Note: Although we use it in the examples below, for `\"cloudpath_object\"` and `\"tmp_dir\"` you normally shouldn't need to explicitly call `del`. Letting Python garbage collection run on its own once all references to the object leave scope should be sufficient. See details [in the Python docs](https://docs.python.org/3/reference/datamodel.html?highlight=__del__#object.__del__)).\n" ] @@ -712,6 +711,73 @@ "shutil.rmtree(client_cache_dir)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### File cache mode: `\"streaming\"`\n", + "\n", + "The `\"streaming\"` mode provides direct streaming I/O without any local caching. This is ideal for:\n", + "\n", + "- **Large files** that don't fit in memory or disk\n", + "- **Partial reads** where you only need part of a file\n", + "- **Sequential processing** where you read/write once\n", + "- **Memory-constrained environments**\n", + "\n", + "Unlike other cache modes, streaming mode:\n", + "- Reads data directly from cloud storage using range requests\n", + "- Writes data directly to cloud storage using multipart/block uploads\n", + "- Never creates cached files on disk (only uses small in-memory buffers)\n", + "- Works with standard Python file-like interfaces\n", + "\n", + "**Note:** Streaming mode is currently supported for S3, Azure Blob Storage, Google Cloud Storage, and HTTP/HTTPS (read-only).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Streaming a large file\n", + "streaming_client = S3Client(file_cache_mode=FileCacheMode.streaming)\n", + "\n", + "flood_image = streaming_client.CloudPath(\n", + " \"s3://ladi/Images/FEMA_CAP/2020/70349/DSC_0002_a89f1b79-786f-4dac-9dcc-609fb1a977b1.jpg\"\n", + ")\n", + "\n", + "# Read the image in streaming mode - no cache file created\n", + "with flood_image.open(\"rb\") as f:\n", + " i = Image.open(f)\n", + " print(\"Image loaded via streaming...\")\n", + "\n", + "# No cache file exists - streaming mode doesn't create one\n", + "print(\"Cache file exists: \", flood_image._local.exists())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example: Reading in chunks for memory efficiency\n", + "streaming_client = S3Client(file_cache_mode=FileCacheMode.streaming)\n", + "\n", + "flood_image = streaming_client.CloudPath(\n", + " \"s3://ladi/Images/FEMA_CAP/2020/70349/DSC_0002_a89f1b79-786f-4dac-9dcc-609fb1a977b1.jpg\"\n", + ")\n", + "\n", + "# Read file in 8KB chunks without loading entire file\n", + "print(\"Reading file in chunks:\")\n", + "chunk_count = 0\n", + "with flood_image.open(\"rb\", buffer_size=8192) as f:\n", + " while chunk := f.read(8192):\n", + " chunk_count += 1\n", + "\n", + "print(f\"Read {chunk_count} chunks without caching the file\")" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/docs/streaming_io.md b/docs/docs/streaming_io.md new file mode 100644 index 00000000..d0e67730 --- /dev/null +++ b/docs/docs/streaming_io.md @@ -0,0 +1,875 @@ +# Streaming I/O + +cloudpathlib provides streaming I/O capabilities for cloud storage through Python's standard I/O interfaces. + +## Overview + +By default, CloudPathLib downloads files to a local cache before opening them. While this works well for many use cases, it can be inefficient for: + +- **Large files** that don't fit in memory or disk +- **Partial reads** where you only need to access part of a file +- **Sequential processing** where you read a file once and discard it +- **Write-only workflows** where you're generating data to upload + +The streaming I/O system solves these problems by: + +- Reading data directly from cloud storage using range requests +- Writing data directly to cloud storage using multipart/block uploads +- Providing standard Python file-like objects that work with any library +- Eliminating the need for local disk caching + +## Quick Start + +### Enable Streaming Mode + +To use streaming I/O, set your client's `file_cache_mode` to `FileCacheMode.streaming`: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Option 1: Set streaming mode on the client +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + for line in f: + print(line.strip()) + +# Option 2: Change mode on existing client +client = S3Client() +client.file_cache_mode = FileCacheMode.streaming + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + content = f.read() + +# Option 3: Temporarily enable streaming +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +original_mode = path.client.file_cache_mode +path.client.file_cache_mode = FileCacheMode.streaming + +with path.open("rt") as f: + content = f.read() + +path.client.file_cache_mode = original_mode # Restore +``` + +### Basic Examples + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Create a client with streaming enabled +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read a text file +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + for line in f: + print(line.strip()) + +# Write a binary file +path = S3Path("s3://bucket/output.bin", client=client) +with path.open("wb") as f: + f.write(b"Hello, cloud!") + +# Read binary data in chunks +path = S3Path("s3://bucket/large-file.bin", client=client) +with path.open("rb") as f: + while chunk := f.read(8192): + process(chunk) +``` + +## API Reference + +!!! tip "Prefer `CloudPath.open()`" + The usual entry point is `CloudPath.open()` with `FileCacheMode.streaming`. `CloudBufferedIO` and `CloudTextIO` are also public for integrations that need to construct a provider-backed file object directly. + +### `FileCacheMode` Enum + +Controls how `CloudPath.open()` handles file caching: + +- `FileCacheMode.cloudpath_object`: Default - cache files in CloudPath object +- `FileCacheMode.tmp_dir`: Cache files in temporary directory +- `FileCacheMode.persistent`: Cache files persistently +- `FileCacheMode.close_file`: Close file after reading +- **`FileCacheMode.streaming`**: Stream directly without caching + +```python +from cloudpathlib.enums import FileCacheMode + +# Set on client initialization +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Or change dynamically +client.file_cache_mode = FileCacheMode.streaming +``` + +### `CloudPath.open()` + +Opens a cloud file in streaming mode when `file_cache_mode` is set to `FileCacheMode.streaming`. + +```python +CloudPath.open( + mode: str = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + *, + buffer_size: Optional[int] = None, +) -> Union[CloudBufferedIO, CloudTextIO, IO] +``` + +**Parameters:** + +- `mode`: File mode - binary (`'rb'`, `'wb'`, etc.) or text (`'r'`, `'w'`, `'rt'`, `'wt'`, etc.) +- `buffering`: Standard Python buffering control. Binary mode supports `0` for an unbuffered raw stream. +- `encoding`: Text encoding (default: platform locale, text mode only) +- `errors`: Error handling strategy (default: `"strict"`, text mode only) +- `newline`: Newline handling (text mode only) +- `buffer_size`: Size of read/write buffer in bytes (default: 5 MiB) + +**Returns:** + +- `CloudBufferedIO` for binary modes (when streaming) +- `CloudTextIO` for text modes (when streaming) +- Standard file object (when not streaming) + +### Compatibility with standard interfaces + +Streaming file objects subclass the standard `io` base classes, so code written +against Python's file protocol works without modification: + +- `CloudBufferedIO` is an `io.BufferedIOBase` (and `io.IOBase`); `CloudTextIO` + is an `io.TextIOWrapper`. `isinstance` checks against the `io` ABCs pass. +- `CloudPath.open()` keeps the same signature and mode grammar as + `pathlib.Path.open()`, so call sites that accept either a `Path` or a + `CloudPath` behave the same in both cases. +- Any library that accepts an open file object works with streaming streams: + `json`, `csv`, `pickle`, `zipfile`, `tarfile`, `pandas`, `PIL.Image.open`, + `pyarrow`, etc. Read streams are seekable, so formats that require random + access (zip archives, parquet) also work. +- The one intentional gap is `os.PathLike`: `os.fspath(path)` / + `path.fspath` raise `CloudPathNotImplementedError` in streaming mode because + there is no local file to point at. Pass the open file object instead of the + path to libraries that require a filesystem path. + +### `CloudBufferedIO` + +Binary file-like object implementing `io.BufferedIOBase`. + +!!! note "Usually returned by `CloudPath.open()`" + Most applications should let `CloudPath.open()` construct this class. Direct construction is supported when implementing file-object integrations. + +**Key Methods:** + +- `read(size=-1)`: Read up to size bytes (all if size is -1) +- `read1(size=-1)`: Read up to size bytes with one underlying read call +- `readinto(b)`: Read bytes into a pre-allocated buffer +- `write(b)`: Write bytes +- `flush()`: Flush write buffer to cloud storage +- `seek(offset, whence=SEEK_SET)`: Change stream position +- `tell()`: Return current stream position +- `close()`: Close file and finalize upload + +**Properties:** + +- `name`: The cloud path +- `mode`: File mode (e.g., `"rb"`, `"wb"`) +- `closed`: Whether the file is closed + +**Capability Flags:** + +- `readable()`: Returns True for read modes +- `writable()`: Returns True for write modes +- `seekable()`: Returns `True` for readable streams. Streaming writes are sequential and return `False`. + +### `CloudTextIO` + +Text file-like object implementing `io.TextIOBase`. + +!!! note "Usually returned by `CloudPath.open()`" + Most applications should let `CloudPath.open()` construct this class. Direct construction is supported when implementing file-object integrations. + +**Key Methods:** + +- `read(size=-1)`: Read up to size characters +- `readline(size=-1)`: Read one line +- `readlines(hint=-1)`: Read list of lines +- `write(s)`: Write string +- `writelines(lines)`: Write list of strings +- `flush()`: Flush write buffer +- `seek(offset, whence=SEEK_SET)`: Change position +- `tell()`: Return current position +- `close()`: Close file + +**Properties:** + +- `name`: The cloud path +- `mode`: File mode (e.g., `"rt"`, `"wt"`) +- `encoding`: Text encoding +- `errors`: Error handling strategy +- `newlines`: Newline(s) encountered +- `buffer`: Underlying binary buffer (CloudBufferedIO) +- `closed`: Whether the file is closed + +**Iteration:** + +CloudTextIO supports iteration: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + for line in f: + process(line) +``` + +## Usage Examples + +### Reading Large Files in Chunks + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/huge-file.csv", client=client) + +# Process a large file without loading it entirely into memory +with path.open("rt") as f: + header = f.readline() + for line in f: + process_csv_line(line) +``` + +### Partial File Reads + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.bin", client=client) + +# Read just the header of a file +with path.open("rb") as f: + header = f.read(1024) # Read first 1KB + parse_header(header) + + # Seek to specific position + f.seek(10000) + chunk = f.read(100) +``` + +### Streaming Uploads + +```python +from cloudpathlib import AzureBlobPath, AzureBlobClient +from cloudpathlib.enums import FileCacheMode +import json + +client = AzureBlobClient(file_cache_mode=FileCacheMode.streaming) +path = AzureBlobPath("az://container/output.json", client=client) + +# Write data directly to cloud without local file +with path.open("wt") as f: + f.write('{"items": [\n') + for i, item in enumerate(generate_items()): + if i > 0: + f.write(',\n') + f.write(json.dumps(item)) + f.write('\n]}') +``` + +### Using with pandas + +```python +import pandas as pd +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read CSV directly from cloud +read_path = S3Path("s3://bucket/data.csv", client=client) +with read_path.open("rt") as f: + df = pd.read_csv(f) + +# Write CSV directly to cloud +write_path = S3Path("s3://bucket/output.csv", client=client) +with write_path.open("wt") as f: + df.to_csv(f, index=False) +``` + + +### Using with parquet + +Streaming read streams are seekable, which is exactly what columnar formats +need: `pyarrow` seeks to the parquet footer to read the file metadata, then +fetches only the byte ranges for the row groups and columns you ask for — the +rest of the object is never downloaded. + +```python +import pyarrow.parquet as pq +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.parquet", client=client) + +with path.open("rb") as f: + parquet_file = pq.ParquetFile(f) + + # metadata comes from the footer alone + print(parquet_file.metadata.num_rows, parquet_file.schema_arrow) + + # reads only the column chunks for "user_id" + table = parquet_file.read(columns=["user_id"]) +``` + +For column-slicing workloads, a smaller `buffer_size` (e.g. 64 KiB - 1 MiB) +reduces over-fetch around the footer and column chunk boundaries; for reading +most of the file, keep the default. + +### Using with PIL/Pillow + +```python +from PIL import Image +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Read image +read_path = S3Path("s3://bucket/image.jpg", client=client) +with read_path.open("rb") as f: + img = Image.open(f) + img.show() + +# Write image +write_path = S3Path("s3://bucket/output.png", client=client) +with write_path.open("wb") as f: + img.save(f, format="PNG") +``` + +### Custom Buffer Size + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +# Use a larger buffer for better throughput on fast connections +path = S3Path("s3://bucket/large-file.bin", client=client) +with path.open("rb", buffer_size=16 * 1024 * 1024) as f: + data = f.read() + +# Use a smaller buffer for memory-constrained environments +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt", buffer_size=64 * 1024) as f: + for line in f: + process(line) +``` + +## Performance Considerations + +### Buffer Size + +The `buffer_size` parameter controls how much data is fetched from/written to cloud storage in each request: + +- **Default** (5 MiB): Matches the multi-MiB block sizes used by comparable + tools (fsspec/s3fs/gcsfs) so per-request latency does not dominate sequential + throughput. Reads never fetch past EOF, so small objects only pay for their + actual size. +- **Smaller buffers** (64 KiB - 1 MiB): Less memory per open stream and less + over-fetch when reading small slices of large objects, at the cost of more + requests for sequential scans. +- **Larger buffers**: Fewer, bigger requests; memory per open stream grows to + match. + +Two request-count notes: + +- A full-object `read()` is always satisfied with a single ranged request + regardless of `buffer_size`. +- Each buffered refill costs one ranged request, so a sequential scan of a + 1 GiB object makes ~205 requests at the 5 MiB default versus ~16,000 at + 64 KiB. + +### Read Patterns + +- **Sequential reads**: Optimal performance - data is fetched ahead as needed +- **Random seeks**: Each seek may trigger a new range request - less efficient +- **Small random reads**: Consider downloading the file to cache instead + +### Concurrency + +Pass `streaming_max_concurrency=N` when constructing a client to let each open +streaming stream issue up to `N` requests in parallel (the default of `1` is +fully sequential): + +```python +client = S3Client( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=4, +) +``` + +- **Writes**: completed parts upload in a background thread pool while your + code keeps writing; the stream blocks only when `N` parts are already in + flight, so buffered memory is bounded by roughly `N x` part size. Works for + S3, Azure, and GCS (all use order-independent multipart/block uploads); + HTTP writes are a single request and are unaffected. +- **Reads**: the next `N` byte ranges are prefetched in the background while + you consume the current one, pipelining sequential scans. Seeking outside + the prefetched window discards it. +- Failures in background requests surface on the next `write()`/`close()` + (aborting the upload) or the next `read()`, exactly like sequential errors. + +Semantics that no configuration changes: + +- **Streams are not thread-safe.** Like ordinary Python file objects, a single + `CloudBufferedIO`/`CloudTextIO` instance must not be shared between threads + without external locking. Open one stream per thread instead. +- **Concurrent readers are safe.** Any number of streams (across threads or + processes) can read the same object simultaneously; each issues independent + range requests and holds independent positions. +- **Concurrent writers to the same object are last-committer-wins.** Each + writer's upload is isolated (S3 multipart upload IDs; per-session Azure block + IDs), so writers cannot corrupt each other's data — whichever stream closes + last determines the final object. +- **Conflict detection**: open a write stream with + `force_overwrite_to_cloud=False` and closing raises `OverwriteNewerCloudError` + instead of overwriting a version of the object that was uploaded while the + stream was open. + +### Write Contract + +Object storage is fundamentally a **sequential, write-once** medium. +Streaming mode reflects that contract: + +| Mode | Streaming behaviour | +|------|---------------------| +| `wb`, `w`, `xb`, `x` | True streaming — data is forwarded to the provider as it arrives | +| `ab`, `a`, `r+b`, `r+`, `w+b`, `w+` | **Falls back to cache** — the object is downloaded, mutated locally, then re-uploaded on close. Semantics are correct; performance matches the cached path. | + +!!! warning "There is no true streaming append" + Object stores cannot append to (or modify a byte range of) an existing + object — every write creates a whole new object. Anything that opens an + existing file for appending or in-place update (`a`, `a+`, `r+`, `w+`) + therefore cannot stream: cloudpathlib downloads the entire object to the + local cache, applies the writes there, and re-uploads the entire object on + close. That is correct but costs a full download plus a full upload (and + temporary disk space) proportional to the object's size — for a + log-appending workload, prefer writing many small objects or a + provider-native mechanism instead. + +Attempting to seek on a write-only streaming stream raises +`io.UnsupportedOperation` because the provider has already accepted +the earlier bytes. + +Streaming writes honor `force_overwrite_to_cloud` (and the +`CLOUDPATHLIB_FORCE_OVERWRITE_TO_CLOUD` environment variable): when it +resolves to `False`, closing the stream raises `OverwriteNewerCloudError` +instead of clobbering a version of the object that was uploaded while the +stream was open. + +### Multipart/Block Uploads + +For write operations, the streaming I/O system automatically handles: + +- **S3**: Multipart upload; non-final parts are buffered until they reach + the provider minimum of **5 MiB** (S3 rejects smaller non-final parts). + The final part may be smaller than 5 MiB. +- **Azure**: Block blob staging — blocks grow adaptively during very large uploads + and are committed on close. +- **GCS**: XML API multipart upload — parts are buffered to the provider + minimum of **5 MiB** and assembled on close, mirroring the S3 mechanism + (set an `AbortIncompleteMultipartUpload` bucket lifecycle rule to expire + uploads orphaned by hardware failure). + +## Provider-Specific Behavior + +### AWS S3 + +- Uses boto3 `get_object()` with `Range` header for reads +- Uses boto3 multipart upload API for writes +- Supports all S3-compatible storage (MinIO, Ceph, etc.) + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +### Azure Blob Storage + +- Uses Azure SDK `download_blob()` with offset/length for reads +- Uses block blob staging and commit for writes +- Compatible with Azure Data Lake Storage Gen2 + +```python +from cloudpathlib import AzureBlobPath, AzureBlobClient +from cloudpathlib.enums import FileCacheMode + +client = AzureBlobClient(file_cache_mode=FileCacheMode.streaming) +path = AzureBlobPath("az://container/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +### Google Cloud Storage + +- Uses GCS SDK `download_as_bytes()` with start/end for reads +- Uses the XML API multipart upload for writes (via the SDK's transfer-manager + machinery), which supports concurrent part uploads +- Supports GCS-specific features through client configuration + +```python +from cloudpathlib import GSPath, GSClient +from cloudpathlib.enums import FileCacheMode + +client = GSClient(file_cache_mode=FileCacheMode.streaming) +path = GSPath("gs://bucket/file.txt", client=client) + +with path.open("rt") as f: + content = f.read() +``` + +### HTTP and HTTPS + +- Requires servers to honor byte-range requests for streaming reads +- Uses the client's configured `write_file_http_method` for writes +- Spools single-request upload bodies with bounded memory, using a temporary file above 8 MiB + +## Comparison with Cached Mode + +| Feature | Streaming (`FileCacheMode.streaming`) | Cached (default) | +|---------|--------------------------------------|------------------| +| **Disk usage** | None for cloud providers; HTTP uploads may use a temporary spool | Full file size | +| **Memory usage** | Configurable buffer | Varies | +| **Read performance** | Sequential: Good
Random: Moderate | Fast (local disk) | +| **Write performance** | Good (direct upload) | Fast write, slower close | +| **Partial reads** | Efficient | Downloads full file | +| **Large files** | Excellent | Limited by disk space | +| **Offline access** | No | Yes (after download) | +| **Compatibility** | Standard I/O interfaces | Standard I/O interfaces | + +## Best Practices + +### When to Use Streaming I/O + +✅ **Good use cases:** + +- Large files that don't fit in memory/disk +- Reading only part of a file (e.g., headers, metadata) +- Sequential processing (one-pass reads) +- Direct upload of generated content +- Integration with libraries that accept file-like objects + +❌ **Consider caching instead:** + +- Small files (< 10 MB) +- Frequent random access to same file +- Multiple passes over the same data +- Offline processing +- Maximum read performance required +- Libraries that require file paths (`.fspath` not available in streaming mode) + +### Error Handling + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +try: + with path.open("rt") as f: + content = f.read() +except FileNotFoundError: + print("File not found in cloud storage") +except PermissionError: + print("Access denied") +except Exception as e: + print(f"Error: {e}") +``` + +### Resource Management + +Always use context managers to ensure proper cleanup: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# ✅ Good - file is automatically closed +with path.open("rt") as f: + content = f.read() + +# ❌ Bad - must remember to close manually +f = path.open("rt") +content = f.read() +f.close() # Easy to forget! +``` + +### Streaming Mode Limitations + +When using `FileCacheMode.streaming`, certain CloudPath features are not available because streaming mode avoids the local file cache (the append/update modes of `open` are the exception — they fall back to the cache, which is cleaned up when the client is garbage collected): + +**Not Available:** +- `.fspath` property - Raises `CloudPathNotImplementedError` +- `.__fspath__()` method - Raises `CloudPathNotImplementedError` +- Passing CloudPath as `os.PathLike` to libraries that need file paths + +**Workaround:** +Use `CloudPath.open()` and pass the file-like object to libraries that accept file handles instead of file paths. + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode +import pandas as pd + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/data.csv", client=client) + +# ❌ This will raise an error in streaming mode +# df = pd.read_csv(path.fspath) + +# ✅ Use this instead - pass the open file handle +with path.open("rt") as f: + df = pd.read_csv(f) +``` + +## Compatibility + +### Python I/O Interfaces + +The streaming I/O classes are fully compatible with Python's I/O hierarchy: + +```python +import io +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) + +path = S3Path("s3://bucket/file.bin", client=client) +with path.open("rb") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.BufferedIOBase) + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.TextIOBase) +``` + +### Third-Party Libraries + +Works with any library that accepts file-like objects: + +- **Data processing**: pandas, NumPy, PyArrow +- **Images**: PIL/Pillow, OpenCV +- **Compression**: gzip, zipfile, tarfile +- **Serialization**: pickle, json, yaml +- **Scientific**: h5py, netCDF4 + +## Troubleshooting + +### "File not found" errors + +Ensure the file exists and you have read permissions: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +if path.exists(): + with path.open("rt") as f: + content = f.read() +``` + +### Slow performance + +Try increasing buffer size: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# Larger buffer for faster networks +with path.open("rb", buffer_size=16 * 1024 * 1024) as f: + data = f.read() +``` + +### Out of memory + +Try smaller buffer size or process in chunks: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/huge.bin", client=client) + +# Process large file in chunks +with path.open("rb", buffer_size=8192) as f: + while chunk := f.read(8192): + process_chunk(chunk) +``` + +## Migration Guide + +### From Cached to Streaming + +Before: + +```python +from cloudpathlib import S3Path + +path = S3Path("s3://bucket/file.txt") +with path.open("rt") as f: # Downloads to cache + content = f.read() +``` + +After: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +# Option 1: Set on client initialization +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: # Streams directly + content = f.read() + +# Option 2: Change client mode +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +path.client.file_cache_mode = FileCacheMode.streaming +with path.open("rt") as f: # Streams directly + content = f.read() +``` + +## Advanced Topics + +### Custom Clients + +Pass custom clients with specific configurations: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode +# Custom S3-compatible endpoint and upload metadata +client = S3Client( + file_cache_mode=FileCacheMode.streaming, + endpoint_url="https://objects.example.com", + addressing_style="path", + extra_args={"ServerSideEncryption": "AES256"}, +) + +path = S3Path("s3://bucket/file.txt", client=client) +with path.open("rt") as f: + content = f.read() +``` + +### Multiple Files + +Process multiple files efficiently: + +```python +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +bucket = S3Path("s3://bucket/", client=client) + +for file_path in bucket.glob("*.csv"): + with file_path.open("rt") as f: + process_csv(f) +``` + +### Encoding Detection + +For files with unknown encoding: + +```python +import chardet +from cloudpathlib import S3Path, S3Client +from cloudpathlib.enums import FileCacheMode + +client = S3Client(file_cache_mode=FileCacheMode.streaming) +path = S3Path("s3://bucket/file.txt", client=client) + +# Read a small sample to detect encoding +with path.open("rb") as f: + sample = f.read(10000) + detected = chardet.detect(sample) + encoding = detected['encoding'] + +# Re-open with detected encoding +with path.open("rt", encoding=encoding) as f: + content = f.read() +``` + +### Context Manager for Temporary Streaming + +Use a context manager to temporarily enable streaming mode: + +```python +from contextlib import contextmanager +from cloudpathlib import S3Client +from cloudpathlib.enums import FileCacheMode + +@contextmanager +def streaming_mode(client): + """Temporarily enable streaming mode on a client.""" + original_mode = client.file_cache_mode + try: + client.file_cache_mode = FileCacheMode.streaming + yield client + finally: + client.file_cache_mode = original_mode + +# Usage +client = S3Client() +path = S3Path("s3://bucket/file.txt", client=client) + +with streaming_mode(client): + with path.open("rt") as f: + content = f.read() # Uses streaming + +# Back to cached mode +with path.open("rt") as f: + content = f.read() # Uses caching +``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index cd917ce3..1fcbd3d7 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -21,6 +21,7 @@ nav: - AnyPath: "anypath-polymorphism.md" - HTTP URLs: "http.md" - Caching: "caching.ipynb" + - Streaming I/O: "streaming_io.md" - Compatibility: "patching_builtins.ipynb" - Other Client settings: "other_client_settings.md" - Testing code that uses cloudpathlib: "testing_mocked_cloudpathlib.ipynb" diff --git a/requirements-dev.txt b/requirements-dev.txt index dc9e3560..2f6ca636 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,6 +20,7 @@ nbautoexport pandas pillow psutil +pyarrow pydantic pytest<9.1 pytest-cases>=3.9.1 diff --git a/tests/conftest.py b/tests/conftest.py index 9732c39a..dddb0a4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,20 +21,14 @@ from shortuuid import uuid from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed -from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path -from cloudpathlib.cloudpath import implementation_registry -from cloudpathlib.http.httpclient import HttpClient, HttpsClient -from cloudpathlib.http.httppath import HttpPath, HttpsPath +from cloudpathlib.cloudpath import implementation_registry, CloudImplementation from cloudpathlib.local import ( local_azure_blob_implementation, LocalAzureBlobClient, - LocalAzureBlobPath, local_gs_implementation, LocalGSClient, - LocalGSPath, local_s3_implementation, LocalS3Client, - LocalS3Path, ) import cloudpathlib.azure.azblobclient from cloudpathlib.azure.azblobclient import _hns_rmtree @@ -80,8 +74,7 @@ class CloudProviderTestRig: def __init__( self, - path_class: type, - client_class: type, + cloud_implementation: CloudImplementation, drive: str = "drive", test_dir: str = "", live_server: bool = False, @@ -92,8 +85,7 @@ def __init__( path_class (type): CloudPath subclass client_class (type): Client subclass """ - self.path_class = path_class - self.client_class = client_class + self.cloud_implementation = cloud_implementation self.drive = drive self.test_dir = test_dir self.live_server = live_server # if the server is a live server @@ -101,6 +93,18 @@ def __init__( required_client_kwargs if required_client_kwargs is not None else {} ) + @property + def path_class(self): + return self.cloud_implementation.path_class + + @property + def client_class(self): + return self.cloud_implementation.client_class + + @property + def raw_io_class(self): + return self.cloud_implementation.raw_io_class + @property def cloud_prefix(self): return self.path_class.cloud_prefix @@ -201,8 +205,7 @@ def _azure_fixture(conn_str_env_var, adls_gen2, request, monkeypatch, assets_dir ) rig = CloudProviderTestRig( - path_class=AzureBlobPath, - client_class=AzureBlobClient, + cloud_implementation=implementation_registry["azure"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -285,8 +288,7 @@ def gs_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setattr(cloudpathlib.gs.gsclient, "google_default_auth", mock_default_auth) rig = CloudProviderTestRig( - path_class=GSPath, - client_class=GSClient, + cloud_implementation=implementation_registry["gs"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -335,8 +337,7 @@ def s3_rig(request, monkeypatch, assets_dir, live_server): ) rig = CloudProviderTestRig( - path_class=S3Path, - client_class=S3Client, + cloud_implementation=implementation_registry["s3"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -419,8 +420,7 @@ def _spin_up_bucket(): ) rig = CloudProviderTestRig( - path_class=S3Path, - client_class=S3Client, + cloud_implementation=implementation_registry["s3"], drive=drive, test_dir=test_dir, live_server=live_server, @@ -458,8 +458,7 @@ def local_azure_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "azure", local_azure_blob_implementation) rig = CloudProviderTestRig( - path_class=LocalAzureBlobPath, - client_class=LocalAzureBlobClient, + cloud_implementation=local_azure_blob_implementation, drive=drive, test_dir=test_dir, ) @@ -489,8 +488,7 @@ def local_gs_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "gs", local_gs_implementation) rig = CloudProviderTestRig( - path_class=LocalGSPath, - client_class=LocalGSClient, + cloud_implementation=local_gs_implementation, drive=drive, test_dir=test_dir, ) @@ -519,8 +517,7 @@ def local_s3_rig(request, monkeypatch, assets_dir, live_server): monkeypatch.setitem(implementation_registry, "s3", local_s3_implementation) rig = CloudProviderTestRig( - path_class=LocalS3Path, - client_class=LocalS3Client, + cloud_implementation=local_s3_implementation, drive=drive, test_dir=test_dir, ) @@ -558,8 +555,7 @@ def http_rig(request, assets_dir, http_server): # noqa: F811 _sync_filesystem() rig = CloudProviderTestRig( - path_class=HttpPath, - client_class=HttpClient, + cloud_implementation=implementation_registry["http"], drive=drive, test_dir=test_dir, ) @@ -590,8 +586,7 @@ def https_rig(request, assets_dir, https_server): # noqa: F811 skip_verify_ctx.load_verify_locations(utilities_dir / "insecure-test.pem") rig = CloudProviderTestRig( - path_class=HttpsPath, - client_class=HttpsClient, + cloud_implementation=implementation_registry["https"], drive=drive, test_dir=test_dir, required_client_kwargs=dict( diff --git a/tests/http_fixtures.py b/tests/http_fixtures.py index d43ce236..dbf86f43 100644 --- a/tests/http_fixtures.py +++ b/tests/http_fixtures.py @@ -75,7 +75,61 @@ def do_POST(self): @retry(stop=stop_after_attempt(5), wait=wait_fixed(0.1)) def do_GET(self): - super().do_GET() + """Handle GET requests with optional Range header support.""" + # Check if this is a range request + range_header = self.headers.get("Range") + if range_header: + self._handle_range_request(range_header) + else: + super().do_GET() + + def _handle_range_request(self, range_header): + """Handle Range requests for partial content.""" + path = Path(self.translate_path(self.path)) + + if not path.exists() or not path.is_file(): + self.send_error(404, "File not found") + return + + # Parse the Range header (format: "bytes=start-end") + try: + range_spec = range_header.replace("bytes=", "").strip() + parts = range_spec.split("-") + start = int(parts[0]) if parts[0] else 0 + + file_size = path.stat().st_size + + # Handle end byte + if len(parts) > 1 and parts[1]: + end = int(parts[1]) + else: + end = file_size - 1 + + # Validate range (RFC 7233: a range is unsatisfiable only when start is past + # EOF; an end past EOF is clamped to the final byte, as real servers do) + if start < 0 or start >= file_size or start > end: + self.send_error(416, "Requested Range Not Satisfiable") + self.send_header("Content-Range", f"bytes */{file_size}") + self.end_headers() + return + end = min(end, file_size - 1) + + # Read the requested range + with path.open("rb") as f: + f.seek(start) + content = f.read(end - start + 1) + + # Send partial content response + self.send_response(206) # Partial Content + self.send_header("Content-Type", self.guess_type(str(path))) + self.send_header("Content-Length", str(len(content))) + self.send_header("Content-Range", f"bytes {start}-{end}/{file_size}") + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + self.wfile.write(content) + + except (ValueError, IndexError) as e: + self.send_error(400, f"Bad Range header: {e}") @retry(stop=stop_after_attempt(5), wait=wait_fixed(0.1)) def do_HEAD(self): diff --git a/tests/mock_clients/mock_azureblob.py b/tests/mock_clients/mock_azureblob.py index 2fc8f7eb..f110c168 100644 --- a/tests/mock_clients/mock_azureblob.py +++ b/tests/mock_clients/mock_azureblob.py @@ -8,7 +8,7 @@ from azure.storage.blob import BlobProperties from azure.storage.blob._list_blobs_helper import BlobPrefix from azure.storage.blob._shared.authentication import SharedKeyCredentialPolicy -from azure.core.exceptions import ResourceNotFoundError +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from .utils import delete_empty_parents_up_to_root @@ -61,6 +61,9 @@ def __init__(self, test_dir, adls): self.metadata_cache = _JsonCache(self.root / ".metadata") self.adls_gen2 = adls + # For block blob uploads (multipart) - shared across all blob clients + self._staged_blocks = {} + @classmethod def from_connection_string(cls, conn_str, credential): # configured in conftest.py @@ -114,7 +117,7 @@ def url(self): def get_blob_properties(self): path = self.root / self.key if path.exists() and path.is_file(): - return BlobProperties( + props = BlobProperties( **{ "name": self.key, "Last-Modified": datetime.fromtimestamp(path.stat().st_mtime), @@ -125,11 +128,22 @@ def get_blob_properties(self): "metadata": dict(), } ) + # Set size directly as BlobProperties doesn't accept it in constructor + props.size = path.stat().st_size + return props else: raise ResourceNotFoundError - def download_blob(self): - return MockStorageStreamDownloader(self.root, self.key) + def download_blob(self, offset=None, length=None): + path = self.root / self.key + if not (path.exists() and path.is_file()): + raise ResourceNotFoundError + if offset is not None and offset >= path.stat().st_size: + # real Azure rejects ranges starting past EOF (an end past EOF is clamped) + error = HttpResponseError("The range specified is invalid for the current size") + error.status_code = 416 + raise error + return MockStorageStreamDownloader(self.root, self.key, offset=offset, length=length) def set_blob_metadata(self, metadata): path = self.root / self.key @@ -148,21 +162,55 @@ def delete_blob(self): def upload_blob(self, data, overwrite, content_settings=None): path = self.root / self.key path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data.read()) + if isinstance(data, bytes): + path.write_bytes(data) + else: + path.write_bytes(data.read()) if content_settings is not None: self.service_client.metadata_cache[self.root / self.key] = ( content_settings.content_type ) + def stage_block(self, block_id, data, length): + """Stage a block for block blob upload.""" + # Store the block data indexed by blob key in service client's staged blocks + if self.key not in self.service_client._staged_blocks: + self.service_client._staged_blocks[self.key] = {} + self.service_client._staged_blocks[self.key][block_id] = data + + def commit_block_list(self, block_ids, content_settings=None): + """Commit a list of staged blocks to create a blob.""" + path = self.root / self.key + path.parent.mkdir(parents=True, exist_ok=True) + + # Concatenate blocks in order + if self.key in self.service_client._staged_blocks: + complete_data = b"" + for block_id in block_ids: + complete_data += self.service_client._staged_blocks[self.key][block_id] + + path.write_bytes(complete_data) + + # Clean up staged blocks + del self.service_client._staged_blocks[self.key] + class MockStorageStreamDownloader: - def __init__(self, root, key): + def __init__(self, root, key, offset=None, length=None): self.root = root self.key = key + self.offset = offset + self.length = length def readall(self): - return (self.root / self.key).read_bytes() + data = (self.root / self.key).read_bytes() + if self.offset is not None: + if self.length is not None: + return data[self.offset : self.offset + self.length] + else: + return data[self.offset :] + return data def content_as_bytes(self): return self.readall() diff --git a/tests/mock_clients/mock_gs.py b/tests/mock_clients/mock_gs.py index 26487767..a2d67dc9 100644 --- a/tests/mock_clients/mock_gs.py +++ b/tests/mock_clients/mock_gs.py @@ -4,7 +4,12 @@ import shutil from tempfile import TemporaryDirectory -from google.api_core.exceptions import NotFound +import urllib.parse +from uuid import uuid4 +from xml.etree import ElementTree + +from google.api_core.exceptions import NotFound, RequestRangeNotSatisfiable +import requests from .utils import delete_empty_parents_up_to_root @@ -21,6 +26,8 @@ def __init__(self, *args, **kwargs): shutil.copytree(TEST_ASSETS, self.tmp_path / test_dir) self.metadata_cache = {} + self._connection = _MockConnection() + self._http = MockMPUTransport(self) @classmethod def create_anonymous_client(cls): @@ -72,6 +79,33 @@ def download_to_filename(self, filename, timeout=None, retry=None): to_path.parent.mkdir(exist_ok=True, parents=True) to_path.write_bytes(from_path.read_bytes()) + def download_as_bytes(self, start=None, end=None, timeout=None, retry=None): + """Download blob content as bytes with optional byte range.""" + # if timeout is not None, assume that the test wants a timeout and throw it + if timeout is not None: + raise TimeoutError("Download timed out") + + # indicate that retry object made it through to the GS lib + if retry is not None: + retry.mocked_retries = 1 + + from_path = self.bucket / self.name + if not (from_path.exists() and from_path.is_file()): + raise NotFound(f"blob not found: {self.name}") + data = from_path.read_bytes() + + # real GCS rejects ranges starting past EOF (an end past EOF is clamped) + if start is not None and start >= len(data): + raise RequestRangeNotSatisfiable("The requested range is not satisfiable") + + # Handle byte range if specified + if start is not None: + if end is not None: + return data[start : end + 1] + else: + return data[start:] + return data + def patch(self): if "updated" in self.metadata: (self.bucket / self.name).touch() @@ -89,7 +123,9 @@ def reload( timeout=None, retry=None, ): - pass + path = self.bucket / self.name + if not (path.exists() and path.is_file()): + raise NotFound(f"blob not found: {self.name}") def upload_from_filename(self, filename, content_type=None, timeout=None, retry=None): # if timeout is not None, assume that the test wants a timeout and throw it @@ -107,6 +143,25 @@ def upload_from_filename(self, filename, content_type=None, timeout=None, retry= self.client.metadata_cache[self.bucket / self.name] = content_type + def upload_from_string(self, data, content_type=None, timeout=None, retry=None): + """Upload from bytes/string data.""" + # if timeout is not None, assume that the test wants a timeout and throw it + if timeout is not None: + raise TimeoutError("Upload timed out") + + # indicate that retry object made it through to the GS lib + if retry is not None: + retry.mocked_retries = 1 + + path = self.bucket / self.name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(data, str): + path.write_text(data) + else: + path.write_bytes(data) + + self.client.metadata_cache[self.bucket / self.name] = content_type + @property def etag(self): return "etag" @@ -232,3 +287,86 @@ def download_chunks_concurrently( def mock_default_auth(): return "fake-credentials", "fake-default-project" + + +class _MockConnection: + """Just enough of google.cloud.storage._http.Connection for the XML MPU URL.""" + + API_BASE_URL = "https://storage.googleapis.com" + + +def _mpu_response(status, headers=None, body=b""): + response = requests.Response() + response.status_code = status + response.headers.update(headers or {}) + response._content = body if isinstance(body, bytes) else body.encode() + return response + + +class MockMPUTransport: + """Fake authorized session implementing the GCS XML multipart-upload API.""" + + _XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/" + + def __init__(self, client): + self.client = client + self.uploads = {} + + def request(self, method, url, data=None, headers=None, **kwargs): + parsed = urllib.parse.urlsplit(url) + query = dict(urllib.parse.parse_qsl(parsed.query, keep_blank_values=True)) + # url path is /{bucket}/{blob}; the mock stores blobs under tmp_path directly + _, _, blob = parsed.path.lstrip("/").partition("/") + blob = urllib.parse.unquote(blob) + + if method == "POST" and "uploads" in query: + upload_id = uuid4().hex + self.uploads[upload_id] = { + "blob": blob, + "parts": {}, + "content_type": (headers or {}).get("content-type"), + } + body = ( + f'' + f"{upload_id}" + ) + return _mpu_response(200, body=body) + + upload_id = query.get("uploadId") + if upload_id not in self.uploads: + return _mpu_response(404, body="NoSuchUpload") + + if method == "PUT" and "partNumber" in query: + part_number = int(query["partNumber"]) + etag = f'"mock-etag-{part_number}"' + self.uploads[upload_id]["parts"][part_number] = (etag, bytes(data)) + return _mpu_response(200, headers={"etag": etag}) + + if method == "POST": + upload = self.uploads.pop(upload_id) + root = ElementTree.fromstring(data) + parts = [] + for part_element in root.findall("Part"): + part_number = int(part_element.find("PartNumber").text) + etag = part_element.find("ETag").text + stored_etag, part_data = upload["parts"][part_number] + assert etag == stored_etag, "ETag mismatch in CompleteMultipartUpload" + parts.append(part_data) + # real GCS enforces a 5 MiB minimum for all non-final parts at finalize + if any(len(part_data) < 5 * 1024 * 1024 for part_data in parts[:-1]): + return _mpu_response(400, body="EntityTooSmall") + content = b"".join(parts) + target = self.client.tmp_path / upload["blob"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + if upload["content_type"] is not None: + self.client.metadata_cache[self.client.tmp_path / upload["blob"]] = upload[ + "content_type" + ] + return _mpu_response(200, body="") + + if method == "DELETE": + self.uploads.pop(upload_id, None) + return _mpu_response(204) + + return _mpu_response(400, body="Unsupported mock MPU request") diff --git a/tests/mock_clients/mock_s3.py b/tests/mock_clients/mock_s3.py index a2f850ca..c049fbee 100644 --- a/tests/mock_clients/mock_s3.py +++ b/tests/mock_clients/mock_s3.py @@ -257,6 +257,144 @@ def head_object(self, Bucket, Key, **kwargs): "Metadata": {}, } + def get_object(self, Bucket, Key, Range=None, **kwargs): + """Get an S3 object with optional byte range.""" + if ( + not (self.root / Key).exists() + or (self.root / Key).is_dir() + or Bucket != DEFAULT_S3_BUCKET_NAME + ): + raise ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "The specified key does not exist."}}, + {}, + ) + + path = self.root / Key + data = path.read_bytes() + + if Range: + import re + + match = re.match(r"bytes=(\d+)-(\d+)", Range) + if match: + start, end = int(match.group(1)), int(match.group(2)) + if start >= len(data): + # real S3 rejects ranges starting past EOF (an end past EOF is clamped) + raise ClientError( + { + "Error": { + "Code": "InvalidRange", + "Message": "The requested range is not satisfiable", + } + }, + {}, + ) + data = data[start : end + 1] + else: + raise ClientError( + { + "Error": { + "Code": "InvalidRange", + "Message": "The requested range is not satisfiable", + } + }, + {}, + ) + + from io import BytesIO + + return {"Body": BytesIO(data), "ContentLength": len(data)} + + def create_multipart_upload(self, Bucket, Key, **kwargs): + """Start a multipart upload.""" + import uuid + + upload_id = str(uuid.uuid4()) + if not hasattr(self, "_uploads"): + self._uploads = {} + self._uploads[upload_id] = {"Bucket": Bucket, "Key": Key, "Parts": []} + return {"UploadId": upload_id} + + def upload_part(self, Bucket, Key, UploadId, PartNumber, Body, **kwargs): + """Upload a part in a multipart upload.""" + if not hasattr(self, "_uploads") or UploadId not in self._uploads: + raise ClientError( + { + "Error": { + "Code": "NoSuchUpload", + "Message": "The specified upload does not exist.", + } + }, + {}, + ) + + upload = self._uploads[UploadId] + if isinstance(Body, bytes): + data = Body + else: + data = Body.read() if hasattr(Body, "read") else Body + + upload["Parts"].append({"PartNumber": PartNumber, "Data": data}) + + import hashlib + + etag = hashlib.md5(data).hexdigest() + return {"ETag": etag} + + def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload, **kwargs): + """Complete a multipart upload.""" + if not hasattr(self, "_uploads") or UploadId not in self._uploads: + raise ClientError( + { + "Error": { + "Code": "NoSuchUpload", + "Message": "The specified upload does not exist.", + } + }, + {}, + ) + + upload = self._uploads[UploadId] + parts = sorted(upload["Parts"], key=lambda p: p["PartNumber"]) + complete_data = b"".join([p["Data"] for p in parts]) + + path = self.root / Key + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(complete_data) + + del self._uploads[UploadId] + + return {"Location": f"https://{Bucket}.s3.amazonaws.com/{Key}"} + + def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): + """Abort a multipart upload.""" + if hasattr(self, "_uploads") and UploadId in self._uploads: + del self._uploads[UploadId] + + def put_object(self, Bucket, Key, Body=b"", **kwargs): + """Upload data directly (single PUT).""" + if Bucket != DEFAULT_S3_BUCKET_NAME and ".mrap" not in Bucket: + raise ClientError( + { + "Error": { + "Code": "NoSuchBucket", + "Message": "The specified bucket does not exist.", + } + }, + "PutObject", + ) + path = self.root / Key + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(Body, bytes): + path.write_bytes(Body) + else: + path.write_bytes(Body.read() if hasattr(Body, "read") else b"") + if "ContentType" in kwargs and self.session is not None: + self.session.metadata_cache[path] = kwargs["ContentType"] + import hashlib + + return {"ETag": f'"{hashlib.md5(path.read_bytes()).hexdigest()}"'} + def generate_presigned_url(self, op: str, Params: dict, ExpiresIn: int): mock_presigned_url = f"https://{Params['Bucket']}.s3.amazonaws.com/{Params['Key']}?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=TEST%2FTEST%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240131T194721Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=TEST" return mock_presigned_url diff --git a/tests/test_caching.py b/tests/test_caching.py index 4fce4f6f..108b58c6 100644 --- a/tests/test_caching.py +++ b/tests/test_caching.py @@ -540,3 +540,54 @@ def test_reuse_cache_after_manual_cache_clear(rig: CloudProviderTestRig): _ = f.read() assert cp._local.exists() + + +def test_write_mtime_tie_does_not_raise(rig: CloudProviderTestRig): + """A save that leaves the cache file's mtime exactly equal to the cloud version's + (coarse-resolution filesystems, no-op writes) must bump the mtime and upload rather + than raise OverwriteNewerCloudError.""" + client = rig.client_class(**rig.required_client_kwargs) + cp = rig.create_cloud_path("dir_0/file0_0.txt", client=client) + + cp.write_text("v1") + _sync_filesystem() + + # re-sync the cache from the cloud so the cache file's mtime equals the cloud mtime + cp.clear_cache() + cp.read_text() + cloud_mtime = cp.stat().st_mtime + + with cp.open("w") as f: + f.write("v2") + f.flush() + # simulate a write that leaves the mtime unchanged (e.g. same-second write on a + # coarse-resolution filesystem) + os.utime(cp._local, times=(cloud_mtime, cloud_mtime)) + + assert cp.read_text() == "v2" + + +def test_streaming_append_fallback_cache_cleaned_up(rig: CloudProviderTestRig): + """Append/update modes fall back to the cache in streaming mode; those cache files + must be cleaned up when the client is garbage collected, like other cache modes.""" + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, **rig.required_client_kwargs + ) + cp = rig.create_cloud_path("dir_0/file0_0.txt", client=client) + + with cp.open("a") as f: + f.write("appended") + + # the fallback created a real cache file + assert cp._local.exists() + + cache_path = cp._local + client_cache_dir = client._local_cache_dir + del f # the with-statement target outlives the block and holds the path + del cp + del client + # the patched close handle forms a reference cycle, so collection is not immediate + gc.collect() + + assert not cache_path.exists() + assert not client_cache_dir.exists() diff --git a/tests/test_client.py b/tests/test_client.py index 3eceafc8..8160b246 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -7,11 +7,16 @@ import pytest from cloudpathlib import CloudPath -from cloudpathlib.client import register_client_class -from cloudpathlib.cloudpath import implementation_registry, register_path_class +from cloudpathlib.client import Client, register_client_class +from cloudpathlib.cloudpath import ( + implementation_registry, + register_path_class, + register_raw_io_class, +) from cloudpathlib.http.httpclient import HttpClient, HttpsClient from cloudpathlib.s3.s3client import S3Client from cloudpathlib.s3.s3path import S3Path +from cloudpathlib.s3.s3_io import _S3StorageRaw def test_default_client_instantiation(rig): @@ -140,6 +145,10 @@ class MyS3Path(S3Path): class MyS3Client(S3Client): pass + @register_raw_io_class("mys3") + class MyS3StorageRaw(_S3StorageRaw): + pass + yield (MyS3Path, MyS3Client) # cleanup after use @@ -172,3 +181,22 @@ def test_custom_mys3client_default_client(custom_s3_path): path = CloudPath("mys3://bucket/dir/file.txt") assert isinstance(path.client, CustomClient) assert path.cloud_prefix == "mys3://" + + +@pytest.mark.parametrize( + "call", + [ + lambda client, path: Client._range_download(client, path, 0, 0), + lambda client, path: Client._get_content_length(client, path), + lambda client, path: Client._initiate_multipart_upload(client, path), + lambda client, path: Client._upload_part(client, path, "upload", 1, b"data"), + lambda client, path: Client._complete_multipart_upload(client, path, "upload", []), + lambda client, path: Client._abort_multipart_upload(client, path, "upload"), + lambda client, path: Client._put_empty_object(client, path), + ], +) +def test_default_streaming_hooks_raise_not_implemented(local_s3_rig, call): + path = local_s3_rig.create_cloud_path("unsupported-stream.bin") + + with pytest.raises(NotImplementedError, match="streaming I/O"): + call(path.client, path) diff --git a/tests/test_cloud_io.py b/tests/test_cloud_io.py new file mode 100644 index 00000000..6bea2f5b --- /dev/null +++ b/tests/test_cloud_io.py @@ -0,0 +1,2772 @@ +""" +Tests for cloud storage streaming I/O. + +Tests CloudBufferedIO, CloudTextIO, and streaming mode for direct +streaming without local caching. +""" + +import io +import threading +import time +import zipfile +import pytest + +from cloudpathlib import S3Path, AzureBlobPath, GSPath +from cloudpathlib import CloudBufferedIO, CloudTextIO +from cloudpathlib.cloud_io import _CloudStorageRaw +from cloudpathlib.enums import FileCacheMode +from cloudpathlib.exceptions import ( + CloudPathFileNotFoundError, + CloudPathNotImplementedError, + OverwriteNewerCloudError, +) + +# Sample test data +BINARY_DATA = b"Hello, World! This is binary data.\n" * 100 +TEXT_DATA = "Hello, World! This is text data.\n" * 100 +MULTILINE_TEXT = """Line 1 +Line 2 +Line 3 +Line 4 with special chars: éñ中文 +""" + + +@pytest.fixture +def temp_cloud_file(rig): + """Create a temporary cloud file for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_io.txt") + path.write_text(TEXT_DATA) + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +@pytest.fixture +def temp_cloud_binary_file(rig): + """Create a temporary cloud binary file for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_io.bin") + path.write_bytes(BINARY_DATA) + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +@pytest.fixture +def temp_cloud_multiline_file(rig): + """Create a temporary cloud file with multiple lines for testing.""" + # Skip if streaming IO is not implemented for this provider + # HTTP/HTTPS support streaming reads and the test server supports writes + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_streaming_multiline.txt") + path.write_text(MULTILINE_TEXT, encoding="utf-8") + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + yield path + # Restore original mode + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# CloudBufferedIO tests (binary streaming) +# ============================================================================ + + +def test_read_binary_stream(temp_cloud_binary_file): + """Test reading binary data via streaming.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Verify it's the right type + assert isinstance(f, CloudBufferedIO) + assert isinstance(f, io.BufferedIOBase) + + # Read all data + data = f.read() + assert data == BINARY_DATA + + +def test_read_chunks(temp_cloud_binary_file): + """Test reading data in chunks.""" + chunk_size = 100 + with temp_cloud_binary_file.open(mode="rb") as f: + chunks = [] + while True: + chunk = f.read(chunk_size) + if not chunk: + break + chunks.append(chunk) + assert len(chunk) <= chunk_size + + # Verify we got all data + assert b"".join(chunks) == BINARY_DATA + + +def test_read1(temp_cloud_binary_file): + """Test read1 method.""" + with temp_cloud_binary_file.open(mode="rb") as f: + chunk = f.read1(50) + assert len(chunk) <= 50 + assert len(chunk) > 0 + + +def test_readinto(temp_cloud_binary_file): + """Test readinto method.""" + with temp_cloud_binary_file.open(mode="rb") as f: + buf = bytearray(100) + n = f.readinto(buf) + assert n > 0 + assert n <= 100 + assert buf[:n] == BINARY_DATA[:n] + + +def test_seek_tell(temp_cloud_binary_file): + """Test seek and tell operations.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Initial position + assert f.tell() == 0 + + # Read some data + f.read(50) + assert f.tell() == 50 + + # Seek to beginning + pos = f.seek(0) + assert pos == 0 + assert f.tell() == 0 + + # Seek relative + pos = f.seek(10, io.SEEK_CUR) + assert pos == 10 + + # Seek from end + pos = f.seek(-10, io.SEEK_END) + assert pos == len(BINARY_DATA) - 10 + + +def test_seekable_readable_writable(temp_cloud_binary_file): + """Test capability flags.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert f.readable() + assert not f.writable() + assert f.seekable() + + +def test_buffered_io_properties(temp_cloud_binary_file): + """Test file properties.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert f.name == str(temp_cloud_binary_file) + assert f.mode == "rb" + assert not f.closed + + assert f.closed + + +def test_buffered_io_context_manager(temp_cloud_binary_file): + """Test context manager protocol.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert not f.closed + data = f.read(10) + assert len(data) == 10 + + assert f.closed + + +def test_write_binary_stream(rig): + """Test writing binary data via streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_binary.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Write data + with path.open(mode="wb") as f: + assert isinstance(f, CloudBufferedIO) + assert f.writable() + assert not f.readable() + + n = f.write(BINARY_DATA) + assert n == len(BINARY_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify data was written + assert path.exists() + assert path.read_bytes() == BINARY_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_write_chunks(rig): + """Test writing data in chunks.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_chunks.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + chunk_size = 100 + with path.open(mode="wb", buffer_size=chunk_size) as f: + for i in range(0, len(BINARY_DATA), chunk_size): + chunk = BINARY_DATA[i : i + chunk_size] + f.write(chunk) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify + assert path.read_bytes() == BINARY_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_flush(rig): + """Test explicit flush.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_flush.bin") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + f.write(b"First chunk") + f.flush() + f.write(b" Second chunk") + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_bytes() == b"First chunk Second chunk" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_buffered_io_isinstance_checks(temp_cloud_binary_file): + """Test that instances pass isinstance checks.""" + with temp_cloud_binary_file.open(mode="rb") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.BufferedIOBase) + assert not isinstance(f, io.TextIOBase) + + +def test_not_found_error(rig): + """Test error when file doesn't exist.""" + path = rig.create_cloud_path("nonexistent.bin") + + with pytest.raises(FileNotFoundError): + with path.open(mode="rb") as f: + f.read() + + +# ============================================================================ +# CloudTextIO tests (text streaming) +# ============================================================================ + + +def test_read_text_stream(temp_cloud_file): + """Test reading text data via streaming.""" + with temp_cloud_file.open(mode="rt") as f: + # Verify it's the right type + assert isinstance(f, CloudTextIO) + assert isinstance(f, io.TextIOBase) + + # Read all data + data = f.read() + assert data == TEXT_DATA + + +def test_read_text_mode_without_t(temp_cloud_file): + """Test reading text with mode 'r' (without explicit 't').""" + with temp_cloud_file.open(mode="r") as f: + assert isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + + +def test_readline(temp_cloud_multiline_file): + """Test readline method.""" + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: + line1 = f.readline() + assert line1 == "Line 1\n" + + line2 = f.readline() + assert line2 == "Line 2\n" + + +def test_readlines(temp_cloud_multiline_file): + """Test readlines method.""" + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: + lines = f.readlines() + assert len(lines) == 4 + assert lines[0] == "Line 1\n" + assert "special chars" in lines[3] + + +def test_iteration(temp_cloud_multiline_file): + """Test iterating over lines.""" + with temp_cloud_multiline_file.open(mode="rt", encoding="utf-8") as f: + lines = list(f) + assert len(lines) == 4 + assert lines[0] == "Line 1\n" + + +def test_encoding(rig): + """Test different encodings.""" + path = rig.create_cloud_path("test_encoding.txt") + utf8_text = "Hello 世界 🌍" + + try: + # Write with UTF-8 + path.write_text(utf8_text, encoding="utf-8") + + # Read with UTF-8 + with path.open(mode="rt", encoding="utf-8") as f: + assert f.encoding == "utf-8" + data = f.read() + assert data == utf8_text + finally: + try: + path.unlink() + except Exception: + pass + + +def test_text_properties(temp_cloud_file): + """Test text mode properties.""" + with temp_cloud_file.open(mode="rt", encoding="utf-8", errors="strict") as f: + assert f.encoding == "utf-8" + assert f.errors == "strict" + assert f.name == str(temp_cloud_file) + assert "r" in f.mode + + +def test_write_text_stream(rig): + """Test writing text data via streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_text.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt") as f: + assert isinstance(f, CloudTextIO) + n = f.write(TEXT_DATA) + assert n == len(TEXT_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + # Verify + assert path.read_text() == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_writelines(rig): + """Test writelines method.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_writelines.txt") + lines = ["Line 1\n", "Line 2\n", "Line 3\n"] + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt") as f: + f.writelines(lines) + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_text() == "".join(lines) + finally: + try: + path.unlink() + except Exception: + pass + + +def test_text_io_isinstance_checks(temp_cloud_file): + """Test that instances pass isinstance checks.""" + with temp_cloud_file.open(mode="rt") as f: + assert isinstance(f, io.IOBase) + assert isinstance(f, io.TextIOBase) + assert not isinstance(f, io.BufferedIOBase) + + +def test_buffer_property(temp_cloud_file): + """Test access to underlying binary buffer.""" + with temp_cloud_file.open(mode="rt") as f: + assert hasattr(f, "buffer") + assert isinstance(f.buffer, CloudBufferedIO) + + +# ============================================================================ +# CloudPath.open streaming integration tests +# ============================================================================ + + +def test_cloudpath_stream_read(temp_cloud_file): + """Test CloudPath.open with streaming mode for reading.""" + # The temp_cloud_file fixture already sets streaming mode + with temp_cloud_file.open(mode="r") as f: + assert isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + + +def test_cloudpath_stream_write(rig): + """Test CloudPath.open with streaming mode for writing.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_stream_write.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="w") as f: + assert isinstance(f, CloudTextIO) + f.write(TEXT_DATA) + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.read_text() == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_cloudpath_stream_binary(temp_cloud_binary_file): + """Test CloudPath.open with streaming mode for binary.""" + # The temp_cloud_binary_file fixture already sets streaming mode + with temp_cloud_binary_file.open(mode="rb") as f: + assert isinstance(f, CloudBufferedIO) + data = f.read() + assert data == BINARY_DATA + + +def test_cloudpath_stream_false_uses_cache(rig): + """Test that non-streaming mode uses traditional caching.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_caching.txt") + path.write_text(TEXT_DATA) + + try: + # Default mode (not streaming) should use caching + assert path.client.file_cache_mode != FileCacheMode.streaming + + with path.open(mode="r") as f: + # Should not be a CloudTextIO instance + assert not isinstance(f, CloudTextIO) + # Should still read correctly + data = f.read() + assert data == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +def test_cloudpath_default_no_streaming(rig): + """Test that default behavior uses caching, not streaming.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_default.txt") + path.write_text(TEXT_DATA) + + try: + # Default client mode should not be streaming + assert path.client.file_cache_mode != FileCacheMode.streaming + + with path.open(mode="r") as f: + # Default should not use streaming + assert not isinstance(f, CloudTextIO) + data = f.read() + assert data == TEXT_DATA + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# CloudPath.open factory tests +# ============================================================================ + + +def test_auto_client_s3(temp_cloud_file): + """Test auto-detection of S3 client.""" + if not isinstance(temp_cloud_file, S3Path): + pytest.skip("Not testing S3") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_auto_client_azure(temp_cloud_file): + """Test auto-detection of Azure client.""" + if not isinstance(temp_cloud_file, AzureBlobPath): + pytest.skip("Not testing Azure") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_auto_client_gs(temp_cloud_file): + """Test auto-detection of GCS client.""" + if not isinstance(temp_cloud_file, GSPath): + pytest.skip("Not testing GCS") + + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_explicit_client(temp_cloud_file): + """Test passing explicit client.""" + with temp_cloud_file.open(mode="rt") as f: + data = f.read() + assert len(data) > 0 + + +def test_buffer_size_parameter(temp_cloud_binary_file): + """Test custom buffer size.""" + buffer_size = 1024 + with temp_cloud_binary_file.open(mode="rb", buffer_size=buffer_size) as f: + assert f._buffer_size == buffer_size + + +def test_text_parameters(rig): + """Test text-specific parameters.""" + path = rig.create_cloud_path("test_params.txt") + text = "Test data" + + try: + path.write_text(text) + + with path.open(mode="rt", encoding="utf-8", errors="strict", newline=None) as f: + assert f.encoding == "utf-8" + assert f.errors == "strict" + data = f.read() + assert data == text + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Edge cases and error conditions +# ============================================================================ + + +def test_empty_file_read(rig): + """Test reading an empty file.""" + path = rig.create_cloud_path("test_empty.txt") + + try: + path.write_text("") + + with path.open(mode="rt") as f: + data = f.read() + assert data == "" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_empty_file_write(rig): + """Test writing an empty file.""" + # Skip if streaming IO is not implemented for this provider + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_empty_write.txt") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wt"): + pass # Write nothing + + # Restore original mode + path.client.file_cache_mode = original_mode + + assert path.exists() + assert path.read_text() == "" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_large_file_streaming(rig): + """Test streaming a larger file.""" + path = rig.create_cloud_path("test_large.bin") + # 1 MB of data + large_data = b"X" * (1024 * 1024) + + try: + path.write_bytes(large_data) + + # Read in chunks + with path.open(mode="rb", buffer_size=8192) as f: + chunks = [] + while True: + chunk = f.read(8192) + if not chunk: + break + chunks.append(chunk) + + result = b"".join(chunks) + assert len(result) == len(large_data) + assert result == large_data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_seek_beyond_eof(temp_cloud_binary_file): + """Test seeking beyond end of file.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek beyond EOF + size = len(BINARY_DATA) + pos = f.seek(size + 1000) + assert pos == size + 1000 + + # Reading should return empty + data = f.read(10) + assert data == b"" + + +def test_closed_file_operations(temp_cloud_file): + """Test operations on closed file raise errors.""" + with temp_cloud_file.open(mode="rt") as f: + pass # Just open and close + + # Now f is closed + with pytest.raises(ValueError): + f.read() + + with pytest.raises(ValueError): + f.readline() + + +def test_binary_mode_required_for_buffered(temp_cloud_file): + """Test that CloudBufferedIO requires binary mode.""" + # Get the raw IO class + raw_io_class = temp_cloud_file._cloud_meta.raw_io_class + if raw_io_class is None: + pytest.skip("No raw IO class registered") + + # This should raise an error + with pytest.raises(ValueError, match="binary mode"): + CloudBufferedIO( + raw_io_class=raw_io_class, + client=temp_cloud_file.client, + cloud_path=temp_cloud_file, + mode="r", + ) + + +def test_text_mode_required_for_text(temp_cloud_file): + """Test that CloudTextIO requires text mode.""" + # Get the raw IO class + raw_io_class = temp_cloud_file._cloud_meta.raw_io_class + if raw_io_class is None: + pytest.skip("No raw IO class registered") + + with pytest.raises(ValueError, match="text mode"): + CloudTextIO( + raw_io_class=raw_io_class, + client=temp_cloud_file.client, + cloud_path=temp_cloud_file, + mode="rb", + ) + + +def test_unsupported_operations(temp_cloud_file): + """Test unsupported operations raise appropriate errors.""" + with temp_cloud_file.open(mode="rt") as f: + # fileno() should raise + with pytest.raises(OSError): + f.fileno() + + # isatty() should return False + assert not f.isatty() + + +def test_read_write_mode_not_implemented(temp_cloud_file): + """Test that read/write modes work as expected.""" + # For now, r+ and w+ may have limitations + # Test basic write mode + with temp_cloud_file.open(mode="wt") as f: + assert f.writable() + assert not f.readable() + + +# ============================================================================ +# Provider-specific tests +# ============================================================================ + + +def test_s3_multipart_upload(rig): + """Test that S3 multipart upload is triggered for large writes.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("Not testing S3") + + path = rig.create_cloud_path("test_multipart.bin") + # Write enough data to trigger multiple parts (> 64KB buffer) + large_data = b"X" * (200 * 1024) # 200 KB + + try: + with path.open(mode="wb", buffer_size=64 * 1024) as f: + f.write(large_data) + + # Verify data was uploaded correctly + assert path.read_bytes() == large_data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_azure_block_upload(rig): + """Test that Azure block upload works.""" + if rig.path_class.cloud_prefix != "az://": + pytest.skip("Not testing Azure") + + path = rig.create_cloud_path("test_blocks.bin") + data = b"Block data " * 1000 + + try: + with path.open(mode="wb") as f: + f.write(data) + + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_gs_multipart_streaming_upload(rig): + """Test that GCS streaming upload works (XML multipart under the hood).""" + if rig.path_class.cloud_prefix != "gs://": + pytest.skip("Not testing GCS") + + path = rig.create_cloud_path("test_resumable.bin") + data = b"GCS data " * 1000 + + try: + with path.open(mode="wb") as f: + f.write(data) + + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Performance and efficiency tests +# ============================================================================ + + +def test_small_buffer_many_reads(temp_cloud_binary_file): + """Test reading with small buffer size.""" + with temp_cloud_binary_file.open(mode="rb", buffer_size=128) as f: + data = f.read() + assert data == BINARY_DATA + + +def test_large_buffer_few_reads(temp_cloud_binary_file): + """Test reading with large buffer size.""" + with temp_cloud_binary_file.open(mode="rb", buffer_size=1024 * 1024) as f: + data = f.read() + assert data == BINARY_DATA + + +def test_sequential_reads(temp_cloud_binary_file): + """Test sequential reading pattern.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pos = 0 + while pos < len(BINARY_DATA): + chunk = f.read(100) + if not chunk: + break + assert chunk == BINARY_DATA[pos : pos + 100] + pos += len(chunk) + + +def test_random_seeks(temp_cloud_binary_file): + """Test random seek pattern.""" + positions = [0, 100, 50, 200, 10] + + with temp_cloud_binary_file.open(mode="rb") as f: + for pos in positions: + f.seek(pos) + assert f.tell() == pos + chunk = f.read(10) + assert chunk == BINARY_DATA[pos : pos + 10] + + +# ============================================================================ +# Additional coverage tests for error paths and edge cases +# ============================================================================ + + +def test_readinto_on_closed_file(temp_cloud_binary_file): + """Test readinto on closed file raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pass + + buf = bytearray(100) + with pytest.raises(ValueError, match="closed file"): + f.readinto(buf) + + +def test_read_on_write_only_file(rig): + """Test reading from write-only file raises error.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_write_only.bin") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + # Try to read from write-only file + with pytest.raises(io.UnsupportedOperation): + f.read() + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_readinto_empty_buffer(temp_cloud_binary_file): + """Test readinto with empty buffer returns 0.""" + with temp_cloud_binary_file.open(mode="rb") as f: + buf = bytearray(0) + n = f.readinto(buf) + assert n == 0 + + +def test_seek_with_invalid_whence(temp_cloud_binary_file): + """Test seek with invalid whence raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + with pytest.raises((ValueError, OSError)): + f.seek(0, 999) # Invalid whence value + + +def test_negative_seek_position(temp_cloud_binary_file): + """Test seeking to negative position raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + with pytest.raises(ValueError, match="negative seek position"): + f.seek(-10, io.SEEK_SET) + + +def test_seek_on_closed_file(temp_cloud_binary_file): + """Test seek on closed file raises ValueError.""" + with temp_cloud_binary_file.open(mode="rb") as f: + pass + + with pytest.raises(ValueError, match="closed file"): + f.seek(0) + + +def test_write_empty_chunks(rig): + """Test that empty write chunks are handled correctly.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_empty_chunks.bin") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open(mode="wb") as f: + # Write empty data - should be no-op + f.write(b"") + # Write actual data + f.write(b"real data") + # Write more empty data + f.write(b"") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"real data" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_write_error_cleanup(rig): + """A failed part upload must abort rather than commit earlier parts.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific failure injection") + + path = rig.create_cloud_path("test_error_cleanup.bin") + original_upload_part = path.client._upload_part + original_abort = path.client._abort_multipart_upload + abort_calls = [] + + def fail_second_part(cloud_path, upload_id, part_number, data): + if part_number == 2: + raise RuntimeError("simulated part failure") + return original_upload_part(cloud_path, upload_id, part_number, data) + + def record_abort(cloud_path, upload_id): + abort_calls.append(upload_id) + return original_abort(cloud_path, upload_id) + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + path.client._upload_part = fail_second_part + path.client._abort_multipart_upload = record_abort + + stream = path.open(mode="wb") + with pytest.raises(RuntimeError, match="simulated part failure"): + stream.write(b"x" * (11 * 1024 * 1024)) + with pytest.raises(RuntimeError, match="simulated part failure"): + stream.close() + + assert len(abort_calls) == 1 + assert not path.exists() + finally: + path.client._upload_part = original_upload_part + path.client._abort_multipart_upload = original_abort + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +def test_http_write_empty_file(rig): + """Test HTTP write for empty file.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("Test is specific to HTTP/HTTPS") + + path = rig.create_cloud_path("test_http_empty.bin") + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("wb"): + pass + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"" + finally: + path.client.file_cache_mode = original_mode + path.unlink(missing_ok=True) + + +def test_seek_from_end_without_size(rig, monkeypatch): + """Test SEEK_END when size cannot be determined.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://", "http://", "https://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_no_size.bin") + path.write_bytes(b"test data") + + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Monkeypatch _get_size to raise an error + def mock_get_size(): + raise OSError("Cannot determine size") + + with path.open(mode="rb") as f: + # CloudBufferedIO has a _buffer attribute that wraps the raw IO + # Access the raw IO object through _buffer + raw = f._buffer.raw if hasattr(f, "_buffer") else f.raw + monkeypatch.setattr(raw, "_get_size", mock_get_size) + monkeypatch.setattr(raw, "_size", None) + + # Try to seek from end - should raise error (either from mock or from handler) + with pytest.raises(OSError): + f.seek(-5, io.SEEK_END) + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_read_at_eof_returns_empty(temp_cloud_binary_file): + """Test that reading at EOF returns empty bytes.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek to end + f.seek(0, io.SEEK_END) + # Try to read + data = f.read(100) + assert data == b"" + + +def test_readinto_at_eof_returns_zero(temp_cloud_binary_file): + """Test that readinto at EOF returns 0.""" + with temp_cloud_binary_file.open(mode="rb") as f: + # Seek to end + f.seek(0, io.SEEK_END) + # Try to readinto + buf = bytearray(100) + n = f.readinto(buf) + assert n == 0 + + +def test_fspath_raises_in_streaming_mode(rig): + """Test that fspath raises an error in streaming mode.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_fspath.txt") + path.write_text("test data") + + try: + # Set client to streaming mode + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + # Try to access fspath - should raise error + from cloudpathlib.exceptions import CloudPathNotImplementedError + + with pytest.raises( + CloudPathNotImplementedError, match="fspath is not available in streaming mode" + ): + _ = path.fspath + + # Also test __fspath__ directly + with pytest.raises( + CloudPathNotImplementedError, match="fspath is not available in streaming mode" + ): + _ = path.__fspath__() + + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# Step 8 regression tests — one test per bug from the plan +# ============================================================================ + + +# H1 — finalize-error propagates (no silent data loss) +def test_finalize_error_propagates(rig): + """A failed upload must raise out of the with-block; silent data loss is not allowed.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_finalize_error.bin") + raw_io_class = path._cloud_meta.raw_io_class + + class _FailingRaw(raw_io_class): + def _finalize_upload(self) -> None: + raise RuntimeError("simulated upload failure") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(RuntimeError, match="simulated upload failure"): + with CloudBufferedIO( + raw_io_class=_FailingRaw, + client=path.client, + cloud_path=path, + mode="wb", + ) as f: + f.write(b"data that should not survive") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H2 — exclusive create raises for 'xb' and 'xt' when object already exists +def test_exclusive_create_xb_raises_when_exists(rig): + """open('xb') must raise CloudPathFileExistsError when the object already exists.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = rig.create_cloud_path("test_exclusive_create.bin") + path.write_bytes(b"existing content") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(CloudPathFileExistsError): + path.open("xb") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +def test_exclusive_create_xt_raises_when_exists(rig): + """open('xt') must also raise CloudPathFileExistsError (mode='x' alone was not enough).""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = rig.create_cloud_path("test_exclusive_create_xt.txt") + path.write_text("existing") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with pytest.raises(CloudPathFileExistsError): + path.open("xt") + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H2 — append/r+ fall back to cache (correct semantics over streaming) +def test_append_mode_uses_cache_fallback(rig): + """Append mode with streaming file_cache_mode must fall back to the cached path.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_append_fallback.bin") + path.write_bytes(b"hello ") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("ab") as f: + assert not isinstance(f, CloudBufferedIO), "append mode must use cache, not streaming" + f.write(b"world") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"hello world" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +def test_append_mode_creates_missing_file(local_s3_rig): + path = local_s3_rig.create_cloud_path("new-append.txt") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("a") as stream: + stream.write("created") + + path.client.file_cache_mode = FileCacheMode.cloudpath_object + assert path.read_text() == "created" + + +def test_rplus_mode_uses_cache_fallback(rig): + """r+b mode with streaming file_cache_mode must fall back to the cached path.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path = rig.create_cloud_path("test_rplus_fallback.bin") + path.write_bytes(b"hello world") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("r+b") as f: + assert not isinstance(f, CloudBufferedIO), "r+b must use cache, not streaming" + f.seek(6) + f.write(b"there") + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == b"hello there" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H3 — S3 part-size floor: no non-final part < 5 MiB +def test_s3_no_small_non_final_parts(rig): + """All non-final S3 multipart parts must be >= 5 MiB.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific test") + + from cloudpathlib.s3.s3_io import _S3StorageRaw + + path = rig.create_cloud_path("test_part_size.bin") + data = b"X" * (12 * 1024 * 1024) # 12 MiB → two 5 MiB parts + one 2 MiB final + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + uploaded_parts = [] + real_upload_part = path.client._upload_part + + def spy_upload_part(cloud_path, upload_id, part_number, part_data): + uploaded_parts.append(len(part_data)) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + path.client._upload_part = spy_upload_part + + try: + with path.open("wb") as f: + f.write(data) + + path.client.file_cache_mode = original_mode + assert path.read_bytes() == data + + min_size = _S3StorageRaw._MIN_PART_SIZE + for part_size in uploaded_parts[:-1]: # all except last + assert ( + part_size >= min_size + ), f"Non-final part is {part_size} bytes, below 5 MiB minimum" + finally: + path.client._upload_part = real_upload_part + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H3b — S3 aborts multipart upload when complete fails +def test_s3_abort_multipart_on_complete_failure(rig): + """If _complete_multipart_upload fails, _abort_multipart_upload is attempted.""" + if rig.path_class.cloud_prefix != "s3://": + pytest.skip("S3-specific test") + + path = rig.create_cloud_path("test_abort_on_complete_fail.bin") + data = b"X" * (6 * 1024 * 1024) # 6 MiB → one 5 MiB part + final part + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + real_complete = path.client._complete_multipart_upload + real_abort = path.client._abort_multipart_upload + abort_calls = [] + + def fail_complete(cloud_path, upload_id, parts): + raise RuntimeError("complete failed") + + def spy_abort(cloud_path, upload_id): + abort_calls.append(upload_id) + return real_abort(cloud_path, upload_id) + + path.client._complete_multipart_upload = fail_complete + path.client._abort_multipart_upload = spy_abort + + try: + with pytest.raises(RuntimeError, match="complete failed"): + with path.open("wb") as f: + f.write(data) + + assert len(abort_calls) == 1 + assert not path.exists() + finally: + path.client._complete_multipart_upload = real_complete + path.client._abort_multipart_upload = real_abort + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +# H4 — concurrent writes to two paths on one client don't cross buffers +def test_concurrent_writes_dont_cross_buffers(rig): + """Two simultaneous streaming writers must not share upload state.""" + if rig.path_class.cloud_prefix not in ("s3://", "az://", "gs://"): + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + path_a = rig.create_cloud_path("test_concurrent_a.bin") + path_b = rig.create_cloud_path("test_concurrent_b.bin") + data_a = b"AAAA" * 1024 + data_b = b"BBBB" * 1024 + errors = [] + + def write_path(path, data): + try: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + with path.open("wb") as f: + f.write(data) + path.client.file_cache_mode = original_mode + except Exception as e: + errors.append(e) + + t_a = threading.Thread(target=write_path, args=(path_a, data_a)) + t_b = threading.Thread(target=write_path, args=(path_b, data_b)) + t_a.start() + t_b.start() + t_a.join() + t_b.join() + + assert not errors, f"Concurrent write errors: {errors}" + + try: + assert path_a.read_bytes() == data_a, "path_a has wrong data (buffer cross)" + assert path_b.read_bytes() == data_b, "path_b has wrong data (buffer cross)" + finally: + for p in (path_a, path_b): + try: + p.unlink() + except Exception: + pass + + +# M6/M7 — custom Client without raw_io_class still instantiates in cached mode +def test_custom_client_without_raw_io_class_instantiates(local_s3_rig, monkeypatch): + """A cached custom provider need not implement the optional streaming hooks.""" + from cloudpathlib.cloudpath import CloudImplementation + + minimal = CloudImplementation() + minimal.name = "minimal" + minimal._client_class = local_s3_rig.client_class + minimal._path_class = local_s3_rig.path_class + minimal._raw_io_class = None + monkeypatch.setattr(local_s3_rig.path_class, "_cloud_meta", minimal) + + path = local_s3_rig.create_cloud_path("no-raw-io.txt") + path.write_text("cached") + assert path.read_text() == "cached" + + path.client.file_cache_mode = FileCacheMode.streaming + with pytest.raises(CloudPathNotImplementedError, match="Streaming I/O is not implemented"): + path.open("r") + + +# M5 — HTTP range reads return the correct slice +def test_http_range_read_returns_correct_bytes(rig): + """HTTP range reads must return exactly the requested byte slice.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("HTTP/HTTPS-specific test") + + path = rig.create_cloud_path("test_range_slice.bin") + content = b"0123456789abcdef" + path.write_bytes(content) + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("rb", buffer_size=4) as f: + chunk = f.read(4) + assert chunk == b"0123", f"Expected first 4 bytes, got {chunk!r}" + chunk2 = f.read(4) + assert chunk2 == b"4567", f"Expected bytes 4-7, got {chunk2!r}" + finally: + path.client.file_cache_mode = original_mode + try: + path.unlink() + except Exception: + pass + + +@pytest.mark.parametrize("mode", ["", "rw", "rr", "r++", "rbt", "q"]) +def test_streaming_open_rejects_invalid_modes_without_mutating(local_s3_rig, mode): + path = local_s3_rig.create_cloud_path("invalid-mode.txt") + path.write_text("preserve me") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError): + path.open(mode) + + path.client.file_cache_mode = FileCacheMode.cloudpath_object + assert path.read_text() == "preserve me" + + +def test_streaming_open_rejects_non_string_mode(local_s3_rig): + path = local_s3_rig.create_cloud_path("invalid-mode.txt") + + with pytest.raises(TypeError, match="mode must be a string"): + path.open(None) + + +@pytest.mark.parametrize( + "keyword,value,message", + [ + ("encoding", "utf-8", "encoding"), + ("errors", "ignore", "errors"), + ("newline", "", "newline"), + ], +) +def test_streaming_binary_mode_rejects_text_arguments(local_s3_rig, keyword, value, message): + path = local_s3_rig.create_cloud_path("binary-arguments.bin") + path.write_bytes(b"data") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError, match=message): + path.open("rb", **{keyword: value}) + + +def test_streaming_text_mode_rejects_unbuffered_io(local_s3_rig): + path = local_s3_rig.create_cloud_path("unbuffered.txt") + path.write_text("data") + path.client.file_cache_mode = FileCacheMode.streaming + + with pytest.raises(ValueError, match="unbuffered text"): + path.open("r", buffering=0) + + +def test_streaming_text_mode_honors_line_buffering(local_s3_rig): + path = local_s3_rig.create_cloud_path("line-buffered.txt") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("w", buffering=1) as stream: + assert stream.line_buffering + stream.write("line\n") + + +def test_streaming_binary_mode_supports_unbuffered_io(local_s3_rig): + path = local_s3_rig.create_cloud_path("unbuffered.bin") + path.write_bytes(b"data") + path.client.file_cache_mode = FileCacheMode.streaming + + with path.open("rb", buffering=0) as stream: + assert isinstance(stream, io.RawIOBase) + assert stream.read() == b"data" + + +def test_s3_streaming_routes_extra_args_by_operation(s3_rig): + if s3_rig.live_server: + pytest.skip("Synthetic SDK argument-routing test") + + path = s3_rig.create_cloud_path("streaming-extra-args.bin") + client = path.client + original_extra_args = client.boto3_ul_extra_args + original_create = client.client.create_multipart_upload + original_upload = client.client.upload_part + original_complete = client.client.complete_multipart_upload + calls = {} + + def record_create(**kwargs): + calls["create"] = kwargs.copy() + return original_create(**kwargs) + + def record_upload(**kwargs): + calls["upload"] = kwargs.copy() + return original_upload(**kwargs) + + def record_complete(**kwargs): + calls["complete"] = kwargs.copy() + return original_complete(**kwargs) + + client.boto3_ul_extra_args = { + "ChecksumCRC32": "whole-object-checksum", + "SSECustomerAlgorithm": "AES256", + "SSECustomerKey": "secret", + } + client.client.create_multipart_upload = record_create + client.client.upload_part = record_upload + client.client.complete_multipart_upload = record_complete + client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("wb") as stream: + stream.write(b"x" * (6 * 1024 * 1024)) + + assert "ChecksumCRC32" not in calls["create"] + assert calls["create"]["SSECustomerKey"] == "secret" + assert calls["upload"]["SSECustomerKey"] == "secret" + assert calls["complete"]["ChecksumCRC32"] == "whole-object-checksum" + finally: + client.boto3_ul_extra_args = original_extra_args + client.client.create_multipart_upload = original_create + client.client.upload_part = original_upload + client.client.complete_multipart_upload = original_complete + path.unlink(missing_ok=True) + + +def test_s3_streaming_ignores_automatic_part_checksums(s3_rig, monkeypatch): + path = s3_rig.create_cloud_path("automatic-checksum.bin") + client = path.client + original_extra_args = client.boto3_ul_extra_args + + monkeypatch.setattr( + client.client, + "upload_part", + lambda **kwargs: {"ETag": '"etag"', "ChecksumCRC32": "checksum"}, + ) + try: + client.boto3_ul_extra_args = {} + part = client._upload_part(path, "upload", 1, b"data") + assert part == {"PartNumber": 1, "ETag": '"etag"'} + + client.boto3_ul_extra_args = {"ChecksumAlgorithm": "CRC32"} + part = client._upload_part(path, "upload", 1, b"data") + assert part["ChecksumCRC32"] == "checksum" + finally: + client.boto3_ul_extra_args = original_extra_args + + +def test_gs_streaming_range_is_inclusive_and_forwards_options(gs_rig, monkeypatch): + if gs_rig.live_server: + pytest.skip("Synthetic SDK option-forwarding test") + + from tests.mock_clients.mock_gs import MockBlob + + path = gs_rig.create_cloud_path("range-options.bin") + path.write_bytes(b"0123456789") + calls = {} + + def record_download(self, start=None, end=None, **kwargs): + calls.update(start=start, end=end, **kwargs) + return b"2345" + + monkeypatch.setattr(MockBlob, "download_as_bytes", record_download) + retry = object() + original_kwargs = path.client.blob_kwargs + path.client.blob_kwargs = {"timeout": 12, "retry": retry} + try: + assert path.client._range_download(path, 2, 5) == b"2345" + assert calls == {"start": 2, "end": 5, "timeout": 12, "retry": retry} + finally: + path.client.blob_kwargs = original_kwargs + + +def test_http_streaming_rejects_servers_that_ignore_ranges(http_rig, monkeypatch): + class FullResponse(io.BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def ignore_range(request): + assert request.headers["Range"] == "bytes=2-5" + return FullResponse(b"0123456789") + + path = http_rig.create_cloud_path("ignored-range.bin") + monkeypatch.setattr(path.client.opener, "open", ignore_range) + + with pytest.raises(OSError, match="ignored the Range header"): + path.client._range_download(path, 2, 5) + + +def test_http_streaming_upload_uses_client_configuration(http_rig, monkeypatch): + class CreatedResponse: + status = 201 + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + calls = {} + + def record_request(request): + calls["request"] = request + calls["body"] = request.data.read() + return CreatedResponse() + + path = http_rig.create_cloud_path("configured.txt") + path.client.write_file_http_method = "PATCH" + monkeypatch.setattr(path.client.opener, "open", record_request) + path.client._put_data(path, io.BytesIO(b"abc"), 3) + + request = calls["request"] + assert request.method == "PATCH" + assert request.headers["Content-type"] == "text/plain" + assert request.headers["Content-length"] == "3" + assert calls["body"] == b"abc" + + +def test_provider_part_sizes_grow_for_large_streams(): + from cloudpathlib.azure.azure_io import _AzureBlobStorageRaw + from cloudpathlib.s3.s3_io import _S3StorageRaw + + assert ( + _S3StorageRaw._part_size_for_number(_S3StorageRaw._PARTS_PER_SIZE_TIER + 1) + == 2 * _S3StorageRaw._MIN_PART_SIZE + ) + + assert ( + _AzureBlobStorageRaw._block_size_for_number(_AzureBlobStorageRaw._BLOCKS_PER_SIZE_TIER + 1) + == 2 * _AzureBlobStorageRaw._BLOCK_SIZE + ) + + +def test_s3_invalid_object_state_is_not_eof(s3_rig): + path = s3_rig.create_cloud_path("archived.bin") + raw = path._cloud_meta.raw_io_class(path.client, path, "rb") + assert not raw._is_eof_error(Exception("InvalidObjectState")) + + +def test_raw_read_errors_follow_file_object_semantics(local_s3_rig): + class Raw(_CloudStorageRaw): + def _upload_chunk(self, data): + pass + + def _finalize_upload(self): + pass + + path = local_s3_rig.create_cloud_path("raw-errors.bin") + + unreadable = Raw(path.client, path, "wb") + with pytest.raises(io.UnsupportedOperation, match="not readable"): + unreadable.readinto(bytearray(1)) + + unwritable = Raw(path.client, path, "rb") + with pytest.raises(io.UnsupportedOperation, match="not writable"): + unwritable.write(b"x") + + unwritable.close() + for operation in ( + lambda: unwritable.readinto(bytearray(1)), + lambda: unwritable.seek(0), + unwritable.tell, + lambda: unwritable.write(b"x"), + ): + with pytest.raises(ValueError, match="closed file"): + operation() + + +def test_raw_read_handles_unknown_size_and_provider_eof(local_s3_rig): + class Raw(_CloudStorageRaw): + eof = False + empty = False + + def _get_size(self): + raise OSError("size unavailable") + + def _range_get(self, start, end): + if self.empty: + return b"" + raise OSError("range unavailable") + + def _is_eof_error(self, error): + return self.eof + + def _upload_chunk(self, data): + pass + + def _finalize_upload(self): + pass + + path = local_s3_rig.create_cloud_path("raw-eof.bin") + raw = Raw(path.client, path, "rb") + + with pytest.raises(OSError, match="range unavailable"): + raw.readinto(bytearray(1)) + + raw.eof = True + assert raw.readinto(bytearray(1)) == 0 + + raw.eof = False + raw.empty = True + assert raw.readinto(bytearray(1)) == 0 + + +def test_raw_write_failure_is_sticky_and_aborts_on_close(local_s3_rig): + error = OSError("upload failed") + + class Raw(_CloudStorageRaw): + aborted = False + + def _upload_chunk(self, data): + raise error + + def _finalize_upload(self): + pass + + def _abort_upload(self): + self.aborted = True + + path = local_s3_rig.create_cloud_path("raw-upload-error.bin") + raw = Raw(path.client, path, "wb") + + with pytest.raises(OSError, match="upload failed"): + raw.write(b"first") + with pytest.raises(OSError, match="upload failed"): + raw.write(b"second") + with pytest.raises(OSError, match="upload failed"): + raw.close() + + assert raw.aborted + assert raw.closed + + +@pytest.mark.parametrize("fail_during_write", [True, False]) +def test_raw_abort_failure_does_not_mask_original_error(local_s3_rig, fail_during_write): + original_error = OSError("original failure") + + class Raw(_CloudStorageRaw): + def _upload_chunk(self, data): + if fail_during_write: + raise original_error + + def _finalize_upload(self): + if not fail_during_write: + raise original_error + + def _abort_upload(self): + raise OSError("cleanup failure") + + path = local_s3_rig.create_cloud_path("raw-abort-error.bin") + raw = Raw(path.client, path, "wb") + + if fail_during_write: + with pytest.raises(OSError, match="original failure"): + raw.write(b"data") + else: + raw.write(b"data") + + with pytest.raises(OSError, match="original failure"): + raw.close() + + +# ============================================================================ +# Regression tests — PR #535 review fixes +# ============================================================================ + +_STREAMING_PREFIXES = ("s3://", "az://", "gs://", "http://", "https://") + + +def _skip_if_no_streaming(rig): + if rig.path_class.cloud_prefix not in _STREAMING_PREFIXES: + pytest.skip(f"Streaming I/O not implemented for {rig.path_class.cloud_prefix}") + + +@pytest.fixture +def streaming_rig(rig): + """The rig with its default client switched to streaming mode for the test.""" + _skip_if_no_streaming(rig) + client = rig.client_class._default_client + original_mode = client.file_cache_mode + client.file_cache_mode = FileCacheMode.streaming + yield rig + client.file_cache_mode = original_mode + + +def test_write_tell_tracks_position(streaming_rig): + """tell() on streaming write streams must report total bytes written, not just + the bytes pending in the buffer (write() previously never advanced the raw position).""" + path = streaming_rig.create_cloud_path("test_write_tell.bin") + + try: + # small buffer so most bytes reach the raw layer instead of sitting in the buffer + with path.open("wb", buffer_size=64 * 1024) as f: + assert f.tell() == 0 + f.write(b"x" * 200_000) # larger than the buffer + assert f.tell() == 200_000 + f.write(b"y" * 100) # small write held in the buffer + assert f.tell() == 200_100 + assert path.read_bytes() == b"x" * 200_000 + b"y" * 100 + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_zipfile_write_roundtrip(streaming_rig): + """Position-dependent writers like zipfile rely on tell(); a streaming write + stream must produce a valid archive.""" + path = streaming_rig.create_cloud_path("test_streaming_archive.zip") + big_member = b"data" * 50_000 # > the 64 KiB buffer below so bytes reach the raw layer + + try: + with path.open("wb", buffer_size=64 * 1024) as f: + with zipfile.ZipFile(f, "w") as zf: + zf.writestr("a.txt", b"hello world") + zf.writestr("b.bin", big_member) + + with zipfile.ZipFile(io.BytesIO(path.read_bytes())) as zf: + assert zf.testzip() is None + assert zf.read("a.txt") == b"hello world" + assert zf.read("b.bin") == big_member + finally: + try: + path.unlink() + except Exception: + pass + + +def test_raw_write_stream_rejects_seek(streaming_rig): + """seek() on a write-only raw stream must raise io.UnsupportedOperation instead of + silently succeeding while writes keep appending.""" + path = streaming_rig.create_cloud_path("test_raw_seek_write.bin") + + try: + with path.open("wb", buffering=0) as f: + assert not f.seekable() + f.write(b"data") + with pytest.raises(io.UnsupportedOperation): + f.seek(0) + assert path.read_bytes() == b"data" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_full_read_uses_single_range_request(streaming_rig, monkeypatch): + """read() to EOF must fetch the remaining bytes in one ranged request (readall), + not fall back to one request per 8 KiB default chunk.""" + rig = streaming_rig + content = bytes(range(256)) * 2048 # 512 KiB + + # write in cached mode, then stream the read + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_readall.bin") + path.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + calls = [] + original_range_download = type(client)._range_download + + def counting_range_download(self, cloud_path, start, end): + calls.append((start, end)) + return original_range_download(self, cloud_path, start, end) + + monkeypatch.setattr(type(client), "_range_download", counting_range_download) + + try: + with path.open("rb") as f: + data = f.read() + assert data == content + assert len(calls) <= 2, f"expected a single ranged request, got {calls}" + finally: + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_streaming_size_fetch_failure_is_memoized(streaming_rig, monkeypatch): + """A failing content-length lookup must be attempted at most once per stream, + not re-issued before every chunk read.""" + rig = streaming_rig + content = b"z" * (200 * 1024) + + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_size_memo.bin") + path.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + calls = {"n": 0} + + def failing_get_content_length(self, cloud_path): + calls["n"] += 1 + raise OSError("no size available") + + monkeypatch.setattr(type(client), "_get_content_length", failing_get_content_length) + + try: + with path.open("rb", buffer_size=16 * 1024) as f: + data = f.read() + assert data == content + assert calls["n"] == 1 + finally: + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_gs_range_download_transient_error_not_treated_as_eof(gs_rig, monkeypatch): + """Errors that merely contain '416' in their message (request IDs, generation + numbers) must propagate; only true 416 range errors read as EOF.""" + path = gs_rig.create_cloud_path("test_416_matching.bin") + + class FakeServiceUnavailable(Exception): + code = 503 + + class FakeRangeError(Exception): + code = 416 + + def make_stub(error): + class StubBlob: + def download_as_bytes(self, start=None, end=None, **kwargs): + raise error + + class StubBucket: + def blob(self, name): + return StubBlob() + + return lambda name: StubBucket() + + # transient error whose message contains "416" must raise, not return EOF + monkeypatch.setattr( + path.client.client, + "bucket", + make_stub(FakeServiceUnavailable("503 GET /o/file?generation=1234164 backend error")), + ) + with pytest.raises(FakeServiceUnavailable): + path.client._range_download(path, 0, 9) + + # structured 416 still reads as EOF + monkeypatch.setattr( + path.client.client, "bucket", make_stub(FakeRangeError("range not satisfiable")) + ) + assert path.client._range_download(path, 0, 9) == b"" + + # exact reason phrase still reads as EOF (some layers do not expose a code) + monkeypatch.setattr( + path.client.client, "bucket", make_stub(Exception("Requested Range Not Satisfiable")) + ) + assert path.client._range_download(path, 0, 9) == b"" + + +def test_azure_block_ids_namespaced_per_upload(azure_rig): + """Concurrent streaming writers to the same blob must stage blocks under distinct + IDs so they cannot clobber each other's uncommitted blocks.""" + path = azure_rig.create_cloud_path("test_block_ids.bin") + + upload_a = path.client._initiate_multipart_upload(path) + upload_b = path.client._initiate_multipart_upload(path) + assert upload_a and upload_b and upload_a != upload_b + + try: + part_a = path.client._upload_part(path, upload_a, 1, b"A" * 16) + part_b = path.client._upload_part(path, upload_b, 1, b"B" * 16) + assert part_a["block_id"] != part_b["block_id"] + + # committing B yields exactly B's data even though A staged the same part number + path.client._complete_multipart_upload(path, upload_b, [part_b]) + assert path.read_bytes() == b"B" * 16 + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_cross_client_copy(streaming_rig): + """copy() between paths on different client instances must work in streaming mode + (the cached implementation round-trips through fspath, which streaming forbids).""" + rig = streaming_rig + content = b"copy me" * 100 + + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + src = rig.create_cloud_path("test_copy_src.bin") + src.write_bytes(content) + client.file_cache_mode = FileCacheMode.streaming + + other_client = rig.client_class(**rig.required_client_kwargs) + dst = other_client.CloudPath(str(rig.create_cloud_path("test_copy_dst.bin"))) + assert src.client is not dst.client + + try: + result = src.copy(dst) + assert result.read_bytes() == content + finally: + for p in (src, dst): + try: + p.unlink() + except Exception: + pass + + +def test_http_rename_in_streaming_mode(rig): + """rename()/replace() on HTTP paths must work in streaming mode without fspath.""" + if rig.path_class.cloud_prefix not in ("http://", "https://"): + pytest.skip("HTTP/HTTPS-specific test") + + path = rig.create_cloud_path("test_rename_src.bin") + path.write_bytes(b"move me") + target = rig.create_cloud_path("test_rename_dst.bin") + + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + result = path.rename(target) + assert result.read_bytes() == b"move me" + assert not path.exists() + finally: + path.client.file_cache_mode = original_mode + for p in (path, target): + try: + p.unlink() + except Exception: + pass + + +def test_streaming_write_conflict_raises(streaming_rig): + """A streaming write with force_overwrite_to_cloud=False must not clobber a + version uploaded while the stream was open.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_conflict.bin") + path.write_bytes(b"original") + + try: + f = path.open("wb", force_overwrite_to_cloud=False) + f.write(b"mine") + + # a concurrent writer replaces the object with a strictly newer version + time.sleep(1.1) # some providers report modification times in whole seconds + path.write_bytes(b"concurrent") + + with pytest.raises(OverwriteNewerCloudError): + f.close() + + assert path.read_bytes() == b"concurrent" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_write_conflict_force_overwrites(streaming_rig): + """force_overwrite_to_cloud=True skips the conflict check and wins.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_conflict_force.bin") + path.write_bytes(b"original") + + try: + f = path.open("wb", force_overwrite_to_cloud=True) + f.write(b"mine") + path.write_bytes(b"concurrent") + f.close() + + assert path.read_bytes() == b"mine" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_write_overwrite_unchanged_cloud_succeeds(streaming_rig): + """Overwriting an object that did not change while the stream was open is a + normal write and must not raise.""" + rig = streaming_rig + path = rig.create_cloud_path("test_stream_overwrite_ok.bin") + path.write_bytes(b"v1") + + try: + with path.open("wb", force_overwrite_to_cloud=False) as f: + f.write(b"v2") + assert path.read_bytes() == b"v2" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_negative_buffering_accepted(rig): + """Any negative buffering value means 'use the default', matching builtins.open.""" + path = rig.create_cloud_path("test_neg_buffering.txt") + + try: + # cached mode passes buffering through to the local filesystem open + with path.open("w", buffering=-2) as f: + f.write("cached") + assert path.read_text() == "cached" + + if rig.path_class.cloud_prefix in _STREAMING_PREFIXES: + original_mode = path.client.file_cache_mode + path.client.file_cache_mode = FileCacheMode.streaming + try: + with path.open("rb", buffering=-2) as f: + assert f.read() == b"cached" + finally: + path.client.file_cache_mode = original_mode + finally: + try: + path.unlink() + except Exception: + pass + + +def test_streaming_not_found_errors_are_cloudpathlib_exceptions(rig): + """Streaming hooks must raise cloudpathlib's exception types so callers catching + CloudPathException (or CloudPathFileNotFoundError) see streaming errors too.""" + _skip_if_no_streaming(rig) + missing = rig.create_cloud_path("definitely_missing_for_streaming.bin") + + with pytest.raises(CloudPathFileNotFoundError): + missing.client._range_download(missing, 0, 9) + + with pytest.raises(CloudPathFileNotFoundError): + missing.client._get_content_length(missing) + + +# ============================================================================ +# Coverage-gap tests — raw-stream edge semantics, provider EOF mapping, guards +# ============================================================================ + + +def test_raw_stream_edge_semantics(local_s3_rig): + """Raw adapter edge cases follow file-object semantics.""" + path = local_s3_rig.create_cloud_path("raw-edges.bin") + path.write_bytes(b"0123456789") + raw = path._cloud_meta.raw_io_class(path.client, path, "rb") + + # empty destination buffer reads zero bytes + assert raw.readinto(bytearray(0)) == 0 + + # relative seek and whence validation at the raw layer + raw.seek(4) + assert raw.seek(2, io.SEEK_CUR) == 6 + with pytest.raises(ValueError, match="invalid whence"): + raw.seek(0, 42) + + # readall from a mid-stream position, then again at EOF + assert raw.readall() == b"6789" + assert raw.readall() == b"" + raw.close() + + # write-only streams cannot readall; closed streams cannot readall + writer = path._cloud_meta.raw_io_class(path.client, path, "wb") + with pytest.raises(io.UnsupportedOperation): + writer.readall() + writer.write(b"replaced") + writer.close() + writer.close() # double close is a no-op + with pytest.raises(ValueError, match="closed file"): + writer.readall() + assert path.read_bytes() == b"replaced" + + +def test_multipart_part_limit_enforced(local_s3_rig): + """Exceeding the provider's maximum part count raises a clear OSError.""" + path = local_s3_rig.create_cloud_path("part-limit.bin") + + class TinyParts(path._cloud_meta.raw_io_class): + _INITIAL_PART_SIZE = 4 + _MAX_PART_SIZE = 4 + _MAX_PARTS = 2 + _PARTS_PER_SIZE_TIER = 1_000 + + raw = TinyParts(path.client, path, "wb") + raw.write(b"x" * 8) # exactly two full parts — at the limit + with pytest.raises(OSError, match="part limit"): + raw.write(b"x" * 4) + with pytest.raises(OSError, match="part limit"): + raw.close() # the write failure is sticky and aborts the upload + + +def test_buffered_io_direct_construction_guards(local_s3_rig): + """Direct construction validates modes that the open() path never forwards.""" + path = local_s3_rig.create_cloud_path("direct-construction.bin") + path.write_bytes(b"0123456789") + raw_cls = path._cloud_meta.raw_io_class + + with pytest.raises(io.UnsupportedOperation, match="append and update"): + CloudBufferedIO(raw_cls, path.client, path, mode="ab") + with pytest.raises(io.UnsupportedOperation, match="append and update"): + CloudTextIO(raw_cls, path.client, path, mode="a") + + # readinto1 delegates to the buffered reader + with CloudBufferedIO(raw_cls, path.client, path, mode="rb") as f: + buf = bytearray(4) + assert f.readinto1(buf) == 4 + assert bytes(buf) == b"0123" + + +def test_streaming_text_exclusive_create(streaming_rig): + """mode='x' in text form creates a new object via streaming and rejects existing ones.""" + from cloudpathlib.exceptions import CloudPathFileExistsError + + path = streaming_rig.create_cloud_path("test_x_create.txt") + try: + with path.open(mode="x") as f: + f.write("created") + assert path.read_text() == "created" + with pytest.raises(CloudPathFileExistsError): + path.open(mode="x") + finally: + try: + path.unlink() + except Exception: + pass + + +def test_range_download_past_eof_returns_empty(streaming_rig): + """A range starting past EOF maps to EOF (empty bytes) on every provider.""" + path = streaming_rig.create_cloud_path("past_eof.bin") + path.write_bytes(b"0123456789") + try: + assert path.client._range_download(path, 100, 199) == b"" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_raw_empty_write_is_noop(streaming_rig): + """Writing b'' at the raw layer uploads nothing but still finalizes correctly.""" + path = streaming_rig.create_cloud_path("empty_chunk.bin") + try: + with path.open("wb", buffering=0) as f: + assert f.write(b"") == 0 + f.write(b"payload") + assert f.write(b"") == 0 + assert path.read_bytes() == b"payload" + finally: + try: + path.unlink() + except Exception: + pass + + +def test_s3_streaming_extra_args_uses_service_model(s3_rig): + """When the boto3 client exposes a service model, allowed params come from it, + and unknown params are dropped.""" + from types import SimpleNamespace + + client = s3_rig.client_class(**s3_rig.required_client_kwargs) + client.boto3_ul_extra_args = {"StorageClass": "STANDARD_IA", "NotARealParam": "x"} + + shape = SimpleNamespace(members={"StorageClass": None}) + operation_model = SimpleNamespace(input_shape=shape) + service_model = SimpleNamespace(operation_model=lambda name: operation_model) + client.client = SimpleNamespace(meta=SimpleNamespace(service_model=service_model)) + + assert client._streaming_extra_args("CreateMultipartUpload") == {"StorageClass": "STANDARD_IA"} + + +def test_s3_streaming_extra_args_fallback_matches_botocore(s3_rig): + """The hard-coded fallback table must filter identically to botocore's real + service model, so it cannot silently drop newly added parameters.""" + from types import SimpleNamespace + + botocore_session = pytest.importorskip("botocore.session") + service_model = botocore_session.get_session().get_service_model("s3") + + for operation in ( + "CreateMultipartUpload", + "UploadPart", + "CompleteMultipartUpload", + "PutObject", + ): + members = set(service_model.operation_model(operation).input_shape.members) + extra_args = {name: "value" for name in sorted(members)} + + fallback_client = s3_rig.client_class(**s3_rig.required_client_kwargs) + fallback_client.boto3_ul_extra_args = extra_args + fallback_client.client = SimpleNamespace() # no .meta -> fallback table + + real_client = s3_rig.client_class(**s3_rig.required_client_kwargs) + real_client.boto3_ul_extra_args = extra_args + real_client.client = SimpleNamespace(meta=SimpleNamespace(service_model=service_model)) + + assert fallback_client._streaming_extra_args( + operation + ) == real_client._streaming_extra_args( + operation + ), f"fallback table diverges from botocore for {operation}" + + +def test_s3_streaming_content_encoding_threaded(s3_rig): + """Encodings from content_type_method are added to streaming upload args.""" + import mimetypes + + client = s3_rig.client_class( + content_type_method=mimetypes.guess_type, **s3_rig.required_client_kwargs + ) + path = s3_rig.create_cloud_path("encoded.txt.gz", client=client) + + args = client._streaming_object_args("CreateMultipartUpload", path) + assert args.get("ContentType") == "text/plain" + assert args.get("ContentEncoding") == "gzip" + + +def test_azure_streaming_content_settings_branches(azure_rig): + """Content settings resolve for absent, empty, and populated content type methods.""" + client_none = azure_rig.client_class( + content_type_method=None, **azure_rig.required_client_kwargs + ) + path = azure_rig.create_cloud_path("content-settings.bin", client=client_none) + assert client_none._streaming_content_settings(path) is None + + client_empty = azure_rig.client_class( + content_type_method=lambda name: (None, None), **azure_rig.required_client_kwargs + ) + assert client_empty._streaming_content_settings(path) is None + + client_full = azure_rig.client_class( + content_type_method=lambda name: ("text/plain", "gzip"), + **azure_rig.required_client_kwargs, + ) + settings = client_full._streaming_content_settings(path) + assert settings.content_type == "text/plain" + assert settings.content_encoding == "gzip" + + # abort is a documented no-op: uncommitted blocks simply expire server-side + client_none._abort_multipart_upload(path, "upload-id") + + +def test_gs_multipart_upload_hooks(gs_rig): + """GS streaming writes use the XML multipart API: unique upload IDs, ordered + assembly of size-compliant parts, and cancellable uploads.""" + client = gs_rig.client_class(**gs_rig.required_client_kwargs) + path = gs_rig.create_cloud_path("mpu-hooks.bin", client=client) + + upload_a = client._initiate_multipart_upload(path) + upload_b = client._initiate_multipart_upload(path) + assert upload_a and upload_b and upload_a != upload_b + + head = b"A" * (5 * 1024 * 1024) # non-final parts must be at least 5 MiB + tail = b"B" * 16 + part_1 = client._upload_part(path, upload_a, 1, head) + part_2 = client._upload_part(path, upload_a, 2, tail) + client._complete_multipart_upload(path, upload_a, [part_1, part_2]) + client._abort_multipart_upload(path, upload_b) + + try: + assert path.read_bytes() == head + tail + finally: + try: + path.unlink() + except Exception: + pass + + +def test_gs_mpu_initiate_threads_content_type_and_encoding(gs_rig): + """Initiate carries the content type and encoding from content_type_method.""" + from types import SimpleNamespace + + client = gs_rig.client_class( + content_type_method=lambda name: ("text/plain", "gzip"), + **gs_rig.required_client_kwargs, + ) + path = gs_rig.create_cloud_path("mpu-headers.txt.gz", client=client) + + captured = {} + + class StubTransport: + def request(self, method, url, data=None, headers=None, **kwargs): + import requests + + captured["method"] = method + captured["url"] = url + captured["headers"] = {key.lower(): value for key, value in (headers or {}).items()} + response = requests.Response() + response.status_code = 200 + response._content = ( + b'' + b"stub-upload" + ) + return response + + client.client = SimpleNamespace( + _connection=SimpleNamespace(API_BASE_URL="https://storage.googleapis.com"), + _http=StubTransport(), + ) + + upload_id = client._initiate_multipart_upload(path) + assert upload_id == "stub-upload" + assert captured["method"] == "POST" + assert captured["url"].endswith("?uploads") + assert captured["headers"]["content-type"] == "text/plain" + assert captured["headers"]["content-encoding"] == "gzip" + + +def test_http_streaming_error_paths(http_rig, monkeypatch): + """HTTP: unexpected range status, missing Content-Length, failing PUT status, + and unsupported PUT method all map to clear errors.""" + import urllib.error + + path = http_rig.create_cloud_path("http_errors.bin") + path.write_bytes(b"0123456789") + + class FakeResponse: + def __init__(self, status, headers=None): + self.status = status + self.headers = headers if headers is not None else {} + self.reason = "stub" + + def read(self, *args): + return b"" + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + # a non-206/200 success status for a range request is unexpected + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(204)) + with pytest.raises(OSError, match="Unexpected status"): + path.client._range_download(path, 0, 3) + + # HEAD without Content-Length cannot size the stream + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(200)) + with pytest.raises(ValueError, match="Content-Length"): + path.client._get_content_length(path) + + # a failing PUT status raises OSError + monkeypatch.setattr(path.client.opener, "open", lambda req: FakeResponse(500)) + with pytest.raises(OSError, match="HTTP PUT failed"): + path.client._put_data(path, io.BytesIO(b"x"), 1) + + # non-404 HTTP errors propagate from reads and size checks + def raise_403(req): + raise urllib.error.HTTPError("url", 403, "Forbidden", {}, None) + + monkeypatch.setattr(path.client.opener, "open", raise_403) + with pytest.raises(urllib.error.HTTPError): + path.client._range_download(path, 0, 3) + with pytest.raises(urllib.error.HTTPError): + path.client._get_content_length(path) + + # servers that reject the write method surface CloudPathNotImplementedError + def raise_405(req): + raise urllib.error.HTTPError("url", 405, "Method Not Allowed", {}, None) + + monkeypatch.setattr(path.client.opener, "open", raise_405) + with pytest.raises(CloudPathNotImplementedError): + path.client._put_data(path, io.BytesIO(b"x"), 1) + + monkeypatch.undo() + try: + path.unlink() + except Exception: + pass + + +def test_open_buffer_size_must_be_positive(streaming_rig): + """buffer_size=0 is rejected up front.""" + path = streaming_rig.create_cloud_path("bad_buffer.bin") + with pytest.raises(ValueError, match="buffer_size"): + path.open("wb", buffer_size=0) + + +def test_open_directory_raises(local_s3_rig): + """Opening a directory raises CloudPathIsADirectoryError in any cache mode.""" + from cloudpathlib.exceptions import CloudPathIsADirectoryError + + file_path = local_s3_rig.create_cloud_path("adir/inner.txt") + file_path.write_text("x") + dir_path = local_s3_rig.create_cloud_path("adir") + + with pytest.raises(CloudPathIsADirectoryError): + dir_path.open("rb") + + +def test_streaming_parquet_metadata_and_column_read(streaming_rig): + """Parquet readers work over a seekable streaming stream: the footer and a + single column can be read without downloading the whole object.""" + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + + rig = streaming_rig + client = rig.client_class._default_client + client.file_cache_mode = FileCacheMode.tmp_dir + path = rig.create_cloud_path("test_columns.parquet") + table = pa.table({"a": list(range(10_000)), "b": ["x" * 20] * 10_000}) + sink = io.BytesIO() + pq.write_table(table, sink) + path.write_bytes(sink.getvalue()) + client.file_cache_mode = FileCacheMode.streaming + + try: + with path.open("rb", buffer_size=64 * 1024) as f: + parquet_file = pq.ParquetFile(f) + assert parquet_file.metadata.num_rows == 10_000 + column = parquet_file.read(columns=["a"]) + assert column.column("a").to_pylist()[:3] == [0, 1, 2] + finally: + try: + path.unlink() + except Exception: + pass + + +# ============================================================================ +# streaming_max_concurrency — concurrent part uploads and read prefetch +# ============================================================================ + + +def test_streaming_max_concurrency_validation(local_s3_rig): + """The concurrency knob must be a positive integer.""" + with pytest.raises(ValueError, match="streaming_max_concurrency"): + local_s3_rig.client_class( + streaming_max_concurrency=0, **local_s3_rig.required_client_kwargs + ) + + +def test_concurrent_multipart_write_correctness(streaming_rig): + """A multi-part streaming write with concurrency > 1 produces identical content, + even when an early part finishes after later ones.""" + from cloudpathlib.cloud_io import _CloudMultipartStorageRaw + + rig = streaming_rig + if not issubclass(rig.raw_io_class, _CloudMultipartStorageRaw): + pytest.skip("provider does not use multipart streaming writes") + + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=4, + **rig.required_client_kwargs, + ) + path = rig.create_cloud_path("test_concurrent_parts.bin", client=client) + data = bytes(range(256)) * (48 * 1024) # 12 MiB -> two 5 MiB parts + final part + + part_numbers = [] + real_upload_part = client._upload_part + + def delaying_upload_part(cloud_path, upload_id, part_number, part_data): + if part_number == 1: + time.sleep(0.2) # force part 1 to finish after later parts + part_numbers.append(part_number) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + client._upload_part = delaying_upload_part + + try: + with path.open("wb") as f: + f.write(data) + assert sorted(part_numbers) == list(range(1, len(part_numbers) + 1)) + assert len(part_numbers) >= 2 + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_concurrent_multipart_write_parallelism_observed(local_s3_rig): + """With concurrency 2, two part uploads genuinely run at the same time.""" + path = local_s3_rig.create_cloud_path("test_parallel_parts.bin") + client = local_s3_rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=2, + **local_s3_rig.required_client_kwargs, + ) + path = local_s3_rig.create_cloud_path("test_parallel_parts.bin", client=client) + + barrier = threading.Barrier(2, timeout=30) + overlapped = [] + real_upload_part = client._upload_part + + def rendezvous_upload_part(cloud_path, upload_id, part_number, part_data): + if part_number <= 2: + barrier.wait() # only passes if both uploads are in flight simultaneously + overlapped.append(part_number) + return real_upload_part(cloud_path, upload_id, part_number, part_data) + + client._upload_part = rendezvous_upload_part + + data = b"Z" * (12 * 1024 * 1024) # two 5 MiB parts + final part + try: + with path.open("wb") as f: + f.write(data) + assert sorted(overlapped) == [1, 2] + assert path.read_bytes() == data + finally: + try: + path.unlink() + except Exception: + pass + + +def test_concurrent_multipart_write_failure_is_sticky_and_aborts(local_s3_rig): + """A failing background part upload surfaces on a later write/close and aborts.""" + client = local_s3_rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=2, + **local_s3_rig.required_client_kwargs, + ) + path = local_s3_rig.create_cloud_path("test_failing_part.bin", client=client) + + aborted = [] + real_abort = client._abort_multipart_upload + + def spy_abort(cloud_path, upload_id): + aborted.append(upload_id) + return real_abort(cloud_path, upload_id) + + def failing_upload_part(cloud_path, upload_id, part_number, part_data): + raise OSError("part upload failed") + + client._upload_part = failing_upload_part + client._abort_multipart_upload = spy_abort + + f = path.open("wb") + with pytest.raises(OSError, match="part upload failed"): + # keep writing until the background failure is harvested + for _ in range(10): + f.write(b"Q" * (6 * 1024 * 1024)) + f.close() + with pytest.raises(OSError, match="part upload failed"): + f.close() + + assert aborted, "failed upload was not aborted" + assert not path.exists() + + +def test_read_prefetch_correctness_and_no_wasted_requests(streaming_rig): + """Sequential reads with prefetch fetch each byte range exactly once and + return identical data; seeking invalidates the prefetch window correctly.""" + rig = streaming_rig + client = rig.client_class( + file_cache_mode=FileCacheMode.streaming, + streaming_max_concurrency=3, + **rig.required_client_kwargs, + ) + path = rig.create_cloud_path("test_prefetch.bin", client=client) + chunk = 128 * 1024 + data = bytes(range(256)) * (4 * 1024) # 1 MiB -> 8 chunks + + path.write_bytes(data) + + calls = [] + real_range_download = client._range_download + + def counting_range_download(cloud_path, start, end): + calls.append((start, end)) + return real_range_download(cloud_path, start, end) + + client._range_download = counting_range_download + + try: + with path.open("rb", buffer_size=chunk) as f: + read_back = b"" + while True: + piece = f.read1(chunk) + if not piece: + break + read_back = read_back + piece + assert read_back == data + starts = sorted(start for start, _ in calls) + assert starts == list(range(0, len(data), chunk)), f"unexpected requests: {calls}" + + # seeking back re-reads correctly even though prefetched chunks are discarded + with path.open("rb", buffer_size=chunk) as f: + f.read1(chunk) + f.seek(3 * chunk) + assert f.read1(chunk) == data[3 * chunk : 4 * chunk] + f.seek(0) + assert f.read1(chunk) == data[:chunk] + finally: + try: + path.unlink() + except Exception: + pass diff --git a/tests/test_cloudpath_instantiation.py b/tests/test_cloudpath_instantiation.py index bbdfa3c3..7a9826d1 100644 --- a/tests/test_cloudpath_instantiation.py +++ b/tests/test_cloudpath_instantiation.py @@ -103,14 +103,14 @@ def test_idempotency(rig): def test_dependencies_not_loaded(rig, monkeypatch): - monkeypatch.setattr(rig.path_class._cloud_meta, "dependencies_loaded", False) + monkeypatch.setattr(rig.cloud_implementation, "dependencies_loaded", False) with pytest.raises(MissingDependenciesError): CloudPath(f"{rig.cloud_prefix}{rig.drive}/{rig.test_dir}/dir_0/file0_0.txt") with pytest.raises(MissingDependenciesError): rig.create_cloud_path("dir_0/file0_0.txt") # manual reset for teardown order so teardown doesn't fail - monkeypatch.setattr(rig.path_class._cloud_meta, "dependencies_loaded", True) + monkeypatch.setattr(rig.cloud_implementation, "dependencies_loaded", True) def test_is_pathlike(rig):