diff --git a/packages/zarr-metadata/changes/4365.bugfix.md b/packages/zarr-metadata/changes/4365.bugfix.md new file mode 100644 index 0000000000..6cda30f2d2 --- /dev/null +++ b/packages/zarr-metadata/changes/4365.bugfix.md @@ -0,0 +1 @@ +The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys ["SHOULD NOT be present ... and SHOULD be ignored"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), unlike `.zgroup`'s ["MUST NOT"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313)), and `filters: []` is accepted (["a list of JSON objects providing codec configurations, or null"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79) sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index e9792d6931..51ec633b82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -38,7 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] -ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): @@ -74,8 +74,14 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] -class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): - """Schema input for the closed, merged v2 array representation.""" +class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for the merged v2 array representation. + + Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT + be present ... and SHOULD be ignored by implementations" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92); the group document's "MUST NOT" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313) keeps + `ZarrV2GroupMetadataJSON` closed. + """ zarr_format: Literal[2] shape: tuple[NonNegativeInt, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 0b562bc188..db708c0152 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -51,7 +51,7 @@ class ZarrV3NamedConfig: Bare names and missing configurations normalize to an empty configuration. Bare names and missing `must_understand` members normalize to the spec's - implicit `True` value. + implicit `True` value (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573). """ name: str @@ -107,8 +107,8 @@ def must_understand_subset( ) -> dict[str, ZarrV3ExtensionField]: """The subset of `extra_fields` the reader is obligated to understand. - Per the v3 spec, an extension field is implicitly `must_understand: True` - unless it explicitly says otherwise, and an implementation MUST fail to + Per the v3 spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), an extension field is implicitly `must_understand: + True` unless it explicitly says otherwise, and an implementation MUST fail to open a group or array carrying fields it does not recognize that are not explicitly `must_understand: false`. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the @@ -310,7 +310,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the array if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -428,7 +428,8 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zarray` content: a conforming `.zarray` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model: every value # that can hold a mutable container is deep-copied. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 63dfe5611f..bb2e14da1b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -168,7 +168,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the group if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -299,7 +299,8 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zgroup` content: a conforming `.zgroup` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313; https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model. out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a12e1911b1..d1cd74b719 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -584,10 +584,12 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) + # Unlike the group document ("Other keys MUST NOT be present", + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be + # present within the metadata object and SHOULD be ignored by + # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so members outside + # ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) - problems.extend( - _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) - ) problems.extend(_check_literal(doc, "zarr_format", 2)) shape_problems = _validate_dim_sequence(doc, "shape") chunks_problems = _validate_dim_sequence(doc, "chunks") @@ -642,10 +644,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) ) elif filters is not None: - if len(cast("Sequence[object]", filters)) == 0: - problems.append( - ValidationProblem(("filters",), "expected at least one filter", "invalid_value") - ) + # "A list of JSON objects providing codec configurations, or + # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index e026e5c655..7e4f87be7c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -81,8 +81,8 @@ class ZarrV2ArrayMetadataJSON(TypedDict): """ Zarr v2 array metadata document, in-memory merged form. - Models the union of `.zarray` (the spec-defined fields) and `.zattrs` - (user attributes). On disk, attributes live in a sibling `.zattrs` file + Models the union of `.zarray` (the spec-defined fields, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L51-L92) + and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk, attributes live in a sibling `.zattrs` file and are not part of `.zarray`; this type folds them in as the `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a @@ -126,8 +126,10 @@ class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Note: v2 array metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + Note: v2 array metadata has no `extra_items` setting: the v2 spec has no + extension-field concept, and other `.zarray` keys "SHOULD be ignored by + implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so nothing beyond the spec-defined + fields is modeled. Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 68785d1660..74e9fe938f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -15,6 +15,7 @@ Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 34d72742c2..3a645dc29b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -16,9 +16,10 @@ class ZarrV2ZGroupJSON(TypedDict): On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for - a v2 group. The spec defines exactly one field. User attributes live - in a sibling `.zattrs` file and are NOT part of this type; see - `ZarrV2ZAttrsJSON`. + a v2 group. The spec defines exactly one field and forbids others. User + attributes live in a sibling `.zattrs` file and are NOT part of this + type; see `ZarrV2ZAttrsJSON`. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313 See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -30,8 +31,8 @@ class ZarrV2GroupMetadataJSON(TypedDict): """ Zarr v2 group metadata document, in-memory merged form. - Models the union of `.zgroup` (the spec-defined `zarr_format` field) - and `.zattrs` (user attributes). On disk these are persisted as two + Models the union of `.zgroup` (the spec-defined `zarr_format` field, + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict @@ -64,7 +65,8 @@ class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): `total=False`). Note: v2 group metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + extension-field concept, and `.zgroup` forbids other keys outright: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), so this partial inherits the same closed shape. Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index e3551e3c72..480f5538fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -1,7 +1,7 @@ """ Rectilinear chunk grid (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ from typing import Final, Literal @@ -42,6 +42,8 @@ class RectilinearChunkGridObject(TypedDict): `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 2f7a089934..a0b33688c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -33,6 +33,8 @@ class RegularChunkGridObject(TypedDict): `chunk_shape` is required and has no default, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py index b6774efbe3..fa4a2aaf84 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py @@ -6,7 +6,9 @@ - `default` -- v3 default encoding (`/`-separated) - `v2` -- v2-compatibility encoding (`.`-separated by default) -Both are defined by the v3 core spec. +Both are defined by the v3 core spec: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst The `ChunkKeyEncodingMetadata` aliases re-exported here are the canonical type for each encoding's permitted JSON shapes. For the underlying diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index c783861b34..39c5385ce4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -31,6 +31,7 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict): """Configuration for the default chunk key encoding. `separator` is optional and defaults to `"/"` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index e2783d296d..3face04f91 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -37,6 +37,7 @@ class V2ChunkKeyEncodingConfiguration(TypedDict): """Configuration for the v2 chunk key encoding. `separator` is optional and defaults to `"."` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 """ separator: NotRequired[V2ChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5a986c8260..4c3631a032 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -50,6 +50,8 @@ class BloscCodecObject(TypedDict): The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, `blocksize`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 04e746f898..43b165f9e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -51,6 +51,8 @@ class BytesCodecObject(TypedDict): at runtime based on data type), so the spec's short-hand-name form is permitted in addition to the object form, and the object form may itself omit `configuration` entirely. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 96c39e5916..d1b878c95c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -1,7 +1,7 @@ """ Cast-value codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ from typing import Final, Literal, NotRequired @@ -89,6 +89,8 @@ class CastValueCodecObject(TypedDict): `configuration.data_type` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 6b9b46c43d..aa72fcae5a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -27,6 +27,7 @@ class Crc32cCodecObject(TypedDict): Per spec the codec has no configuration fields. `configuration` is optional and, if present, should be an empty mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 """ name: Crc32cCodecName @@ -39,6 +40,7 @@ class Crc32cCodecObject(TypedDict): The spec's Extension definition allows extensions with no required configuration to be encoded as a bare short-hand name. CRC32C has no configuration, so both forms are valid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 3b9936f8cd..9a9647263c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -26,6 +26,7 @@ class GzipCodecConfiguration(TypedDict): is required for the metadata to fulfill its reproducibility role, even though the spec text does not mark it required with RFC 2119 keywords. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 """ level: int diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 9701db8497..abf11b4211 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -1,7 +1,7 @@ """ Scale-offset codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/scale_offset +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ from typing import Final, Literal, NotRequired @@ -38,6 +38,7 @@ class ScaleOffsetCodecObject(TypedDict): `configuration` is itself optional per spec — when both `offset` and `scale` are at their identity defaults, the codec is a no-op and the entire `configuration` field may be omitted. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 """ name: ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a8c9247ec4..ac40e78b7b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -35,8 +35,10 @@ class ShardingIndexedCodecConfiguration(TypedDict): `index_codecs` is the codec pipeline applied to the shard index; it must be deterministic (no variable-size compression). + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 `index_location` defaults to `"end"` per the spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 """ chunk_shape: tuple[int, ...] @@ -58,6 +60,8 @@ class ShardingIndexedCodecObject(TypedDict): The configuration has multiple required keys (`chunk_shape`, `codecs`, `index_codecs`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (required members) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index ac469b356a..41d9bcdc6d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -38,6 +38,8 @@ class TransposeCodecObject(TypedDict): `order` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c0faa64bed..9fdf177b4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -1,12 +1,12 @@ """ Zstandard codec types. -See https://github.com/zarr-developers/zarr-specs/pull/256 (unmerged at -time of writing; the configuration shape below reflects the proposed -specification). +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md +(the zarr-extensions registry entry; zarr-specs PR #256, which first +proposed the codec, was never merged). """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -21,11 +21,13 @@ class ZstdCodecConfiguration(TypedDict): """ Configuration for the Zarr v3 `zstd` codec. - Both fields are required per the proposed specification. + `level` is required; `checksum` is optional ("Should be omitted if + false"). + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 """ level: int - checksum: bool + checksum: NotRequired[bool] class ZstdCodecObject(TypedDict): @@ -38,8 +40,10 @@ class ZstdCodecObject(TypedDict): ZstdCodecMetadata = ZstdCodecObject """Permitted JSON shape for `zstd` codec metadata. -Both `level` and `checksum` are required, so only the object form is -valid; the short-hand-name form is not permitted by the spec for this codec. +`level` is required, so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index a9fe0c1f8f..ba7c9aec0d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -25,8 +25,11 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): Inline consolidated metadata embedded in a v3 group. The `metadata` map contains only v3 array and group entries. V2 entries - are excluded from this interoperability convention by design; the v3 core - specification does not define consolidated metadata. + are excluded from this interoperability convention by design. The v3 core + specification acknowledges `consolidated_metadata` as a historical + additional field and fixes this envelope, but leaves the entries to the + reference implementation: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ kind: Literal["inline"] @@ -38,8 +41,12 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): """The key under which consolidated metadata is embedded in a v3 group document. Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata -is carried as an extension field inside the group's own `zarr.json`. Like its v2 -counterpart it is a reference-implementation convention, not a spec artifact. +is carried as an additional field inside the group's own `zarr.json`. The core +spec names the field and its envelope ("For historical reasons, group metadata +documents may contain an additional field named ``consolidated_metadata``"); +the entry format, like the v2 counterpart, is a reference-implementation +convention. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index c7eed64f0f..5892a1bdaa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -1,7 +1,7 @@ """ Zarr `bytes` data type (variable-length raw bytes, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/bytes +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md """ import re diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 41eec441df..264b2c262c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float16`.""" Float16SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat16 = NewType("HexFloat16", str) """A 6-character hex string (`0x` + 4 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float16(value: str) -> HexFloat16: CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" """Canonical hex form of the float16 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 37b7d4f6e8..3b2e786f07 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float32`.""" Float32SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat32 = NewType("HexFloat32", str) """A 10-character hex string (`0x` + 8 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float32(value: str) -> HexFloat32: CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" """Canonical hex form of the float32 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 9a5cf98288..21373d63f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float64`.""" Float64SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat64 = NewType("HexFloat64", str) """An 18-character hex string (`0x` + 16 hex digits) encoding the @@ -46,7 +49,7 @@ def hex_float64(value: str) -> HexFloat64: CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" """Canonical hex form of the float64 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8784160f71..bed264e96e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.datetime64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index f5c8c77bf8..ceb31d8c40 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.timedelta64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index c9c688c9fa..66e69c9b53 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -4,14 +4,19 @@ The `data_type` value is a string of the form `r` where `N` is a positive multiple of 8 (e.g. `r8`, `r16`, `r24`). -See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html +See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html +(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47; fill value: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99) """ import re from typing import Final, NewType RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) -"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`).""" +"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). + +"raw bits, variable size given by *, limited to be a multiple of 8": + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 +""" _RAW_BYTES_RE: Final = re.compile(r"^r(\d+)$") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 0a778ccecc..1e93a95d50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -1,7 +1,7 @@ """ Zarr `string` data type (variable-length utf-8, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/string +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index b1b6b50308..5795c927f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -1,7 +1,7 @@ """ Zarr `struct` data type (heterogeneous record, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/blob/main/data-types/struct/README.md +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 95dc7aea3a..c30f457618 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,6 +180,7 @@ def test_json_value_type_accepts_json_shapes() -> None: def test_string_nan_fill_value_roundtrips() -> None: # Non-finite floats are represented as the spec strings ("NaN", "Infinity", # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 # The string form round-trips cleanly under default dataclass equality, # unlike a raw float('nan') (which is an invalid fill_value the caller must # not pass). @@ -850,21 +851,31 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) -def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: - """Raw `.zarray` documents reject every non-spec member.""" +def test_v2_from_key_value_rejects_zarray_attributes() -> None: + """A raw `.zarray` document must not carry `attributes`: they live in `.zattrs`.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) - doc[extra_key] = {} + doc["attributes"] = {} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ - ((extra_key,), "invalid_value") + (("attributes",), "invalid_value") ] +def test_v2_from_key_value_ignores_zarray_extra_members() -> None: + """Other raw `.zarray` members "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92).""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc["vendor_extension"] = {} + + model = ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert "vendor_extension" not in model.to_json() + + def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and @@ -1326,16 +1337,16 @@ def test_v2_shape_and_chunks_must_have_equal_rank() -> None: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) -def test_v2_filters_must_be_nonempty_when_present() -> None: - """A non-null v2 filter sequence contains one or more codec configurations.""" +def test_v2_filters_may_be_empty() -> None: + """An empty filter list is a list: the spec says "a list ... or null", with no minimum. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("filters",), "invalid_value") - ] - with pytest.raises(MetadataValidationError, match="at least one filter"): - ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + assert validate_array_metadata_v2(doc) == [] + assert ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}).filters == () def test_v2_dimension_separator_literal_enforced() -> None: @@ -1381,13 +1392,15 @@ def test_array_zarr_format_rejects_float( assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] -def test_array_v2_rejects_unknown_document_member() -> None: - """The closed v2 merged-document shape rejects undeclared members.""" +def test_array_v2_ignores_unknown_document_member() -> None: + """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": tolerated, dropped. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("unexpected",), "invalid_value") - ] + assert validate_array_metadata_v2(doc) == [] + assert "unexpected" not in ZarrV2ArrayMetadata.from_json(doc).to_json() def test_array_v3_from_json_materializes_abstract_containers() -> None: @@ -1636,7 +1649,10 @@ def test_must_understand_fields_partition() -> None: """must_understand_fields contains every extra field not explicitly waived with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by - subtracting the extensions it recognizes.""" + subtracting the extensions it recognizes. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1575-L1578 + """ model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, @@ -1662,7 +1678,10 @@ def test_dimension_names_null_field_rejected() -> None: """A dimension_names field whose VALUE is null is invalid: the spec permits null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an - in-memory None sentinel must drop the key, not write null.""" + in-memory None sentinel must drop the key, not write null. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L635-L638 + """ doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4b8c22b84d..d280091ba6 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -503,7 +503,10 @@ def test_v2_consolidated_rejects_unknown_document_member() -> None: def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, - like the array model.""" + like the array model. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573 + """ model = ZarrV3GroupMetadata.create_default( extra_fields={ "waived": {"name": "w", "must_understand": False}, diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index d15b3f118c..80067e81ae 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -182,12 +182,14 @@ def test_array_schemas_reject_negative_dimensions() -> None: _assert_runtime_and_schema_reject(field_type, doc) -def test_v2_array_schema_rejects_empty_filters() -> None: - """The v2 schema mirrors the runtime one-or-more filter rule.""" +def test_v2_array_schema_allows_empty_filters() -> None: + """The v2 schema, like the runtime, takes "a list ... or null" at its word (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79).""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["filters"] = [] + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) - _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + assert adapter.validate_python(doc).filters == () + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) @@ -210,7 +212,6 @@ def test_metadata_field_schema_rejects_unknown_members() -> None: @pytest.mark.parametrize( ("field_type", "source"), [ - (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), ], @@ -225,6 +226,16 @@ def test_v2_schema_rejects_unknown_document_members( _assert_runtime_and_schema_reject(field_type, doc) +def test_v2_array_schema_allows_unknown_document_members() -> None: + """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), in runtime and schema.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["unexpected"] = 1 + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) + + assert "unexpected" not in adapter.validate_python(doc).to_json() + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + def test_v3_array_schema_allows_unknown_extension_fields() -> None: """Schema constraints do not close the v3 top-level extension namespace.""" doc = json.loads(json.dumps(V3_ARRAY_DOC))