Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/zarr-metadata/changes/4365.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys ["SHOULD NOT be present ... and SHOULD be ignored"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), unlike `.zgroup`'s ["MUST NOT"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313)), and `filters: []` is accepted (["a list of JSON objects providing codec configurations, or null"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79) sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length.
12 changes: 9 additions & 3 deletions packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True):
ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON
ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON
ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)]
ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)]
ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...]


class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue):
Expand Down Expand Up @@ -74,8 +74,14 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue):
consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None]


class ZarrV2ArrayMetadataJSON(TypedDict, closed=True):
"""Schema input for the closed, merged v2 array representation."""
class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue):
"""Schema input for the merged v2 array representation.

Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT
be present ... and SHOULD be ignored by implementations"
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92); the group document's "MUST NOT" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313) keeps
`ZarrV2GroupMetadataJSON` closed.
"""

zarr_format: Literal[2]
shape: tuple[NonNegativeInt, ...]
Expand Down
11 changes: 6 additions & 5 deletions packages/zarr-metadata/src/zarr_metadata/model/_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class ZarrV3NamedConfig:

Bare names and missing configurations normalize to an empty configuration.
Bare names and missing `must_understand` members normalize to the spec's
implicit `True` value.
implicit `True` value (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573).
"""

name: str
Expand Down Expand Up @@ -107,8 +107,8 @@ def must_understand_subset(
) -> dict[str, ZarrV3ExtensionField]:
"""The subset of `extra_fields` the reader is obligated to understand.

Per the v3 spec, an extension field is implicitly `must_understand: True`
unless it explicitly says otherwise, and an implementation MUST fail to
Per the v3 spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), an extension field is implicitly `must_understand:
True` unless it explicitly says otherwise, and an implementation MUST fail to
open a group or array carrying fields it does not recognize that are not
explicitly `must_understand: false`. A non-mapping field value cannot
carry the explicit waiver, so it always requires understanding (the
Expand Down Expand Up @@ -310,7 +310,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]:
"""Extra fields the reader is obligated to understand.

Everything in `extra_fields` not explicitly waived with
`must_understand: false` (the spec's implicit-true rule). A compliant
`must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant
reader MUST fail to open the array if this contains any field it does
not recognize; the model layer only partitions by obligation, since
recognition is reader-specific.
Expand Down Expand Up @@ -428,7 +428,8 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON:
`attributes` is included when set (even empty). This is not the
on-disk `.zarray` content: a conforming `.zarray` must exclude
`attributes` (they live in the sibling `.zattrs` file). Use
`to_key_value` to produce the spec-conforming split for storage.
`to_key_value` to produce the spec-conforming split for storage
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330).
"""
# to_json output shares no mutable state with the model: every value
# that can hold a mutable container is deep-copied.
Expand Down
5 changes: 3 additions & 2 deletions packages/zarr-metadata/src/zarr_metadata/model/_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]:
"""Extra fields the reader is obligated to understand.

Everything in `extra_fields` not explicitly waived with
`must_understand: false` (the spec's implicit-true rule). A compliant
`must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant
reader MUST fail to open the group if this contains any field it does
not recognize; the model layer only partitions by obligation, since
recognition is reader-specific.
Expand Down Expand Up @@ -299,7 +299,8 @@ def to_json(self) -> ZarrV2GroupMetadataJSON:
`attributes` is included when set (even empty). This is not the
on-disk `.zgroup` content: a conforming `.zgroup` must exclude
`attributes` (they live in the sibling `.zattrs` file). Use
`to_key_value` to produce the spec-conforming split for storage.
`to_key_value` to produce the spec-conforming split for storage
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313; https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330).
"""
# to_json output shares no mutable state with the model.
out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format}
Expand Down
14 changes: 7 additions & 7 deletions packages/zarr-metadata/src/zarr_metadata/model/_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -584,10 +584,12 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]:
if not isinstance(value, Mapping):
return [ValidationProblem((), "expected a mapping", "invalid_type")]
doc = cast("Mapping[str, object]", value)
# Unlike the group document ("Other keys MUST NOT be present",
# https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be
# present within the metadata object and SHOULD be ignored by
# implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so members outside
# ARRAY_METADATA_STANDARD_KEYS_V2 are not problems.
problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc)
problems.extend(
_unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value))
)
problems.extend(_check_literal(doc, "zarr_format", 2))
shape_problems = _validate_dim_sequence(doc, "shape")
chunks_problems = _validate_dim_sequence(doc, "chunks")
Expand Down Expand Up @@ -642,10 +644,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]:
)
)
elif filters is not None:
if len(cast("Sequence[object]", filters)) == 0:
problems.append(
ValidationProblem(("filters",), "expected at least one filter", "invalid_value")
)
# "A list of JSON objects providing codec configurations, or
# null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list.
for index, item in enumerate(cast("Sequence[object]", filters)):
problems.extend(_prefix("filters", _prefix(index, validate_json(item))))
if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"):
Expand Down
10 changes: 6 additions & 4 deletions packages/zarr-metadata/src/zarr_metadata/v2/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ class ZarrV2ArrayMetadataJSON(TypedDict):
"""
Zarr v2 array metadata document, in-memory merged form.

Models the union of `.zarray` (the spec-defined fields) and `.zattrs`
(user attributes). On disk, attributes live in a sibling `.zattrs` file
Models the union of `.zarray` (the spec-defined fields, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L51-L92)
and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk, attributes live in a sibling `.zattrs` file
and are not part of `.zarray`; this type folds them in as the
`attributes` field so a single TypedDict represents the complete
in-memory state of a v2 array node. Consumers that read or write a
Expand Down Expand Up @@ -126,8 +126,10 @@ class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False):
`tests/test_partial_equivalence.py` passes without special-casing those
fields (PEP 655 explicitly permits `NotRequired` inside `total=False`).

Note: v2 array metadata has no `extra_items` setting (the v2 spec has no
extension-field concept), so this partial inherits the same closed shape.
Note: v2 array metadata has no `extra_items` setting: the v2 spec has no
extension-field concept, and other `.zarray` keys "SHOULD be ignored by
implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so nothing beyond the spec-defined
fields is modeled.

Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by
`tests/test_partial_equivalence.py`.
Expand Down
1 change: 1 addition & 0 deletions packages/zarr-metadata/src/zarr_metadata/v2/attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup`
files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not
constrain the keys or values of the attributes mapping.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330
"""


Expand Down
14 changes: 8 additions & 6 deletions packages/zarr-metadata/src/zarr_metadata/v2/group.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ class ZarrV2ZGroupJSON(TypedDict):
On-disk `.zgroup` file content.

Strict shape of the JSON document persisted at `<path>/.zgroup` for
a v2 group. The spec defines exactly one field. User attributes live
in a sibling `.zattrs` file and are NOT part of this type; see
`ZarrV2ZAttrsJSON`.
a v2 group. The spec defines exactly one field and forbids others. User
attributes live in a sibling `.zattrs` file and are NOT part of this
type; see `ZarrV2ZAttrsJSON`.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313

See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html
"""
Expand All @@ -30,8 +31,8 @@ class ZarrV2GroupMetadataJSON(TypedDict):
"""
Zarr v2 group metadata document, in-memory merged form.

Models the union of `.zgroup` (the spec-defined `zarr_format` field)
and `.zattrs` (user attributes). On disk these are persisted as two
Models the union of `.zgroup` (the spec-defined `zarr_format` field,
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk these are persisted as two
separate files; this type folds them so a single TypedDict represents
the complete in-memory state of a v2 group node. Consumers that read
or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict
Expand Down Expand Up @@ -64,7 +65,8 @@ class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False):
`total=False`).

Note: v2 group metadata has no `extra_items` setting (the v2 spec has no
extension-field concept), so this partial inherits the same closed shape.
extension-field concept, and `.zgroup` forbids other keys outright:
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), so this partial inherits the same closed shape.

Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by
`tests/test_partial_equivalence.py`.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""
Rectilinear chunk grid (zarr-extensions).

See https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear
See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md
"""

from typing import Final, Literal
Expand Down Expand Up @@ -42,6 +42,8 @@ class RectilinearChunkGridObject(TypedDict):

`kind` and `chunk_shapes` are required, so only the object form is valid;
the short-hand-name form is not permitted by the spec for this grid.
https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564
"""

__all__ = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ class RegularChunkGridObject(TypedDict):

`chunk_shape` is required and has no default, so only the object form is
valid; the short-hand-name form is not permitted by the spec for this grid.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration")
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564
"""

__all__ = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
- `default` -- v3 default encoding (`/`-separated)
- `v2` -- v2-compatibility encoding (`.`-separated by default)

Both are defined by the v3 core spec.
Both are defined by the v3 core spec:
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst

The `<X>ChunkKeyEncodingMetadata` aliases re-exported here are the canonical
type for each encoding's permitted JSON shapes. For the underlying
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict):
"""Configuration for the default chunk key encoding.

`separator` is optional and defaults to `"/"` per spec.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29
"""

separator: NotRequired[DefaultChunkKeyEncodingSeparator]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class V2ChunkKeyEncodingConfiguration(TypedDict):
"""Configuration for the v2 chunk key encoding.

`separator` is optional and defaults to `"."` per spec.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29
"""

separator: NotRequired[V2ChunkKeyEncodingSeparator]
Expand Down
2 changes: 2 additions & 0 deletions packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ class BloscCodecObject(TypedDict):
The configuration has multiple required keys (`cname`, `clevel`, `shuffle`,
`blocksize`), so only the object form is valid; the short-hand-name form
is not permitted by the spec for this codec.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters)
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required")
"""

__all__ = [
Expand Down
2 changes: 2 additions & 0 deletions packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class BytesCodecObject(TypedDict):
at runtime based on data type), so the spec's short-hand-name form is
permitted in addition to the object form, and the object form may itself
omit `configuration` entirely.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable")
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564
"""

__all__ = [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""
Cast-value codec types.

See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value
See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md
"""

from typing import Final, Literal, NotRequired
Expand Down Expand Up @@ -89,6 +89,8 @@ class CastValueCodecObject(TypedDict):

`configuration.data_type` is required, so only the object form is valid;
the short-hand-name form is not permitted by the spec for this codec.
https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields)
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required")
"""


Expand Down
2 changes: 2 additions & 0 deletions packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class Crc32cCodecObject(TypedDict):

Per spec the codec has no configuration fields. `configuration` is
optional and, if present, should be an empty mapping.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66
"""

name: Crc32cCodecName
Expand All @@ -39,6 +40,7 @@ class Crc32cCodecObject(TypedDict):
The spec's Extension definition allows extensions with no required
configuration to be encoded as a bare short-hand name. CRC32C has no
configuration, so both forms are valid.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564
"""


Expand Down
1 change: 1 addition & 0 deletions packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class GzipCodecConfiguration(TypedDict):
is required for the metadata to fulfill its reproducibility role,
even though the spec text does not mark it required with RFC 2119
keywords.
https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66
"""

level: int
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""
Scale-offset codec types.

See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/scale_offset
See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md
"""

from typing import Final, Literal, NotRequired
Expand Down Expand Up @@ -38,6 +38,7 @@ class ScaleOffsetCodecObject(TypedDict):
`configuration` is itself optional per spec — when both `offset` and
`scale` are at their identity defaults, the codec is a no-op and the
entire `configuration` field may be omitted.
https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35
"""

name: ScaleOffsetCodecName
Expand Down
Loading
Loading