From 1508f97846028491e588f18d247252b64179793b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 16 Sep 2026 14:32:05 +0200 Subject: [PATCH 1/4] fix(zarr-metadata): v2 array document is open and filters may be empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two v2 structural rules were stricter than the spec, and the shared conformance corpus (zarr-metadata.js, conformance/v2_array.json cases 5 and 8) has been updated first; this brings the reference implementation back into agreement. - Members outside the .zarray definition were rejected. The spec: "Other keys SHOULD NOT be present within the metadata object and SHOULD be ignored by implementations" — a recommendation, unlike .zgroup's "Other keys MUST NOT be present". Extras are now tolerated by the validator, dropped by the model, and permitted by the Pydantic schema (the TypedDict is open, like the v3 one). The on-disk `.zarray` rule that `attributes` belongs in `.zattrs` is unchanged. - filters: [] was rejected ("expected at least one filter"). The spec: "A list of JSON objects providing codec configurations, or null" — an empty list is a list. The Pydantic schema's min_length is dropped. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/_pydantic_schema.py | 11 +++-- .../src/zarr_metadata/model/_validation.py | 13 +++--- .../zarr-metadata/tests/model/test_array.py | 40 +++++++++++-------- .../tests/model/test_pydantic_module.py | 19 +++++++-- 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index e9792d6931..509c1082a6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -38,7 +38,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3MetadataFieldJSON = str | ZarrV3NamedConfigJSON ZarrV3MandatoryMetadataFieldJSON = str | ZarrV3MandatoryNamedConfigJSON ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MetadataFieldJSON, ...], Field(min_length=1)] -ZarrV2FilterPipelineJSON = Annotated[tuple[ZarrV2CodecMetadata, ...], Field(min_length=1)] +ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=JSONValue): @@ -74,8 +74,13 @@ class ZarrV3GroupMetadataJSON(TypedDict, extra_items=JSONValue): consolidated_metadata: NotRequired[ZarrV3ConsolidatedMetadataJSON | None] -class ZarrV2ArrayMetadataJSON(TypedDict, closed=True): - """Schema input for the closed, merged v2 array representation.""" +class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): + """Schema input for the merged v2 array representation. + + Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT + be present ... and SHOULD be ignored by implementations" (the group + document's "MUST NOT" keeps `ZarrV2GroupMetadataJSON` closed). + """ zarr_format: Literal[2] shape: tuple[NonNegativeInt, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index a12e1911b1..4fac38f8af 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -584,10 +584,11 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) + # Unlike the group document ("Other keys MUST NOT be present"), the v2 + # array document is open: other keys "SHOULD NOT be present within the + # metadata object and SHOULD be ignored by implementations", so members + # outside ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) - problems.extend( - _unexpected_keys(ARRAY_METADATA_STANDARD_KEYS_V2, cast("Mapping[object, object]", value)) - ) problems.extend(_check_literal(doc, "zarr_format", 2)) shape_problems = _validate_dim_sequence(doc, "shape") chunks_problems = _validate_dim_sequence(doc, "chunks") @@ -642,10 +643,8 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) ) elif filters is not None: - if len(cast("Sequence[object]", filters)) == 0: - problems.append( - ValidationProblem(("filters",), "expected at least one filter", "invalid_value") - ) + # "A list of JSON objects providing codec configurations, or + # null": an empty list is a list. for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 95dc7aea3a..348bc7e619 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -850,21 +850,31 @@ def test_v2_from_key_value_remerges_zattrs() -> None: assert model.shape == (10,) -@pytest.mark.parametrize("extra_key", ["attributes", "vendor_extension"]) -def test_v2_from_key_value_rejects_zarray_extra_members(extra_key: str) -> None: - """Raw `.zarray` documents reject every non-spec member.""" +def test_v2_from_key_value_rejects_zarray_attributes() -> None: + """A raw `.zarray` document must not carry `attributes`: they live in `.zattrs`.""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) - doc[extra_key] = {} + doc["attributes"] = {} with pytest.raises(MetadataValidationError) as exc_info: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) assert [(problem.loc, problem.kind) for problem in exc_info.value.problems] == [ - ((extra_key,), "invalid_value") + (("attributes",), "invalid_value") ] +def test_v2_from_key_value_ignores_zarray_extra_members() -> None: + """Other raw `.zarray` members "SHOULD be ignored by implementations".""" + doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) + doc.pop("attributes", None) + doc["vendor_extension"] = {} + + model = ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + + assert "vendor_extension" not in model.to_json() + + def test_v2_zattrs_presence_round_trips() -> None: """The .zattrs file's presence is part of the store: an absent file reads as UNSET and emits no .zattrs; an explicit empty file reads as {} and @@ -1326,16 +1336,13 @@ def test_v2_shape_and_chunks_must_have_equal_rank() -> None: ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) -def test_v2_filters_must_be_nonempty_when_present() -> None: - """A non-null v2 filter sequence contains one or more codec configurations.""" +def test_v2_filters_may_be_empty() -> None: + """An empty filter list is a list: the spec says "a list ... or null", with no minimum.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("filters",), "invalid_value") - ] - with pytest.raises(MetadataValidationError, match="at least one filter"): - ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}) + assert validate_array_metadata_v2(doc) == [] + assert ZarrV2ArrayMetadata.from_key_value({".zarray": json.dumps(doc).encode()}).filters == () def test_v2_dimension_separator_literal_enforced() -> None: @@ -1381,13 +1388,12 @@ def test_array_zarr_format_rejects_float( assert [(p.loc, p.kind) for p in validate(document)] == [(("zarr_format",), "invalid_value")] -def test_array_v2_rejects_unknown_document_member() -> None: - """The closed v2 merged-document shape rejects undeclared members.""" +def test_array_v2_ignores_unknown_document_member() -> None: + """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": tolerated, dropped.""" doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} - assert [(p.loc, p.kind) for p in validate_array_metadata_v2(doc)] == [ - (("unexpected",), "invalid_value") - ] + assert validate_array_metadata_v2(doc) == [] + assert "unexpected" not in ZarrV2ArrayMetadata.from_json(doc).to_json() def test_array_v3_from_json_materializes_abstract_containers() -> None: diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index d15b3f118c..67e207708a 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -182,12 +182,14 @@ def test_array_schemas_reject_negative_dimensions() -> None: _assert_runtime_and_schema_reject(field_type, doc) -def test_v2_array_schema_rejects_empty_filters() -> None: - """The v2 schema mirrors the runtime one-or-more filter rule.""" +def test_v2_array_schema_allows_empty_filters() -> None: + """The v2 schema, like the runtime, takes "a list ... or null" at its word: no minimum.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["filters"] = [] + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) - _assert_runtime_and_schema_reject(zmp.ZarrV2ArrayMetadata, doc) + assert adapter.validate_python(doc).filters == () + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) @pytest.mark.parametrize("field", ["data_type", "chunk_grid", "chunk_key_encoding"]) @@ -210,7 +212,6 @@ def test_metadata_field_schema_rejects_unknown_members() -> None: @pytest.mark.parametrize( ("field_type", "source"), [ - (zmp.ZarrV2ArrayMetadata, V2_ARRAY_DOC), (zmp.ZarrV2GroupMetadata, V2_GROUP_DOC), (zmp.ZarrV2ConsolidatedMetadata, V2_CONSOLIDATED_DOC), ], @@ -225,6 +226,16 @@ def test_v2_schema_rejects_unknown_document_members( _assert_runtime_and_schema_reject(field_type, doc) +def test_v2_array_schema_allows_unknown_document_members() -> None: + """The v2 array document is open (other keys SHOULD be ignored), in runtime and schema.""" + doc = json.loads(json.dumps(V2_ARRAY_DOC)) + doc["unexpected"] = 1 + adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) + + assert "unexpected" not in adapter.validate_python(doc).to_json() + assert Draft202012Validator(adapter.json_schema()).is_valid(doc) + + def test_v3_array_schema_allows_unknown_extension_fields() -> None: """Schema constraints do not close the v3 top-level extension namespace.""" doc = json.loads(json.dumps(V3_ARRAY_DOC)) From 00aca5f08337017273dc2a4f12593d86b6dc35cf Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 16 Sep 2026 14:32:18 +0200 Subject: [PATCH 2/4] docs(zarr-metadata): changelog fragment for #4365 Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4365.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/4365.bugfix.md diff --git a/packages/zarr-metadata/changes/4365.bugfix.md b/packages/zarr-metadata/changes/4365.bugfix.md new file mode 100644 index 0000000000..5694005267 --- /dev/null +++ b/packages/zarr-metadata/changes/4365.bugfix.md @@ -0,0 +1 @@ +The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys "SHOULD NOT be present ... and SHOULD be ignored", unlike `.zgroup`'s "MUST NOT"), and `filters: []` is accepted ("a list of JSON objects providing codec configurations, or null" sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length. From f651b73f313a71b4af816365fc1ce285945788c1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 16 Sep 2026 15:47:13 +0200 Subject: [PATCH 3/4] docs(zarr-metadata): link the spec text the v2 open-array change cites Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- packages/zarr-metadata/changes/4365.bugfix.md | 2 +- .../src/zarr_metadata/_pydantic_schema.py | 5 +++-- .../src/zarr_metadata/model/_validation.py | 11 ++++++----- packages/zarr-metadata/tests/model/test_array.py | 12 +++++++++--- .../tests/model/test_pydantic_module.py | 4 ++-- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/zarr-metadata/changes/4365.bugfix.md b/packages/zarr-metadata/changes/4365.bugfix.md index 5694005267..6cda30f2d2 100644 --- a/packages/zarr-metadata/changes/4365.bugfix.md +++ b/packages/zarr-metadata/changes/4365.bugfix.md @@ -1 +1 @@ -The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys "SHOULD NOT be present ... and SHOULD be ignored", unlike `.zgroup`'s "MUST NOT"), and `filters: []` is accepted ("a list of JSON objects providing codec configurations, or null" sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length. +The v2 array validator follows the spec on two points it was stricter than: members outside the `.zarray` definition are tolerated and dropped (the spec says other keys ["SHOULD NOT be present ... and SHOULD be ignored"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), unlike `.zgroup`'s ["MUST NOT"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313)), and `filters: []` is accepted (["a list of JSON objects providing codec configurations, or null"](https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79) sets no minimum). The Pydantic schema follows: `ZarrV2ArrayMetadataJSON` is open and the filter pipeline has no minimum length. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index 509c1082a6..51ec633b82 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -78,8 +78,9 @@ class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue): """Schema input for the merged v2 array representation. Open, like the runtime validator: the v2 spec says other keys "SHOULD NOT - be present ... and SHOULD be ignored by implementations" (the group - document's "MUST NOT" keeps `ZarrV2GroupMetadataJSON` closed). + be present ... and SHOULD be ignored by implementations" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92); the group document's "MUST NOT" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313) keeps + `ZarrV2GroupMetadataJSON` closed. """ zarr_format: Literal[2] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py index 4fac38f8af..d1cd74b719 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_validation.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_validation.py @@ -584,10 +584,11 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: if not isinstance(value, Mapping): return [ValidationProblem((), "expected a mapping", "invalid_type")] doc = cast("Mapping[str, object]", value) - # Unlike the group document ("Other keys MUST NOT be present"), the v2 - # array document is open: other keys "SHOULD NOT be present within the - # metadata object and SHOULD be ignored by implementations", so members - # outside ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. + # Unlike the group document ("Other keys MUST NOT be present", + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), the v2 array document is open: other keys "SHOULD NOT be + # present within the metadata object and SHOULD be ignored by + # implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so members outside + # ARRAY_METADATA_STANDARD_KEYS_V2 are not problems. problems: list[ValidationProblem] = _missing_keys(ARRAY_METADATA_REQUIRED_KEYS_V2, doc) problems.extend(_check_literal(doc, "zarr_format", 2)) shape_problems = _validate_dim_sequence(doc, "shape") @@ -644,7 +645,7 @@ def validate_array_metadata_v2(value: object) -> list[ValidationProblem]: ) elif filters is not None: # "A list of JSON objects providing codec configurations, or - # null": an empty list is a list. + # null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list. for index, item in enumerate(cast("Sequence[object]", filters)): problems.extend(_prefix("filters", _prefix(index, validate_json(item)))) if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"): diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 348bc7e619..692cbe76cc 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -865,7 +865,7 @@ def test_v2_from_key_value_rejects_zarray_attributes() -> None: def test_v2_from_key_value_ignores_zarray_extra_members() -> None: - """Other raw `.zarray` members "SHOULD be ignored by implementations".""" + """Other raw `.zarray` members "SHOULD be ignored by implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92).""" doc: dict[str, object] = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc.pop("attributes", None) doc["vendor_extension"] = {} @@ -1337,7 +1337,10 @@ def test_v2_shape_and_chunks_must_have_equal_rank() -> None: def test_v2_filters_may_be_empty() -> None: - """An empty filter list is a list: the spec says "a list ... or null", with no minimum.""" + """An empty filter list is a list: the spec says "a list ... or null", with no minimum. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) doc["filters"] = () @@ -1389,7 +1392,10 @@ def test_array_zarr_format_rejects_float( def test_array_v2_ignores_unknown_document_member() -> None: - """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": tolerated, dropped.""" + """Other .zarray keys "SHOULD NOT be present ... and SHOULD be ignored": tolerated, dropped. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92 + """ doc = dict(ZarrV2ArrayMetadata.create_default().to_json()) | {"unexpected": 1} assert validate_array_metadata_v2(doc) == [] diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index 67e207708a..80067e81ae 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -183,7 +183,7 @@ def test_array_schemas_reject_negative_dimensions() -> None: def test_v2_array_schema_allows_empty_filters() -> None: - """The v2 schema, like the runtime, takes "a list ... or null" at its word: no minimum.""" + """The v2 schema, like the runtime, takes "a list ... or null" at its word (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79).""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["filters"] = [] adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) @@ -227,7 +227,7 @@ def test_v2_schema_rejects_unknown_document_members( def test_v2_array_schema_allows_unknown_document_members() -> None: - """The v2 array document is open (other keys SHOULD be ignored), in runtime and schema.""" + """The v2 array document is open ("SHOULD be ignored", https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), in runtime and schema.""" doc = json.loads(json.dumps(V2_ARRAY_DOC)) doc["unexpected"] = 1 adapter = TypeAdapter(zmp.ZarrV2ArrayMetadata) From 2e6b275ed4c929bc02e47f73a4e0788a181118e8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Wed, 16 Sep 2026 15:57:30 +0200 Subject: [PATCH 4/4] docs(zarr-metadata): link the spec text behind every spec statement Every docstring or comment that cites the Zarr spec or a zarr-extensions README now carries a commit-pinned permalink with a line range (zarr-specs fc7dd9c; zarr-extensions 4da7b37, the registry commit the TypeScript port vendors). Unpinned zarr-extensions `tree/main` page links are pinned the same way. Three statements were wrong or stale and are corrected: - zstd: `checksum` was typed required "per the proposed specification" (zarr-specs PR #256, never merged). The published zarr-extensions entry makes it optional ("Should be omitted if false"; schema requires only `level`), so it is now `NotRequired[bool]`. - v3 consolidated metadata was described as "not a spec artifact"; since zarr-specs #373 the core spec names the field and fixes its envelope (core/index.rst L802-L816); the entry format remains a convention. - The v2 array `*Partial` docstring spoke of a "closed shape"; the array document is open (other keys SHOULD be ignored), unlike `.zgroup`. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 --- .../src/zarr_metadata/model/_array.py | 11 +++++----- .../src/zarr_metadata/model/_group.py | 5 +++-- .../src/zarr_metadata/v2/array.py | 10 ++++++---- .../src/zarr_metadata/v2/attributes.py | 1 + .../src/zarr_metadata/v2/group.py | 14 +++++++------ .../v3/chunk_grid/rectilinear.py | 4 +++- .../zarr_metadata/v3/chunk_grid/regular.py | 2 ++ .../v3/chunk_key_encoding/__init__.py | 4 +++- .../v3/chunk_key_encoding/default.py | 1 + .../zarr_metadata/v3/chunk_key_encoding/v2.py | 1 + .../src/zarr_metadata/v3/codec/blosc.py | 2 ++ .../src/zarr_metadata/v3/codec/bytes.py | 2 ++ .../src/zarr_metadata/v3/codec/cast_value.py | 4 +++- .../src/zarr_metadata/v3/codec/crc32c.py | 2 ++ .../src/zarr_metadata/v3/codec/gzip.py | 1 + .../zarr_metadata/v3/codec/scale_offset.py | 3 ++- .../v3/codec/sharding_indexed.py | 4 ++++ .../src/zarr_metadata/v3/codec/transpose.py | 2 ++ .../src/zarr_metadata/v3/codec/zstd.py | 20 +++++++++++-------- .../src/zarr_metadata/v3/consolidated.py | 15 ++++++++++---- .../src/zarr_metadata/v3/data_type/bytes.py | 2 +- .../src/zarr_metadata/v3/data_type/float16.py | 7 +++++-- .../src/zarr_metadata/v3/data_type/float32.py | 7 +++++-- .../src/zarr_metadata/v3/data_type/float64.py | 7 +++++-- .../v3/data_type/numpy_datetime64.py | 2 +- .../v3/data_type/numpy_timedelta64.py | 2 +- .../src/zarr_metadata/v3/data_type/raw.py | 9 +++++++-- .../src/zarr_metadata/v3/data_type/string.py | 2 +- .../src/zarr_metadata/v3/data_type/struct.py | 2 +- .../zarr-metadata/tests/model/test_array.py | 11 ++++++++-- .../zarr-metadata/tests/model/test_group.py | 5 ++++- 31 files changed, 115 insertions(+), 49 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array.py b/packages/zarr-metadata/src/zarr_metadata/model/_array.py index 0b562bc188..db708c0152 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array.py @@ -51,7 +51,7 @@ class ZarrV3NamedConfig: Bare names and missing configurations normalize to an empty configuration. Bare names and missing `must_understand` members normalize to the spec's - implicit `True` value. + implicit `True` value (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573). """ name: str @@ -107,8 +107,8 @@ def must_understand_subset( ) -> dict[str, ZarrV3ExtensionField]: """The subset of `extra_fields` the reader is obligated to understand. - Per the v3 spec, an extension field is implicitly `must_understand: True` - unless it explicitly says otherwise, and an implementation MUST fail to + Per the v3 spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578), an extension field is implicitly `must_understand: + True` unless it explicitly says otherwise, and an implementation MUST fail to open a group or array carrying fields it does not recognize that are not explicitly `must_understand: false`. A non-mapping field value cannot carry the explicit waiver, so it always requires understanding (the @@ -310,7 +310,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the array if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -428,7 +428,8 @@ def to_json(self) -> ZarrV2ArrayMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zarray` content: a conforming `.zarray` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model: every value # that can hold a mutable container is deep-copied. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group.py b/packages/zarr-metadata/src/zarr_metadata/model/_group.py index 63dfe5611f..bb2e14da1b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group.py @@ -168,7 +168,7 @@ def must_understand_fields(self) -> dict[str, ZarrV3ExtensionField]: """Extra fields the reader is obligated to understand. Everything in `extra_fields` not explicitly waived with - `must_understand: false` (the spec's implicit-true rule). A compliant + `must_understand: false` (the spec's implicit-true rule, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1578). A compliant reader MUST fail to open the group if this contains any field it does not recognize; the model layer only partitions by obligation, since recognition is reader-specific. @@ -299,7 +299,8 @@ def to_json(self) -> ZarrV2GroupMetadataJSON: `attributes` is included when set (even empty). This is not the on-disk `.zgroup` content: a conforming `.zgroup` must exclude `attributes` (they live in the sibling `.zattrs` file). Use - `to_key_value` to produce the spec-conforming split for storage. + `to_key_value` to produce the spec-conforming split for storage + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313; https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). """ # to_json output shares no mutable state with the model. out: ZarrV2GroupMetadataJSON = {"zarr_format": self.zarr_format} diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index e026e5c655..7e4f87be7c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -81,8 +81,8 @@ class ZarrV2ArrayMetadataJSON(TypedDict): """ Zarr v2 array metadata document, in-memory merged form. - Models the union of `.zarray` (the spec-defined fields) and `.zattrs` - (user attributes). On disk, attributes live in a sibling `.zattrs` file + Models the union of `.zarray` (the spec-defined fields, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L51-L92) + and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk, attributes live in a sibling `.zattrs` file and are not part of `.zarray`; this type folds them in as the `attributes` field so a single TypedDict represents the complete in-memory state of a v2 array node. Consumers that read or write a @@ -126,8 +126,10 @@ class ZarrV2ArrayMetadataJSONPartial(TypedDict, total=False): `tests/test_partial_equivalence.py` passes without special-casing those fields (PEP 655 explicitly permits `NotRequired` inside `total=False`). - Note: v2 array metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + Note: v2 array metadata has no `extra_items` setting: the v2 spec has no + extension-field concept, and other `.zarray` keys "SHOULD be ignored by + implementations" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L91-L92), so nothing beyond the spec-defined + fields is modeled. Drift between this type and `ZarrV2ArrayMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index 68785d1660..74e9fe938f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -15,6 +15,7 @@ Spec-defined keys for arrays / groups live in sibling `.zarray` / `.zgroup` files (modeled by `ZarrV2ZArrayJSON` / `ZarrV2ZGroupJSON`). This type does not constrain the keys or values of the attributes mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index 34d72742c2..3a645dc29b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -16,9 +16,10 @@ class ZarrV2ZGroupJSON(TypedDict): On-disk `.zgroup` file content. Strict shape of the JSON document persisted at `/.zgroup` for - a v2 group. The spec defines exactly one field. User attributes live - in a sibling `.zattrs` file and are NOT part of this type; see - `ZarrV2ZAttrsJSON`. + a v2 group. The spec defines exactly one field and forbids others. User + attributes live in a sibling `.zattrs` file and are NOT part of this + type; see `ZarrV2ZAttrsJSON`. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313 See https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html """ @@ -30,8 +31,8 @@ class ZarrV2GroupMetadataJSON(TypedDict): """ Zarr v2 group metadata document, in-memory merged form. - Models the union of `.zgroup` (the spec-defined `zarr_format` field) - and `.zattrs` (user attributes). On disk these are persisted as two + Models the union of `.zgroup` (the spec-defined `zarr_format` field, + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L306-L313) and `.zattrs` (user attributes, https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L323-L330). On disk these are persisted as two separate files; this type folds them so a single TypedDict represents the complete in-memory state of a v2 group node. Consumers that read or write the real on-disk files should use `ZarrV2ZGroupJSON` (strict @@ -64,7 +65,8 @@ class ZarrV2GroupMetadataJSONPartial(TypedDict, total=False): `total=False`). Note: v2 group metadata has no `extra_items` setting (the v2 spec has no - extension-field concept), so this partial inherits the same closed shape. + extension-field concept, and `.zgroup` forbids other keys outright: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L313), so this partial inherits the same closed shape. Drift between this type and `ZarrV2GroupMetadataJSON` is prevented by `tests/test_partial_equivalence.py`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index e3551e3c72..480f5538fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -1,7 +1,7 @@ """ Rectilinear chunk grid (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-grids/rectilinear +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md """ from typing import Final, Literal @@ -42,6 +42,8 @@ class RectilinearChunkGridObject(TypedDict): `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py index 2f7a089934..a0b33688c8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/regular.py @@ -33,6 +33,8 @@ class RegularChunkGridObject(TypedDict): `chunk_shape` is required and has no default, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L528-L537 ("must be an object with the names name and configuration") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py index b6774efbe3..fa4a2aaf84 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/__init__.py @@ -6,7 +6,9 @@ - `default` -- v3 default encoding (`/`-separated) - `v2` -- v2-compatibility encoding (`.`-separated by default) -Both are defined by the v3 core spec. +Both are defined by the v3 core spec: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst The `ChunkKeyEncodingMetadata` aliases re-exported here are the canonical type for each encoding's permitted JSON shapes. For the underlying diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py index c783861b34..39c5385ce4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/default.py @@ -31,6 +31,7 @@ class DefaultChunkKeyEncodingConfiguration(TypedDict): """Configuration for the default chunk key encoding. `separator` is optional and defaults to `"/"` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/default/index.rst#L27-L29 """ separator: NotRequired[DefaultChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py index e2783d296d..3face04f91 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_key_encoding/v2.py @@ -37,6 +37,7 @@ class V2ChunkKeyEncodingConfiguration(TypedDict): """Configuration for the v2 chunk key encoding. `separator` is optional and defaults to `"."` per spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/chunk-key-encodings/v2/index.rst#L27-L29 """ separator: NotRequired[V2ChunkKeyEncodingSeparator] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py index 5a986c8260..4c3631a032 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/blosc.py @@ -50,6 +50,8 @@ class BloscCodecObject(TypedDict): The configuration has multiple required keys (`cname`, `clevel`, `shuffle`, `blocksize`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/blosc/index.rst#L57-L98 (configuration parameters) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 04e746f898..43b165f9e6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -51,6 +51,8 @@ class BytesCodecObject(TypedDict): at runtime based on data type), so the spec's short-hand-name form is permitted in addition to the object form, and the object form may itself omit `configuration` entirely. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69 ("endian: Required for data types for which endianness is applicable") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 96c39e5916..d1b878c95c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -1,7 +1,7 @@ """ Cast-value codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md """ from typing import Final, Literal, NotRequired @@ -89,6 +89,8 @@ class CastValueCodecObject(TypedDict): `configuration.data_type` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/cast_value/README.md#L33-L36 and #L46-L48 (required fields) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py index 6b9b46c43d..aa72fcae5a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/crc32c.py @@ -27,6 +27,7 @@ class Crc32cCodecObject(TypedDict): Per spec the codec has no configuration fields. `configuration` is optional and, if present, should be an empty mapping. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/crc32c/index.rst#L63-L66 """ name: Crc32cCodecName @@ -39,6 +40,7 @@ class Crc32cCodecObject(TypedDict): The spec's Extension definition allows extensions with no required configuration to be encoded as a bare short-hand name. CRC32C has no configuration, so both forms are valid. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py index 3b9936f8cd..9a9647263c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/gzip.py @@ -26,6 +26,7 @@ class GzipCodecConfiguration(TypedDict): is required for the metadata to fulfill its reproducibility role, even though the spec text does not mark it required with RFC 2119 keywords. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/gzip/index.rst#L57-L66 """ level: int diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 9701db8497..abf11b4211 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -1,7 +1,7 @@ """ Scale-offset codec types. -See https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/scale_offset +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md """ from typing import Final, Literal, NotRequired @@ -38,6 +38,7 @@ class ScaleOffsetCodecObject(TypedDict): `configuration` is itself optional per spec — when both `offset` and `scale` are at their identity defaults, the codec is a no-op and the entire `configuration` field may be omitted. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md#L18 and #L35 """ name: ScaleOffsetCodecName diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py index a8c9247ec4..ac40e78b7b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/sharding_indexed.py @@ -35,8 +35,10 @@ class ShardingIndexedCodecConfiguration(TypedDict): `index_codecs` is the codec pipeline applied to the shard index; it must be deterministic (no variable-size compression). + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L147-L155 `index_location` defaults to `"end"` per the spec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L157-L161 """ chunk_shape: tuple[int, ...] @@ -58,6 +60,8 @@ class ShardingIndexedCodecObject(TypedDict): The configuration has multiple required keys (`chunk_shape`, `codecs`, `index_codecs`), so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/sharding-indexed/index.rst#L141-L155 (required members) + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py index ac469b356a..41d9bcdc6d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/transpose.py @@ -38,6 +38,8 @@ class TransposeCodecObject(TypedDict): `order` is required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/transpose/index.rst#L60-L66 ("order: Required") + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py index c0faa64bed..9fdf177b4c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/zstd.py @@ -1,12 +1,12 @@ """ Zstandard codec types. -See https://github.com/zarr-developers/zarr-specs/pull/256 (unmerged at -time of writing; the configuration shape below reflects the proposed -specification). +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md +(the zarr-extensions registry entry; zarr-specs PR #256, which first +proposed the codec, was never merged). """ -from typing import Final, Literal +from typing import Final, Literal, NotRequired from typing_extensions import TypedDict @@ -21,11 +21,13 @@ class ZstdCodecConfiguration(TypedDict): """ Configuration for the Zarr v3 `zstd` codec. - Both fields are required per the proposed specification. + `level` is required; `checksum` is optional ("Should be omitted if + false"). + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 """ level: int - checksum: bool + checksum: NotRequired[bool] class ZstdCodecObject(TypedDict): @@ -38,8 +40,10 @@ class ZstdCodecObject(TypedDict): ZstdCodecMetadata = ZstdCodecObject """Permitted JSON shape for `zstd` codec metadata. -Both `level` and `checksum` are required, so only the object form is -valid; the short-hand-name form is not permitted by the spec for this codec. +`level` is required, so only the object form is valid; the short-hand-name +form is not permitted by the spec for this codec. + https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/zstd/README.md#L9-L19 + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 (short-hand names only "if no configuration metadata is required") """ __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py index a9fe0c1f8f..ba7c9aec0d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/consolidated.py @@ -25,8 +25,11 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): Inline consolidated metadata embedded in a v3 group. The `metadata` map contains only v3 array and group entries. V2 entries - are excluded from this interoperability convention by design; the v3 core - specification does not define consolidated metadata. + are excluded from this interoperability convention by design. The v3 core + specification acknowledges `consolidated_metadata` as a historical + additional field and fixes this envelope, but leaves the entries to the + reference implementation: + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ kind: Literal["inline"] @@ -38,8 +41,12 @@ class ZarrV3ConsolidatedMetadataJSON(TypedDict): """The key under which consolidated metadata is embedded in a v3 group document. Unlike the v2 `.zmetadata` file, this is not a store key: consolidated metadata -is carried as an extension field inside the group's own `zarr.json`. Like its v2 -counterpart it is a reference-implementation convention, not a spec artifact. +is carried as an additional field inside the group's own `zarr.json`. The core +spec names the field and its envelope ("For historical reasons, group metadata +documents may contain an additional field named ``consolidated_metadata``"); +the entry format, like the v2 counterpart, is a reference-implementation +convention. + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L802-L816 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py index c7eed64f0f..5892a1bdaa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/bytes.py @@ -1,7 +1,7 @@ """ Zarr `bytes` data type (variable-length raw bytes, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/bytes +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/bytes/README.md """ import re diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py index 41eec441df..264b2c262c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float16.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float16`.""" Float16SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat16 = NewType("HexFloat16", str) """A 6-character hex string (`0x` + 4 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float16(value: str) -> HexFloat16: CANONICAL_NAN_HEX_FLOAT16: Final = "0x7e00" """Canonical hex form of the float16 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py index 37b7d4f6e8..3b2e786f07 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float32.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float32`.""" Float32SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat32 = NewType("HexFloat32", str) """A 10-character hex string (`0x` + 8 hex digits) encoding the @@ -45,7 +48,7 @@ def hex_float32(value: str) -> HexFloat32: CANONICAL_NAN_HEX_FLOAT32: Final = "0x7fc00000" """Canonical hex form of the float32 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py index 9a5cf98288..21373d63f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/float64.py @@ -14,7 +14,10 @@ """Literal type of the `data_type` field for `float64`.""" Float64SpecialFillValue = Literal["NaN", "Infinity", "-Infinity"] -"""Named non-finite fill values permitted by the spec for IEEE 754 floats.""" +"""Named non-finite fill values permitted by the spec for IEEE 754 floats. + +https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 +""" HexFloat64 = NewType("HexFloat64", str) """An 18-character hex string (`0x` + 16 hex digits) encoding the @@ -46,7 +49,7 @@ def hex_float64(value: str) -> HexFloat64: CANONICAL_NAN_HEX_FLOAT64: Final = "0x7ff8000000000000" """Canonical hex form of the float64 NaN sentinel `"NaN"`. -Per spec the named `"NaN"` sentinel denotes the float with sign=0, the +Per spec (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L72-L74) the named `"NaN"` sentinel denotes the float with sign=0, the most significant mantissa bit set, and all other mantissa bits zero (the IEEE 754 default quiet NaN). Other NaN bit patterns must be encoded with the explicit hex-string form. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py index 8784160f71..bed264e96e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_datetime64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.datetime64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.datetime64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py index f5c8c77bf8..ceb31d8c40 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/numpy_timedelta64.py @@ -1,7 +1,7 @@ """ Zarr `numpy.timedelta64` data type (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/numpy.timedelta64 +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.timedelta64/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index c9c688c9fa..66e69c9b53 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -4,14 +4,19 @@ The `data_type` value is a string of the form `r` where `N` is a positive multiple of 8 (e.g. `r8`, `r16`, `r24`). -See https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html +See https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html +(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47; fill value: https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L97-L99) """ import re from typing import Final, NewType RawBytesDataTypeName = NewType("RawBytesDataTypeName", str) -"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`).""" +"""A spec-conformant `r` raw-bytes name (e.g. `"r8"`, `"r16"`). + +"raw bits, variable size given by *, limited to be a multiple of 8": + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L46-L47 +""" _RAW_BYTES_RE: Final = re.compile(r"^r(\d+)$") diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py index 0a778ccecc..1e93a95d50 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/string.py @@ -1,7 +1,7 @@ """ Zarr `string` data type (variable-length utf-8, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/tree/main/data-types/string +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/string/README.md """ from typing import Final, Literal diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index b1b6b50308..5795c927f5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -1,7 +1,7 @@ """ Zarr `struct` data type (heterogeneous record, zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/blob/main/data-types/struct/README.md +See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/struct/README.md """ from collections.abc import Mapping diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 692cbe76cc..c30f457618 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -180,6 +180,7 @@ def test_json_value_type_accepts_json_shapes() -> None: def test_string_nan_fill_value_roundtrips() -> None: # Non-finite floats are represented as the spec strings ("NaN", "Infinity", # "-Infinity") by the caller — the metadata layer does not interpret dtypes. + # https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/data-types/index.rst#L63-L79 # The string form round-trips cleanly under default dataclass equality, # unlike a raw float('nan') (which is an invalid fill_value the caller must # not pass). @@ -1648,7 +1649,10 @@ def test_must_understand_fields_partition() -> None: """must_understand_fields contains every extra field not explicitly waived with must_understand: false, including implicitly-true and non-mapping fields, so a reader can discharge the spec's fail-to-open duty by - subtracting the extensions it recognizes.""" + subtracting the extensions it recognizes. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1575-L1578 + """ model = ZarrV3ArrayMetadata.create_default( extra_fields={ "ext_a": {"name": "a", "must_understand": False}, @@ -1674,7 +1678,10 @@ def test_dimension_names_null_field_rejected() -> None: """A dimension_names field whose VALUE is null is invalid: the spec permits null as an element (an unnamed dimension), never as the field value — "not specified" is spelled by omitting the key. Consumers bridging from an - in-memory None sentinel must drop the key, not write null.""" + in-memory None sentinel must drop the key, not write null. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L635-L638 + """ doc = dict(ZarrV3ArrayMetadata.create_default().to_json()) | {"dimension_names": None} problems = validate_array_metadata_v3(doc) assert [(p.loc, p.kind) for p in problems] == [(("dimension_names",), "invalid_type")] diff --git a/packages/zarr-metadata/tests/model/test_group.py b/packages/zarr-metadata/tests/model/test_group.py index 4b8c22b84d..d280091ba6 100644 --- a/packages/zarr-metadata/tests/model/test_group.py +++ b/packages/zarr-metadata/tests/model/test_group.py @@ -503,7 +503,10 @@ def test_v2_consolidated_rejects_unknown_document_member() -> None: def test_group_must_understand_fields_partition() -> None: """The group model partitions extra fields by the spec's implicit-true rule, - like the array model.""" + like the array model. + + https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1571-L1573 + """ model = ZarrV3GroupMetadata.create_default( extra_fields={ "waived": {"name": "w", "must_understand": False},