From 49c7993f9af2344a0e86ea191b2a46e132607254 Mon Sep 17 00:00:00 2001 From: codeofwxz Date: Sat, 12 Sep 2026 20:37:49 +0800 Subject: [PATCH 1/2] Normalize missing JSON fields before generating content keys --- csv_diff/__init__.py | 5 ++++ tests/test_json_normalization.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/test_json_normalization.py diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py index 59a2eaf..c974abb 100644 --- a/csv_diff/__init__.py +++ b/csv_diff/__init__.py @@ -35,6 +35,11 @@ def load_json(fp, key=None): if key: keyfn = lambda r: r[key] else: + # Missing fields and explicit nulls should produce the same row key. + # Keep nested values intact until after hashing. + for row in raw_list: + for field in common_keys: + row.setdefault(field, None) keyfn = lambda r: hashlib.sha1( json.dumps(r, sort_keys=True).encode("utf8") ).hexdigest() diff --git a/tests/test_json_normalization.py b/tests/test_json_normalization.py new file mode 100644 index 0000000..a52bac1 --- /dev/null +++ b/tests/test_json_normalization.py @@ -0,0 +1,41 @@ +from csv_diff import compare, load_json +import io +import json +import pytest + + +@pytest.mark.parametrize("key", [None, "id"]) +def test_missing_and_null_values_compare_equal(key): + previous = [{"id": 1}, {"id": 2, "name": "Cleo"}] + current = [{"id": 1, "name": None}, {"id": 2, "name": "Cleo"}] + result = compare( + load_json(io.StringIO(json.dumps(previous)), key=key), + load_json(io.StringIO(json.dumps(current)), key=key), + ) + assert result == { + "added": [], + "removed": [], + "changed": [], + "columns_added": [], + "columns_removed": [], + } + + +def test_equivalent_normalized_rows_are_deduplicated_without_key(): + rows = [{"id": 1}, {"id": 1, "name": None}] + result = load_json(io.StringIO(json.dumps(rows))) + assert list(result.values()) == [{"id": 1, "name": None}] + + +def test_missing_explicit_key_is_not_filled_with_null(): + rows = [{"id": 1}, {"name": "Cleo"}] + with pytest.raises(KeyError): + load_json(io.StringIO(json.dumps(rows)), key="id") + + +@pytest.mark.parametrize("value", [{"name": "Cleo"}, ["Cleo", None]]) +def test_nested_values_remain_distinct_from_json_strings(value): + rows = [{"value": value}, {"value": json.dumps(value)}] + result = load_json(io.StringIO(json.dumps(rows))) + assert len(result) == 2 + assert list(result.values()) == [{"value": json.dumps(value)}] * 2 From f4c858be1c0760d04b96c04802d5425811e2184b Mon Sep 17 00:00:00 2001 From: codeofwxz Date: Sat, 12 Sep 2026 22:04:56 +0800 Subject: [PATCH 2/2] Keep JSON content keys independent of unrelated columns --- csv_diff/__init__.py | 12 +++---- tests/test_json_normalization.py | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py index c974abb..b222da9 100644 --- a/csv_diff/__init__.py +++ b/csv_diff/__init__.py @@ -35,13 +35,13 @@ def load_json(fp, key=None): if key: keyfn = lambda r: r[key] else: - # Missing fields and explicit nulls should produce the same row key. - # Keep nested values intact until after hashing. - for row in raw_list: - for field in common_keys: - row.setdefault(field, None) + # Missing fields and explicit nulls have the same row identity, + # independent of the other rows' columns. Keep nested values intact. keyfn = lambda r: hashlib.sha1( - json.dumps(r, sort_keys=True).encode("utf8") + json.dumps( + {field: value for field, value in r.items() if value is not None}, + sort_keys=True, + ).encode("utf8") ).hexdigest() return {keyfn(r): _simplify_json_row(r, common_keys) for r in raw_list} diff --git a/tests/test_json_normalization.py b/tests/test_json_normalization.py index a52bac1..86b72eb 100644 --- a/tests/test_json_normalization.py +++ b/tests/test_json_normalization.py @@ -39,3 +39,64 @@ def test_nested_values_remain_distinct_from_json_strings(value): result = load_json(io.StringIO(json.dumps(rows))) assert len(result) == 2 assert list(result.values()) == [{"value": json.dumps(value)}] * 2 + + +def test_unrelated_column_changes_do_not_replace_identical_rows(): + result = compare( + load_json(io.StringIO('[{"id": 1}, {"a": 0}]')), + load_json(io.StringIO('[{"id": 1}, {"b": 0}]')), + ) + assert result == { + "added": [{"b": 0, "id": None}], + "removed": [{"a": 0, "id": None}], + "changed": [], + "columns_added": ["b"], + "columns_removed": ["a"], + } + + +@pytest.mark.parametrize("value", [None, 0, False, "", [], {}]) +def test_new_column_preserves_non_null_row_differences(value): + previous = [{"id": 1}] + current = [{"id": 1, "extra": value}] + result = compare( + load_json(io.StringIO(json.dumps(previous))), + load_json(io.StringIO(json.dumps(current))), + ) + displayed = json.dumps(value) if isinstance(value, (list, dict)) else value + assert result == { + "added": [] if value is None else [{"id": 1, "extra": displayed}], + "removed": [] if value is None else previous, + "changed": [], + "columns_added": ["extra"], + "columns_removed": [], + } + + +def test_nested_null_fields_are_not_ignored(): + result = compare( + load_json(io.StringIO('[{"value": {}}]')), + load_json(io.StringIO('[{"value": {"field": null}}]')), + ) + assert result == { + "added": [{"value": '{"field": null}'}], + "removed": [{"value": "{}"}], + "changed": [], + "columns_added": [], + "columns_removed": [], + } + + +def test_explicit_key_still_uses_last_duplicate_row(): + rows = [{"id": 1, "value": "old"}, {"id": 1, "value": None}] + assert load_json(io.StringIO(json.dumps(rows)), key="id") == { + 1: {"id": 1, "value": None} + } + + +def test_empty_key_argument_still_uses_content_keys(): + source = '[{"": "same", "value": 0}, {"": "same", "value": false}]' + result = load_json(io.StringIO(source), key="") + assert result == load_json(io.StringIO(source)) + assert len(result) == 2 + assert "same" not in result