From bc18f08a32f94da944742f88baa15f72019cdacc Mon Sep 17 00:00:00 2001 From: codeofwxz Date: Sat, 12 Sep 2026 17:59:04 +0800 Subject: [PATCH 1/2] Reject duplicate explicit row keys in CSV and JSON inputs --- README.md | 2 ++ csv_diff/__init__.py | 21 +++++++++++++++++ csv_diff/cli.py | 17 ++++++++------ tests/test_cli.py | 32 +++++++++++++++++++++++++ tests/test_csv_diff.py | 53 +++++++++++++++++++++++++++++++++++++++++- 5 files changed, 117 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 61c9acf..4b03976 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Consider two CSV files: The `--key=id` option means that the `id` column should be treated as the unique key, to identify which records have changed. +When `--key` is specified, duplicate key values within either input are rejected, even if the rows are identical. The error identifies the input file, key column and value, and the first and repeated data row numbers (counting from 1, excluding a CSV/TSV header). These are record numbers, not physical line numbers; a quoted CSV value may span multiple lines. This applies to CSV, TSV and JSON inputs. The Python loaders raise `csv_diff.DuplicateKeyError`, a subclass of `ValueError`. Without `--key`, identical rows continue to be deduplicated by their contents. + The tool will automatically detect if your files are comma- or tab-separated. You can over-ride this automatic detection and force the tool to use a specific format using `--format=tsv` or `--format=csv`. You can also feed it JSON files, provided they are a JSON array of objects where each object has the same keys. Use `--format=json` if your input files are JSON. diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py index 59a2eaf..1db7859 100644 --- a/csv_diff/__init__.py +++ b/csv_diff/__init__.py @@ -4,6 +4,25 @@ import hashlib +class DuplicateKeyError(ValueError): + """An explicitly selected row key occurs more than once in an input.""" + + +def _validate_unique_keys(rows, key): + """Reject repeated explicit keys, reporting one-based data row numbers.""" + seen = {} + for row_number, row in enumerate(rows, 1): + value = row[key] + if value in seen: + raise DuplicateKeyError( + "Duplicate key {!r} in column {!r} at data row {} " + "(first seen at data row {}).".format( + value, key, row_number, seen[value] + ) + ) + seen[value] = row_number + + def load_csv(fp, key=None, dialect=None): if dialect is None and fp.seekable(): # Peek at first 1MB to sniff the delimiter and other dialect details @@ -18,6 +37,7 @@ def load_csv(fp, key=None, dialect=None): headings = next(fp) rows = [dict(zip(headings, line)) for line in fp] if key: + _validate_unique_keys(rows, key) keyfn = lambda r: r[key] else: keyfn = lambda r: hashlib.sha1( @@ -33,6 +53,7 @@ def load_json(fp, key=None): for item in raw_list: common_keys.update(item.keys()) if key: + _validate_unique_keys(raw_list, key) keyfn = lambda r: r[key] else: keyfn = lambda r: hashlib.sha1( diff --git a/csv_diff/cli.py b/csv_diff/cli.py index 261ef6d..2ca7a13 100644 --- a/csv_diff/cli.py +++ b/csv_diff/cli.py @@ -1,6 +1,6 @@ import click import json as std_json -from . import load_csv, load_json, compare, human_text +from . import load_csv, load_json, compare, human_text, DuplicateKeyError @click.command() @@ -63,12 +63,15 @@ def cli(previous, current, key, format, json, singular, plural, show_unchanged, ) def load(filename): - if format == "json": - return load_json(open(filename), key=key) - else: - return load_csv( - open(filename, newline=""), key=key, dialect=dialect.get(format) - ) + try: + if format == "json": + return load_json(open(filename), key=key) + else: + return load_csv( + open(filename, newline=""), key=key, dialect=dialect.get(format) + ) + except DuplicateKeyError as error: + raise click.ClickException("{}: {}".format(filename, error)) from error previous_data = load(previous) current_data = load(current) diff --git a/tests/test_cli.py b/tests/test_cli.py index 12f4e12..3250245 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -300,3 +300,35 @@ def test_diff_with_extras(tmpdir): """ ).strip() assert result.output.strip() == expected + + +@pytest.mark.parametrize("format", ["csv", "tsv", "json"]) +@pytest.mark.parametrize("duplicate_in", ["previous", "current"]) +def test_cli_reports_duplicate_key_with_input_filename(tmpdir, format, duplicate_in): + paths = [] + for side in ("previous", "current"): + path = tmpdir / "{}.{}".format(side, format) + if format == "json": + rows = [{"id": 1, "name": "first"}] + if side == duplicate_in: + rows.append({"id": 1, "name": "last"}) + content = json.dumps(rows) + else: + delimiter = "\t" if format == "tsv" else "," + content = "id{0}name\n1{0}first\n".format(delimiter) + if side == duplicate_in: + content += "1{}last\n".format(delimiter) + path.write(content) + paths.append(str(path)) + result = CliRunner().invoke( + cli.cli, + paths + ["--key", "id", "--format", format, "--json"], + catch_exceptions=False, + ) + assert result.exit_code == 1 + assert result.output.startswith("Error:") + assert "{}.{}".format(duplicate_in, format) in result.output + assert "Duplicate key" in result.output + assert "column 'id'" in result.output + assert "data row 2" in result.output + assert "first seen at data row 1" in result.output diff --git a/tests/test_csv_diff.py b/tests/test_csv_diff.py index 0e3670f..3d567e9 100644 --- a/tests/test_csv_diff.py +++ b/tests/test_csv_diff.py @@ -1,5 +1,7 @@ -from csv_diff import load_csv, compare +from csv_diff import load_csv, load_json, compare import io +import json +import pytest ONE = """id,name,age 1,Cleo,4 @@ -115,3 +117,52 @@ def test_tsv(): "columns_added": [], "columns_removed": [], } == diff + + +@pytest.mark.parametrize( + "content, key_value, duplicate_row", + [ + ("a,b,c,d\n1,2,3,4\n1,2,3\n3,2,3,4", "1", 2), + ("a,b\n1,first\n1,last", "1", 2), + ("a,b\n1,same\n1,same", "1", 2), + ('a,b\n,"first\ncontinued"\n2,other\n,last', "", 3), + ], +) +def test_load_csv_rejects_duplicate_explicit_keys(content, key_value, duplicate_row): + with pytest.raises(ValueError, match="Duplicate key") as error: + load_csv(io.StringIO(content), key="a") + message = str(error.value) + assert repr(key_value) in message + assert "column 'a'" in message + assert "data row {}".format(duplicate_row) in message + assert "first seen at data row 1" in message + + +@pytest.mark.parametrize("key_value", ["1", 0, None]) +def test_load_json_rejects_duplicate_explicit_keys(key_value): + content = json.dumps( + [{"id": key_value, "name": "first"}, {"id": key_value, "name": "last"}] + ) + with pytest.raises(ValueError, match="Duplicate key") as error: + load_json(io.StringIO(content), key="id") + message = str(error.value) + assert repr(key_value) in message + assert "column 'id'" in message + assert "data row 2" in message + assert "first seen at data row 1" in message + + +@pytest.mark.parametrize( + "loader, content, expected", + [ + (load_csv, "id,name\n1,same\n1,same", {"id": "1", "name": "same"}), + ( + load_json, + '[{"id": 1, "name": "same"}, {"id": 1, "name": "same"}]', + {"id": 1, "name": "same"}, + ), + ], +) +def test_load_without_key_still_deduplicates_identical_rows(loader, content, expected): + rows = loader(io.StringIO(content)) + assert list(rows.values()) == [expected] From 89a5ae4fa9439a16346f8a56ea8d18db4176eb4d Mon Sep 17 00:00:00 2001 From: codeofwxz Date: Sat, 12 Sep 2026 18:22:19 +0800 Subject: [PATCH 2/2] Handle explicitly empty key names and tighten exception tests --- README.md | 2 ++ csv_diff/__init__.py | 4 ++-- tests/test_cli.py | 15 +++++++------ tests/test_csv_diff.py | 48 +++++++++++++++++++++++++++++++----------- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4b03976..7df1946 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,8 @@ The `--key=id` option means that the `id` column should be treated as the unique When `--key` is specified, duplicate key values within either input are rejected, even if the rows are identical. The error identifies the input file, key column and value, and the first and repeated data row numbers (counting from 1, excluding a CSV/TSV header). These are record numbers, not physical line numbers; a quoted CSV value may span multiple lines. This applies to CSV, TSV and JSON inputs. The Python loaders raise `csv_diff.DuplicateKeyError`, a subclass of `ValueError`. Without `--key`, identical rows continue to be deduplicated by their contents. +An empty key name (`--key=` on the CLI, or `key=""` in Python) selects an empty-named column or JSON property. Only omitting the key, or passing `key=None` in Python, uses content-based indexing. + The tool will automatically detect if your files are comma- or tab-separated. You can over-ride this automatic detection and force the tool to use a specific format using `--format=tsv` or `--format=csv`. You can also feed it JSON files, provided they are a JSON array of objects where each object has the same keys. Use `--format=json` if your input files are JSON. diff --git a/csv_diff/__init__.py b/csv_diff/__init__.py index 1db7859..64bb839 100644 --- a/csv_diff/__init__.py +++ b/csv_diff/__init__.py @@ -36,7 +36,7 @@ def load_csv(fp, key=None, dialect=None): fp = csv.reader(fp, dialect=(dialect or "excel")) headings = next(fp) rows = [dict(zip(headings, line)) for line in fp] - if key: + if key is not None: _validate_unique_keys(rows, key) keyfn = lambda r: r[key] else: @@ -52,7 +52,7 @@ def load_json(fp, key=None): common_keys = set() for item in raw_list: common_keys.update(item.keys()) - if key: + if key is not None: _validate_unique_keys(raw_list, key) keyfn = lambda r: r[key] else: diff --git a/tests/test_cli.py b/tests/test_cli.py index 3250245..aca3244 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -304,31 +304,34 @@ def test_diff_with_extras(tmpdir): @pytest.mark.parametrize("format", ["csv", "tsv", "json"]) @pytest.mark.parametrize("duplicate_in", ["previous", "current"]) -def test_cli_reports_duplicate_key_with_input_filename(tmpdir, format, duplicate_in): +@pytest.mark.parametrize("key_name", ["id", ""]) +def test_cli_reports_duplicate_key_with_input_filename( + tmpdir, format, duplicate_in, key_name +): paths = [] for side in ("previous", "current"): path = tmpdir / "{}.{}".format(side, format) if format == "json": - rows = [{"id": 1, "name": "first"}] + rows = [{key_name: 1, "name": "first"}] if side == duplicate_in: - rows.append({"id": 1, "name": "last"}) + rows.append({key_name: 1, "name": "last"}) content = json.dumps(rows) else: delimiter = "\t" if format == "tsv" else "," - content = "id{0}name\n1{0}first\n".format(delimiter) + content = "{1}{0}name\n1{0}first\n".format(delimiter, key_name) if side == duplicate_in: content += "1{}last\n".format(delimiter) path.write(content) paths.append(str(path)) result = CliRunner().invoke( cli.cli, - paths + ["--key", "id", "--format", format, "--json"], + paths + ["--key", key_name, "--format", format, "--json"], catch_exceptions=False, ) assert result.exit_code == 1 assert result.output.startswith("Error:") assert "{}.{}".format(duplicate_in, format) in result.output assert "Duplicate key" in result.output - assert "column 'id'" in result.output + assert "column {!r}".format(key_name) in result.output assert "data row 2" in result.output assert "first seen at data row 1" in result.output diff --git a/tests/test_csv_diff.py b/tests/test_csv_diff.py index 3d567e9..16660c6 100644 --- a/tests/test_csv_diff.py +++ b/tests/test_csv_diff.py @@ -1,4 +1,4 @@ -from csv_diff import load_csv, load_json, compare +from csv_diff import DuplicateKeyError, load_csv, load_json, compare import io import json import pytest @@ -128,26 +128,33 @@ def test_tsv(): ('a,b\n,"first\ncontinued"\n2,other\n,last', "", 3), ], ) -def test_load_csv_rejects_duplicate_explicit_keys(content, key_value, duplicate_row): - with pytest.raises(ValueError, match="Duplicate key") as error: - load_csv(io.StringIO(content), key="a") +@pytest.mark.parametrize("key_name", ["a", ""]) +def test_load_csv_rejects_duplicate_explicit_keys( + content, key_value, duplicate_row, key_name +): + content = content.replace("a,", key_name + ",", 1) + with pytest.raises(DuplicateKeyError, match="Duplicate key") as error: + load_csv(io.StringIO(content), key=key_name) + assert isinstance(error.value, ValueError) message = str(error.value) assert repr(key_value) in message - assert "column 'a'" in message + assert "column {!r}".format(key_name) in message assert "data row {}".format(duplicate_row) in message assert "first seen at data row 1" in message @pytest.mark.parametrize("key_value", ["1", 0, None]) -def test_load_json_rejects_duplicate_explicit_keys(key_value): +@pytest.mark.parametrize("key_name", ["id", ""]) +def test_load_json_rejects_duplicate_explicit_keys(key_value, key_name): content = json.dumps( - [{"id": key_value, "name": "first"}, {"id": key_value, "name": "last"}] + [{key_name: key_value, "name": "first"}, {key_name: key_value, "name": "last"}] ) - with pytest.raises(ValueError, match="Duplicate key") as error: - load_json(io.StringIO(content), key="id") + with pytest.raises(DuplicateKeyError, match="Duplicate key") as error: + load_json(io.StringIO(content), key=key_name) + assert isinstance(error.value, ValueError) message = str(error.value) assert repr(key_value) in message - assert "column 'id'" in message + assert "column {!r}".format(key_name) in message assert "data row 2" in message assert "first seen at data row 1" in message @@ -163,6 +170,23 @@ def test_load_json_rejects_duplicate_explicit_keys(key_value): ), ], ) -def test_load_without_key_still_deduplicates_identical_rows(loader, content, expected): - rows = loader(io.StringIO(content)) +@pytest.mark.parametrize("kwargs", [{}, {"key": None}]) +def test_load_without_key_still_deduplicates_identical_rows( + loader, content, expected, kwargs +): + rows = loader(io.StringIO(content), **kwargs) assert list(rows.values()) == [expected] + + +@pytest.mark.parametrize( + "loader, content", + [ + (load_csv, ",name\n1,first\n2,last"), + (load_json, '[{"": "1", "name": "first"}, {"": "2", "name": "last"}]'), + ], +) +def test_load_unique_empty_name_key(loader, content): + assert loader(io.StringIO(content), key="") == { + "1": {"": "1", "name": "first"}, + "2": {"": "2", "name": "last"}, + }