From 5cfa048b9d8416b55155d130d3fabf6745275454 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:39:05 +0100 Subject: [PATCH 1/9] fix(python): open the output only once the column is found --- python/src/ng_postcode/convert.py | 15 ++++++++++++--- python/tests/test_convert.py | 10 +++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index a754fde..75357f6 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -70,11 +70,18 @@ def rows( yield row | added -def write(source: TextIO, target: TextIO, column: str, confirm: Confirm | None) -> Counter[str]: - """Convert `source` to `target` and count the rows by status.""" +def read(source: TextIO, column: str) -> csv.DictReader[str]: + """The rows of `source`, once its header is known to hold `column`.""" reader = csv.DictReader(source) if column not in (reader.fieldnames or []): sys.exit(f"ng-postcode: no column named {column!r}; found {reader.fieldnames}") + return reader + + +def write( + reader: csv.DictReader[str], target: TextIO, column: str, confirm: Confirm | None +) -> Counter[str]: + """Convert the rows of `reader` to `target` and count them by status.""" writer: csv.DictWriter[str] | None = None counts: Counter[str] = Counter() for row in rows(reader, column, confirm): @@ -128,12 +135,14 @@ def main(argv: list[str] | None = None) -> None: if args.file == "-" else files.enter_context(open(args.file, newline="", encoding="utf-8-sig")) ) + reader = read(source, args.column) + # Opening the output empties it, so it waits until the input is known to be usable. target = ( files.enter_context(open(args.out, "w", newline="", encoding="utf-8")) if args.out else sys.stdout ) - counts = write(source, target, args.column, confirm) + counts = write(reader, target, args.column, confirm) parts = [ f"{counts[status]:,} {status}" for status in ("valid", "suggested", "invalid", "empty") ] diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index 16db4b5..1875da0 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -8,7 +8,7 @@ import pytest from ng_postcode import Postcode -from ng_postcode.convert import Converted, convert, main, nipost, rows, write +from ng_postcode.convert import Converted, convert, main, nipost, read, rows @pytest.mark.parametrize( @@ -52,12 +52,16 @@ def test_converts_a_file_and_reports_the_counts( def test_a_missing_column_or_key_stops_before_anything_is_written(tmp_path: Path) -> None: - source = tmp_path / "in.csv" + source, target = tmp_path / "in.csv", tmp_path / "out.csv" source.write_text("id,pc\n1,x\n", encoding="utf-8") + target.write_text("kept", encoding="utf-8") + with pytest.raises(SystemExit, match="no column named 'postcode'"): + read(io.StringIO("id,pc\n1,x\n"), "postcode") with pytest.raises(SystemExit, match="no column named 'postcode'"): - write(source.open(), io.StringIO(), "postcode", None) + main([str(source), "--column", "postcode", "--out", str(target)]) with pytest.raises(SystemExit): main([str(source), "--column", "pc", "--confirm"]) + assert target.read_text(encoding="utf-8") == "kept" def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( From 5b65d17e1d425d1f6fb54e94fb133b0000f059ed Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:39:05 +0100 Subject: [PATCH 2/9] fix(python): refuse to write the converted CSV over its input --- python/src/ng_postcode/convert.py | 3 +++ python/tests/test_convert.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index 75357f6..e65ae18 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -137,6 +137,9 @@ def main(argv: list[str] | None = None) -> None: ) reader = read(source, args.column) # Opening the output empties it, so it waits until the input is known to be usable. + same = args.out and args.file != "-" and os.path.exists(args.out) + if same and os.path.samefile(args.file, args.out): + sys.exit("ng-postcode: --out is the input file, which writing would empty") target = ( files.enter_context(open(args.out, "w", newline="", encoding="utf-8")) if args.out diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index 1875da0..40267db 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -64,6 +64,17 @@ def test_a_missing_column_or_key_stops_before_anything_is_written(tmp_path: Path assert target.read_text(encoding="utf-8") == "kept" +def test_the_input_file_is_never_written_over(tmp_path: Path) -> None: + source = tmp_path / "in.csv" + source.write_text("id,pc\n1,EK01A03FK01\n", encoding="utf-8") + link = tmp_path / "link.csv" + link.symlink_to(source) + for out in (source, link): + with pytest.raises(SystemExit, match="--out is the input file"): + main([str(source), "--column", "pc", "--out", str(out)]) + assert source.read_text(encoding="utf-8") == "id,pc\n1,EK01A03FK01\n" + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From a937d97ce8de9fce33d5a04e89debb4272647823 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:39:17 +0100 Subject: [PATCH 3/9] fix(python): keep the header when a CSV has no rows --- python/src/ng_postcode/convert.py | 9 +++++---- python/tests/test_convert.py | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index e65ae18..16ee3bd 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -82,12 +82,13 @@ def write( reader: csv.DictReader[str], target: TextIO, column: str, confirm: Confirm | None ) -> Counter[str]: """Convert the rows of `reader` to `target` and count them by status.""" - writer: csv.DictWriter[str] | None = None + added = ["compact", "status", "suggestion", "error", *(["assigned"] if confirm else [])] + names = [*(reader.fieldnames or []), *(f"{column}_{name}" for name in added)] + writer = csv.DictWriter(target, fieldnames=names, lineterminator="\n") + # Written before any row, so a file with none keeps its header. + writer.writeheader() counts: Counter[str] = Counter() for row in rows(reader, column, confirm): - if writer is None: - writer = csv.DictWriter(target, fieldnames=list(row), lineterminator="\n") - writer.writeheader() writer.writerow(row) counts[row[f"{column}_status"]] += 1 return counts diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index 40267db..345c067 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -75,6 +75,15 @@ def test_the_input_file_is_never_written_over(tmp_path: Path) -> None: assert source.read_text(encoding="utf-8") == "id,pc\n1,EK01A03FK01\n" +def test_a_file_with_no_rows_keeps_its_header(tmp_path: Path) -> None: + source, target = tmp_path / "in.csv", tmp_path / "out.csv" + source.write_text("id,pc\n", encoding="utf-8") + main([str(source), "--column", "pc", "--out", str(target)]) + assert target.read_text(encoding="utf-8") == ( + "id,pc,pc_compact,pc_status,pc_suggestion,pc_error\n" + ) + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From b636d6e080bbf907bdd70b6f2e01c9c44477570c Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:39:30 +0100 Subject: [PATCH 4/9] fix(python): stop at a row with more cells than the header --- python/src/ng_postcode/convert.py | 3 +++ python/tests/test_convert.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index 16ee3bd..a6965fd 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -89,6 +89,9 @@ def write( writer.writeheader() counts: Counter[str] = Counter() for row in rows(reader, column, confirm): + # The reader files cells past the header under None, and no column can take them. + if None in row: + sys.exit(f"ng-postcode: line {reader.line_num} has more cells than the header") writer.writerow(row) counts[row[f"{column}_status"]] += 1 return counts diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index 345c067..ee2f044 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -2,13 +2,14 @@ import csv import io +import sys from pathlib import Path import httpx import pytest from ng_postcode import Postcode -from ng_postcode.convert import Converted, convert, main, nipost, read, rows +from ng_postcode.convert import Converted, convert, main, nipost, read, rows, write @pytest.mark.parametrize( @@ -84,6 +85,18 @@ def test_a_file_with_no_rows_keeps_its_header(tmp_path: Path) -> None: ) +def test_a_row_longer_than_the_header_stops_the_run(tmp_path: Path) -> None: + source = tmp_path / "in.csv" + source.write_text("id,pc\n1,EK01A03FK01\n2,x,extra\n", encoding="utf-8") + with pytest.raises(SystemExit, match="line 3 has more cells than the header"): + main([str(source), "--column", "pc", "--out", str(tmp_path / "out.csv")]) + + +def test_a_row_shorter_than_the_header_is_empty(capsys: pytest.CaptureFixture[str]) -> None: + write(read(io.StringIO("id,pc\n1\n"), "pc"), sys.stdout, "pc", None) + assert capsys.readouterr().out.splitlines()[1] == "1,,,empty,," + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From 66b419ea4232418bc89f02a07f7c82b2ad3a6b62 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:39:45 +0100 Subject: [PATCH 5/9] fix(python): refuse a CSV that already has a column the run adds --- python/src/ng_postcode/convert.py | 12 ++++++++++-- python/tests/test_convert.py | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index a6965fd..d2e89b4 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -75,15 +75,23 @@ def read(source: TextIO, column: str) -> csv.DictReader[str]: reader = csv.DictReader(source) if column not in (reader.fieldnames or []): sys.exit(f"ng-postcode: no column named {column!r}; found {reader.fieldnames}") + taken = [name for name in reader.fieldnames or [] if name in new_columns(column, True)] + if taken: + sys.exit(f"ng-postcode: the file already has a column named {taken[0]!r}") return reader +def new_columns(column: str, confirming: bool) -> list[str]: + """The columns a run adds, which the file must not already hold.""" + names = ["compact", "status", "suggestion", "error", *(["assigned"] if confirming else [])] + return [f"{column}_{name}" for name in names] + + def write( reader: csv.DictReader[str], target: TextIO, column: str, confirm: Confirm | None ) -> Counter[str]: """Convert the rows of `reader` to `target` and count them by status.""" - added = ["compact", "status", "suggestion", "error", *(["assigned"] if confirm else [])] - names = [*(reader.fieldnames or []), *(f"{column}_{name}" for name in added)] + names = [*(reader.fieldnames or []), *new_columns(column, confirm is not None)] writer = csv.DictWriter(target, fieldnames=names, lineterminator="\n") # Written before any row, so a file with none keeps its header. writer.writeheader() diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index ee2f044..c8cdd08 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -97,6 +97,11 @@ def test_a_row_shorter_than_the_header_is_empty(capsys: pytest.CaptureFixture[st assert capsys.readouterr().out.splitlines()[1] == "1,,,empty,," +def test_a_column_the_run_would_add_is_never_replaced() -> None: + with pytest.raises(SystemExit, match="already has a column named 'pc_status'"): + read(io.StringIO("pc,pc_status\nEK01A03FK01,shipped\n"), "pc") + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From 1d17f93b9811c7fcd2bc40dd68de5f9ba77248d7 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:40:41 +0100 Subject: [PATCH 6/9] fix(python): refuse a CSV whose header repeats a name --- python/src/ng_postcode/convert.py | 6 +++++- python/tests/test_convert.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index d2e89b4..1a56dff 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -75,7 +75,11 @@ def read(source: TextIO, column: str) -> csv.DictReader[str]: reader = csv.DictReader(source) if column not in (reader.fieldnames or []): sys.exit(f"ng-postcode: no column named {column!r}; found {reader.fieldnames}") - taken = [name for name in reader.fieldnames or [] if name in new_columns(column, True)] + names = list(reader.fieldnames or []) + repeated = [name for name in names if names.count(name) > 1] + if repeated: + sys.exit(f"ng-postcode: the header names {repeated[0]!r} more than once") + taken = [name for name in names if name in new_columns(column, True)] if taken: sys.exit(f"ng-postcode: the file already has a column named {taken[0]!r}") return reader diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index c8cdd08..db98bd9 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -102,6 +102,11 @@ def test_a_column_the_run_would_add_is_never_replaced() -> None: read(io.StringIO("pc,pc_status\nEK01A03FK01,shipped\n"), "pc") +def test_a_header_that_repeats_a_name_is_refused() -> None: + with pytest.raises(SystemExit, match="names 'id' more than once"): + read(io.StringIO("id,pc,id\n1,EK01A03FK01,2\n"), "pc") + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From 0f62b2349c8d2144eb8366c4c3222c158b8ebed7 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:40:53 +0100 Subject: [PATCH 7/9] fix(python): report a file that cannot be read without a traceback --- python/src/ng_postcode/convert.py | 41 +++++++++++++++++-------------- python/tests/test_convert.py | 9 +++++++ 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index 1a56dff..5dcdfdf 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -144,24 +144,29 @@ def main(argv: list[str] | None = None) -> None: if args.confirm and not key: cli.error("--confirm needs a NIPOST API key in NG_POSTCODE_API_KEY") confirm = nipost(key) if args.confirm else None - with contextlib.ExitStack() as files: - # utf-8-sig drops the byte order mark that spreadsheet exports begin with. - source = ( - sys.stdin - if args.file == "-" - else files.enter_context(open(args.file, newline="", encoding="utf-8-sig")) - ) - reader = read(source, args.column) - # Opening the output empties it, so it waits until the input is known to be usable. - same = args.out and args.file != "-" and os.path.exists(args.out) - if same and os.path.samefile(args.file, args.out): - sys.exit("ng-postcode: --out is the input file, which writing would empty") - target = ( - files.enter_context(open(args.out, "w", newline="", encoding="utf-8")) - if args.out - else sys.stdout - ) - counts = write(reader, target, args.column, confirm) + try: + with contextlib.ExitStack() as files: + # utf-8-sig drops the byte order mark that spreadsheet exports begin with. + source = ( + sys.stdin + if args.file == "-" + else files.enter_context(open(args.file, newline="", encoding="utf-8-sig")) + ) + reader = read(source, args.column) + # Opening the output empties it, so it waits until the input is known to be usable. + same = args.out and args.file != "-" and os.path.exists(args.out) + if same and os.path.samefile(args.file, args.out): + sys.exit("ng-postcode: --out is the input file, which writing would empty") + target = ( + files.enter_context(open(args.out, "w", newline="", encoding="utf-8")) + if args.out + else sys.stdout + ) + counts = write(reader, target, args.column, confirm) + except UnicodeDecodeError: + sys.exit(f"ng-postcode: {args.file} is not UTF-8; save it as CSV UTF-8 and run again") + except OSError as error: + sys.exit(f"ng-postcode: {error}") parts = [ f"{counts[status]:,} {status}" for status in ("valid", "suggested", "invalid", "empty") ] diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index db98bd9..3e8b98d 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -107,6 +107,15 @@ def test_a_header_that_repeats_a_name_is_refused() -> None: read(io.StringIO("id,pc,id\n1,EK01A03FK01,2\n"), "pc") +def test_a_file_that_cannot_be_read_is_reported_in_one_line(tmp_path: Path) -> None: + with pytest.raises(SystemExit, match=r"ng-postcode: .*No such file"): + main([str(tmp_path / "missing.csv"), "--column", "pc"]) + latin = tmp_path / "latin.csv" + latin.write_bytes("id,pc\n1,caf\xe9\n".encode("latin-1")) + with pytest.raises(SystemExit, match="is not UTF-8"): + main([str(latin), "--column", "pc"]) + + def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: From dcd118562309b69dd059a6c488dee1c4b11481c8 Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:40:54 +0100 Subject: [PATCH 8/9] fix(python): count one row in the singular --- python/src/ng_postcode/convert.py | 3 ++- python/tests/test_convert.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/src/ng_postcode/convert.py b/python/src/ng_postcode/convert.py index 5dcdfdf..6776dde 100644 --- a/python/src/ng_postcode/convert.py +++ b/python/src/ng_postcode/convert.py @@ -170,4 +170,5 @@ def main(argv: list[str] | None = None) -> None: parts = [ f"{counts[status]:,} {status}" for status in ("valid", "suggested", "invalid", "empty") ] - print(f"ng-postcode: {sum(counts.values()):,} rows: {', '.join(parts)}", file=sys.stderr) + total = sum(counts.values()) + print(f"ng-postcode: {total:,} row{'s' * (total != 1)}: {', '.join(parts)}", file=sys.stderr) diff --git a/python/tests/test_convert.py b/python/tests/test_convert.py index 3e8b98d..70e7357 100644 --- a/python/tests/test_convert.py +++ b/python/tests/test_convert.py @@ -92,6 +92,15 @@ def test_a_row_longer_than_the_header_stops_the_run(tmp_path: Path) -> None: main([str(source), "--column", "pc", "--out", str(tmp_path / "out.csv")]) +def test_one_row_is_counted_in_the_singular( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + source = tmp_path / "in.csv" + source.write_text("pc\nEK01A03FK01\n", encoding="utf-8") + main([str(source), "--column", "pc"]) + assert "ng-postcode: 1 row: 1 valid" in capsys.readouterr().err + + def test_a_row_shorter_than_the_header_is_empty(capsys: pytest.CaptureFixture[str]) -> None: write(read(io.StringIO("id,pc\n1\n"), "pc"), sys.stdout, "pc", None) assert capsys.readouterr().out.splitlines()[1] == "1,,,empty,," From be015fba1b9b3714605443b2f20c79c741d7991d Mon Sep 17 00:00:00 2001 From: mkbadeniyi Date: Mon, 12 Oct 2026 05:41:56 +0100 Subject: [PATCH 9/9] docs(python): say what the convert command refuses --- docs/llms-full.txt | 2 +- python/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/llms-full.txt b/docs/llms-full.txt index b3cb292..a6b0047 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -485,7 +485,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error - `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty. - `invalid` rows say what is wrong, and `empty` rows had no value. -It works offline and reads one row at a time, so the size of the file does not matter. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. +It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. ## API diff --git a/python/README.md b/python/README.md index 5695509..e7c4312 100644 --- a/python/README.md +++ b/python/README.md @@ -66,7 +66,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error - `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty. - `invalid` rows say what is wrong, and `empty` rows had no value. -It works offline and reads one row at a time, so the size of the file does not matter. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. +It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. ## API