Skip to content
Merged
2 changes: 1 addition & 1 deletion docs/llms-full.txt
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error
- `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty.
- `invalid` rows say what is wrong, and `empty` rows had no value.

It works offline and reads one row at a time, so the size of the file does not matter. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.

## API

Expand Down
2 changes: 1 addition & 1 deletion python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error
- `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty.
- `invalid` rows say what is wrong, and `empty` rows had no value.

It works offline and reads one row at a time, so the size of the file does not matter. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.

## API

Expand Down
74 changes: 54 additions & 20 deletions python/src/ng_postcode/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,17 +70,40 @@ def rows(
yield row | added


def write(source: TextIO, target: TextIO, column: str, confirm: Confirm | None) -> Counter[str]:
"""Convert `source` to `target` and count the rows by status."""
def read(source: TextIO, column: str) -> csv.DictReader[str]:
"""The rows of `source`, once its header is known to hold `column`."""
reader = csv.DictReader(source)
if column not in (reader.fieldnames or []):
sys.exit(f"ng-postcode: no column named {column!r}; found {reader.fieldnames}")
writer: csv.DictWriter[str] | None = None
names = list(reader.fieldnames or [])
repeated = [name for name in names if names.count(name) > 1]
if repeated:
sys.exit(f"ng-postcode: the header names {repeated[0]!r} more than once")
taken = [name for name in names if name in new_columns(column, True)]
if taken:
sys.exit(f"ng-postcode: the file already has a column named {taken[0]!r}")
return reader


def new_columns(column: str, confirming: bool) -> list[str]:
"""The columns a run adds, which the file must not already hold."""
names = ["compact", "status", "suggestion", "error", *(["assigned"] if confirming else [])]
return [f"{column}_{name}" for name in names]


def write(
reader: csv.DictReader[str], target: TextIO, column: str, confirm: Confirm | None
) -> Counter[str]:
"""Convert the rows of `reader` to `target` and count them by status."""
names = [*(reader.fieldnames or []), *new_columns(column, confirm is not None)]
writer = csv.DictWriter(target, fieldnames=names, lineterminator="\n")
# Written before any row, so a file with none keeps its header.
writer.writeheader()
counts: Counter[str] = Counter()
for row in rows(reader, column, confirm):
if writer is None:
writer = csv.DictWriter(target, fieldnames=list(row), lineterminator="\n")
writer.writeheader()
# The reader files cells past the header under None, and no column can take them.
if None in row:
sys.exit(f"ng-postcode: line {reader.line_num} has more cells than the header")
writer.writerow(row)
counts[row[f"{column}_status"]] += 1
return counts
Expand Down Expand Up @@ -121,20 +144,31 @@ def main(argv: list[str] | None = None) -> None:
if args.confirm and not key:
cli.error("--confirm needs a NIPOST API key in NG_POSTCODE_API_KEY")
confirm = nipost(key) if args.confirm else None
with contextlib.ExitStack() as files:
# utf-8-sig drops the byte order mark that spreadsheet exports begin with.
source = (
sys.stdin
if args.file == "-"
else files.enter_context(open(args.file, newline="", encoding="utf-8-sig"))
)
target = (
files.enter_context(open(args.out, "w", newline="", encoding="utf-8"))
if args.out
else sys.stdout
)
counts = write(source, target, args.column, confirm)
try:
with contextlib.ExitStack() as files:
# utf-8-sig drops the byte order mark that spreadsheet exports begin with.
source = (
sys.stdin
if args.file == "-"
else files.enter_context(open(args.file, newline="", encoding="utf-8-sig"))
)
reader = read(source, args.column)
# Opening the output empties it, so it waits until the input is known to be usable.
same = args.out and args.file != "-" and os.path.exists(args.out)
if same and os.path.samefile(args.file, args.out):
sys.exit("ng-postcode: --out is the input file, which writing would empty")
target = (
files.enter_context(open(args.out, "w", newline="", encoding="utf-8"))
if args.out
else sys.stdout
)
counts = write(reader, target, args.column, confirm)
except UnicodeDecodeError:
sys.exit(f"ng-postcode: {args.file} is not UTF-8; save it as CSV UTF-8 and run again")
except OSError as error:
sys.exit(f"ng-postcode: {error}")
parts = [
f"{counts[status]:,} {status}" for status in ("valid", "suggested", "invalid", "empty")
]
print(f"ng-postcode: {sum(counts.values()):,} rows: {', '.join(parts)}", file=sys.stderr)
total = sum(counts.values())
print(f"ng-postcode: {total:,} row{'s' * (total != 1)}: {', '.join(parts)}", file=sys.stderr)
71 changes: 68 additions & 3 deletions python/tests/test_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@

import csv
import io
import sys
from pathlib import Path

import httpx
import pytest

from ng_postcode import Postcode
from ng_postcode.convert import Converted, convert, main, nipost, rows, write
from ng_postcode.convert import Converted, convert, main, nipost, read, rows, write


@pytest.mark.parametrize(
Expand Down Expand Up @@ -52,12 +53,76 @@ def test_converts_a_file_and_reports_the_counts(


def test_a_missing_column_or_key_stops_before_anything_is_written(tmp_path: Path) -> None:
source = tmp_path / "in.csv"
source, target = tmp_path / "in.csv", tmp_path / "out.csv"
source.write_text("id,pc\n1,x\n", encoding="utf-8")
target.write_text("kept", encoding="utf-8")
with pytest.raises(SystemExit, match="no column named 'postcode'"):
write(source.open(), io.StringIO(), "postcode", None)
read(io.StringIO("id,pc\n1,x\n"), "postcode")
with pytest.raises(SystemExit, match="no column named 'postcode'"):
main([str(source), "--column", "postcode", "--out", str(target)])
with pytest.raises(SystemExit):
main([str(source), "--column", "pc", "--confirm"])
assert target.read_text(encoding="utf-8") == "kept"


def test_the_input_file_is_never_written_over(tmp_path: Path) -> None:
source = tmp_path / "in.csv"
source.write_text("id,pc\n1,EK01A03FK01\n", encoding="utf-8")
link = tmp_path / "link.csv"
link.symlink_to(source)
for out in (source, link):
with pytest.raises(SystemExit, match="--out is the input file"):
main([str(source), "--column", "pc", "--out", str(out)])
assert source.read_text(encoding="utf-8") == "id,pc\n1,EK01A03FK01\n"


def test_a_file_with_no_rows_keeps_its_header(tmp_path: Path) -> None:
source, target = tmp_path / "in.csv", tmp_path / "out.csv"
source.write_text("id,pc\n", encoding="utf-8")
main([str(source), "--column", "pc", "--out", str(target)])
assert target.read_text(encoding="utf-8") == (
"id,pc,pc_compact,pc_status,pc_suggestion,pc_error\n"
)


def test_a_row_longer_than_the_header_stops_the_run(tmp_path: Path) -> None:
source = tmp_path / "in.csv"
source.write_text("id,pc\n1,EK01A03FK01\n2,x,extra\n", encoding="utf-8")
with pytest.raises(SystemExit, match="line 3 has more cells than the header"):
main([str(source), "--column", "pc", "--out", str(tmp_path / "out.csv")])


def test_one_row_is_counted_in_the_singular(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
source = tmp_path / "in.csv"
source.write_text("pc\nEK01A03FK01\n", encoding="utf-8")
main([str(source), "--column", "pc"])
assert "ng-postcode: 1 row: 1 valid" in capsys.readouterr().err


def test_a_row_shorter_than_the_header_is_empty(capsys: pytest.CaptureFixture[str]) -> None:
write(read(io.StringIO("id,pc\n1\n"), "pc"), sys.stdout, "pc", None)
assert capsys.readouterr().out.splitlines()[1] == "1,,,empty,,"


def test_a_column_the_run_would_add_is_never_replaced() -> None:
with pytest.raises(SystemExit, match="already has a column named 'pc_status'"):
read(io.StringIO("pc,pc_status\nEK01A03FK01,shipped\n"), "pc")


def test_a_header_that_repeats_a_name_is_refused() -> None:
with pytest.raises(SystemExit, match="names 'id' more than once"):
read(io.StringIO("id,pc,id\n1,EK01A03FK01,2\n"), "pc")


def test_a_file_that_cannot_be_read_is_reported_in_one_line(tmp_path: Path) -> None:
with pytest.raises(SystemExit, match=r"ng-postcode: .*No such file"):
main([str(tmp_path / "missing.csv"), "--column", "pc"])
latin = tmp_path / "latin.csv"
latin.write_bytes("id,pc\n1,caf\xe9\n".encode("latin-1"))
with pytest.raises(SystemExit, match="is not UTF-8"):
main([str(latin), "--column", "pc"])


def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure(
Expand Down
Loading