Skip to content

Commit 5f0aa34

Browse files
fix(python): wait out a NIPOST rate limit once when confirming a CSV
1 parent 82d7ca3 commit 5f0aa34

4 files changed

Lines changed: 23 additions & 10 deletions

File tree

‎docs/llms-full.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error
485485
- `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty.
486486
- `invalid` rows say what is wrong, and `empty` rows had no value.
487487

488-
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.
488+
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. When NIPOST's rate limit is reached it waits a minute and asks again; after any other failure the remaining rows are left unconfirmed.
489489

490490
## API
491491

‎python/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ id,postcode,postcode_compact,postcode_status,postcode_suggestion,postcode_error
6666
- `suggested` rows only parse after look-alike characters are swapped. The fix is offered for a person to confirm, and the compact column is left empty.
6767
- `invalid` rows say what is wrong, and `empty` rows had no value.
6868

69-
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column.
69+
It works offline and reads one row at a time, so the size of the file does not matter. It never writes over the file it reads, and it stops at a header that already has one of these columns or a row with more cells than the header. Add `--confirm` to ask NIPOST whether each well-formed code is assigned to a building. That needs `pip install "ng-postcode[client]"` and your own key in `NG_POSTCODE_API_KEY`, makes one free level 1 lookup for each distinct code, and adds a `postcode_assigned` column. When NIPOST's rate limit is reached it waits a minute and asks again; after any other failure the remaining rows are left unconfirmed.
7070

7171
## API
7272

‎python/src/ng_postcode/convert.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import csv
1515
import os
1616
import sys
17+
import time
1718
from collections import Counter
1819
from collections.abc import Callable, Iterable, Iterator
1920
from dataclasses import dataclass
@@ -109,11 +110,16 @@ def write(
109110
return counts
110111

111112

112-
def nipost(key: str) -> Confirm:
113-
"""Level 1 lookups with the caller's own key. After the first failure NIPOST is not
114-
asked again, and the remaining rows are left unconfirmed."""
113+
WAIT = 60.0
114+
"""Seconds to wait when NIPOST says the rate limit is reached. It does not publish one."""
115+
116+
117+
def nipost(key: str, sleep: Callable[[float], None] = time.sleep) -> Confirm:
118+
"""Level 1 lookups with the caller's own key. A rate limit is waited out once for each
119+
code. After the first failure NIPOST is not asked again, and the remaining rows are left
120+
unconfirmed."""
115121
try:
116-
from .api import Lookup, lookup
122+
from .api import ApiError, Lookup, lookup
117123
from .client import Client
118124
except ImportError:
119125
sys.exit('ng-postcode: --confirm needs the client: pip install "ng-postcode[client]"')
@@ -124,6 +130,10 @@ def confirm(code: Postcode) -> bool | None:
124130
if failed:
125131
return None
126132
answer = client.send(lookup(code))
133+
if isinstance(answer, ApiError) and answer.status == 429:
134+
print(f"ng-postcode: rate limit reached, waiting {WAIT:.0f} s", file=sys.stderr)
135+
sleep(WAIT)
136+
answer = client.send(lookup(code))
127137
if isinstance(answer, Lookup):
128138
return answer.valid
129139
failed.append(str(answer))

‎python/tests/test_convert.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,15 +125,15 @@ def test_a_file_that_cannot_be_read_is_reported_in_one_line(tmp_path: Path) -> N
125125
main([str(latin), "--column", "pc"])
126126

127127

128-
def test_confirming_asks_once_for_each_code_and_stops_at_the_first_failure(
128+
def test_confirming_asks_once_for_each_code_waits_out_a_rate_limit_and_stops_at_a_failure(
129129
monkeypatch: pytest.MonkeyPatch,
130130
) -> None:
131131
asked: list[str] = []
132132

133133
def handle(request: httpx.Request) -> httpx.Response:
134134
code = request.url.params["code"]
135135
asked.append(code)
136-
if code == "LA-11-W06-TC-10":
136+
if code == "LA-11-W06-TC-10" or (asked.count(code) == 1 and code.startswith("FC")):
137137
error = {"code": "rate_limited", "message": "slow down"}
138138
return httpx.Response(429, json={"error": error})
139139
return httpx.Response(
@@ -144,7 +144,10 @@ def handle(request: httpx.Request) -> httpx.Response:
144144
stub = httpx.MockTransport(handle)
145145
monkeypatch.setattr(httpx, "Client", lambda **options: real(transport=stub, **options))
146146
cells = ["EK01A03FK01", "ek-01-a03-fk-01", "FC03B06AG12", "LA11W06TC10", "EK01A03ZZ14", "x"]
147-
out = list(rows(({"pc": cell} for cell in cells), "pc", nipost("key")))
147+
waits: list[float] = []
148+
out = list(rows(({"pc": cell} for cell in cells), "pc", nipost("key", waits.append)))
148149
assert [row["pc_assigned"] for row in out] == ["true", "true", "false", "", "", ""]
149-
assert asked == ["EK-01-A03-FK-01", "FC-03-B06-AG-12", "LA-11-W06-TC-10"]
150+
# A rate limit is waited out and asked again once: FC then answers, and LA does not.
151+
assert asked == ["EK-01-A03-FK-01", *["FC-03-B06-AG-12"] * 2, *["LA-11-W06-TC-10"] * 2]
152+
assert waits == [60.0, 60.0]
150153
assert Postcode("EK01A03FK01").compact == out[0]["pc_compact"]

0 commit comments

Comments
 (0)