Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions docs/dqx/docs/reference/quality_checks.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ You can also define your own custom checks in Python (see [Creating custom check
| `regex_match` | Checks whether the values in the input column match a given regex. | `column`: column to check (can be a string column name or a column expression); regex: regex to check; `negate`: if the condition should be negated (true) or not |
| `is_valid_email` | Checks whether the values in the input column have valid email address format. | `column`: column to check (can be a string column name or a column expression) |
| `is_valid_national_id` | Checks whether the values in the input column are valid national identification numbers (e.g., US Social Security Numbers) for the given country. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) |
| `is_valid_currency_code` | Checks whether the values in the input column are valid ISO 4217 currency codes (alphabetic, e.g. USD, or numeric, e.g. 840). Source: https://www.iso.org/iso-4217-currency-codes.html | `column`: column to check (can be a string column name or a column expression); `code_format`: ISO 4217 representation, `alphabetic` (default) or `numeric`; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) |
| `is_valid_ipv4_address` | Checks whether the values in the input column have valid IPv4 address format. | `column` to check (can be a string column name or a column expression) |
| `is_ipv4_address_in_cidr` | Checks whether the values in the input column have valid IPv4 address format and fall within the given CIDR block. | `column`: column to check (can be a string column name or a column expression); `cidr_block`: CIDR block string |
| `is_valid_ipv6_address` | Checks whether the values in the input column have valid IPv6 address format. | `column` to check (can be a string column name or a column expression) |
Expand Down Expand Up @@ -541,6 +542,14 @@ For brevity, the `name` field in the examples is omitted and it will be auto-gen
column: col1
country: US

# is_valid_currency_code check
- criticality: error
check:
function: is_valid_currency_code
arguments:
column: col1
code_format: alphabetic

# is_valid_ipv4_address check
- criticality: error
check:
Expand Down Expand Up @@ -1298,6 +1307,16 @@ checks = [
}
),

# is_valid_currency_code check
DQRowRule(
criticality="error",
check_func=check_funcs.is_valid_currency_code,
column="col1", # or as expr: F.col("col1")
check_func_kwargs={
"code_format": "alphabetic"
}
),

# is_valid_ipv4_address check
DQRowRule(
criticality="error",
Expand Down
91 changes: 91 additions & 0 deletions src/databricks/labs/dqx/check_funcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import ipaddress
import uuid
from decimal import Decimal
from importlib.resources import files
from pathlib import Path
from collections.abc import Callable, Sequence
from enum import Enum
from itertools import zip_longest
Expand Down Expand Up @@ -1081,6 +1083,95 @@ def is_valid_national_id(column: str | Column, country: str = "US") -> Column:
return _matches_pattern(column, pattern)


def _load_iso_codes(resource_name: str) -> frozenset[str]:
"""Load a set of standard codes from a newline-delimited data file in the resources package.

The large standard code lists are stored as data files rather than inline literals to keep them
readable and easy to regenerate. See the files under *databricks/labs/dqx/resources* for the
values and their authoritative sources.
"""
resource = Path(str(files("databricks.labs.dqx.resources") / resource_name))
return frozenset(resource.read_text(encoding="utf-8").split())


# ISO 4217 currency codes. The authoritative source is
# https://www.iso.org/iso-4217-currency-codes.html; the values were verified against it and cover
# the active codes as of July 2026. The code lists are stored as data files under the resources
# package and loaded via importlib.resources. To regenerate them, iterate pycountry.currencies
# (which packages the ISO 4217 data) as a convenience, reading each entry's alphabetic code (exposed
# by pycountry as the alpha_3 attribute) and numeric code, then reconcile against the official ISO
# list above before committing.
_ISO_4217_CODES_BY_FORMAT: dict[str, frozenset[str]] = {
"alphabetic": _load_iso_codes("iso_4217_alphabetic.txt"),
"numeric": _load_iso_codes("iso_4217_numeric.txt"),
}


@register_rule("row")
def is_valid_currency_code(
column: str | Column, code_format: str = "alphabetic", case_sensitive: bool = True
) -> Column:
"""Checks whether the values in the input column are valid ISO 4217 currency codes.

ISO 4217 defines two code representations, selected with *code_format*:

* *alphabetic* (default): the three-letter code, e.g. *USD*, *EUR*, *JPY*.
* *numeric*: the three-digit code, e.g. *840*, *978*, *392*.

The valid codes follow the ISO 4217 standard; see https://www.iso.org/iso-4217-currency-codes.html.
Every code assigned by the standard is accepted, which includes codes that are not spendable
currencies, such as *XXX* (no currency), *XTS* (reserved for testing), the precious metals
(*XAU*, *XAG*, *XPT*, *XPD*) and *XDR* (IMF special drawing rights). Numeric codes are the
three-digit, zero-padded form (e.g. *036*), so a numeric input column must preserve the leading
zeros.

By default the comparison is case-sensitive; pass *case_sensitive* as False to accept values in
any case. Null values will pass the check with no violation reported.

Args:
column: column to check; can be a string column name or a column expression
code_format: ISO 4217 code representation to validate against, either *alphabetic* (default)
or *numeric*
case_sensitive: whether to perform a case-sensitive comparison (default: True)

Returns:
Column object for condition

Raises:
MissingParameterError: if *code_format* is None.
InvalidParameterError: if *code_format* is not a string, or is not a supported representation.
"""
if code_format is None:
raise MissingParameterError("'code_format' is not provided.")
if not isinstance(code_format, str):
raise InvalidParameterError(f"'code_format' must be a string, got {type(code_format)} instead.")
normalized_format = code_format.lower()
allowed_codes = _ISO_4217_CODES_BY_FORMAT.get(normalized_format)
if allowed_codes is None:
supported = ", ".join(sorted(_ISO_4217_CODES_BY_FORMAT))
raise InvalidParameterError(
f"Unsupported code_format for currency code validation: '{code_format}'. Supported: [{supported}]."
)
col_str_norm, col_expr_str, col_expr = get_normalized_column_and_expr(column)
if case_sensitive:
col_expr_compare = col_expr
allowed = [F.lit(code) for code in sorted(allowed_codes)]
else:
col_expr_compare = to_lowercase(col_expr)
allowed = [F.lit(code.lower()) for code in sorted(allowed_codes)]
condition = F.when(col_expr.isNotNull(), ~col_expr_compare.isin(*allowed)).otherwise(F.lit(None))
return make_condition(
condition,
F.concat_ws(
"",
F.lit("Value '"),
col_expr.cast("string"),
F.lit(f"' in Column '{col_expr_str}' is not a valid ISO 4217 currency code"),
),
f"{col_str_norm}_is_not_a_valid_currency_code",
)


@register_rule("row")
def is_ipv4_address_in_cidr(column: str | Column, cidr_block: str) -> Column:
"""
Expand Down
Empty file.
178 changes: 178 additions & 0 deletions src/databricks/labs/dqx/resources/iso_4217_alphabetic.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
AED
AFN
ALL
AMD
AOA
ARS
AUD
AWG
AZN
BAM
BBD
BDT
BHD
BIF
BMD
BND
BOB
BOV
BRL
BSD
BTN
BWP
BYN
BZD
CAD
CDF
CHE
CHF
CHW
CLF
CLP
CNY
COP
COU
CRC
CUP
CVE
CZK
DJF
DKK
DOP
DZD
EGP
ERN
ETB
EUR
FJD
FKP
GBP
GEL
GHS
GIP
GMD
GNF
GTQ
GYD
HKD
HNL
HTG
HUF
IDR
ILS
INR
IQD
IRR
ISK
JMD
JOD
JPY
KES
KGS
KHR
KMF
KPW
KRW
KWD
KYD
KZT
LAK
LBP
LKR
LRD
LSL
LYD
MAD
MDL
MGA
MKD
MMK
MNT
MOP
MRU
MUR
MVR
MWK
MXN
MXV
MYR
MZN
NAD
NGN
NIO
NOK
NPR
NZD
OMR
PAB
PEN
PGK
PHP
PKR
PLN
PYG
QAR
RON
RSD
RUB
RWF
SAR
SBD
SCR
SDG
SEK
SGD
SHP
SLE
SOS
SRD
SSP
STN
SVC
SYP
SZL
THB
TJS
TMT
TND
TOP
TRY
TTD
TWD
TZS
UAH
UGX
USD
USN
UYI
UYU
UYW
UZS
VED
VES
VND
VUV
WST
XAD
XAF
XAG
XAU
XBA
XBB
XBC
XBD
XCD
XCG
XDR
XOF
XPD
XPF
XPT
XSU
XTS
XUA
XXX
YER
ZAR
ZMW
ZWG
Loading