Skip to content

Fork test: PR #1347 - Add valid string case check (#1335) #1375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ghanse wants to merge 6 commits into
base: main
Choose a base branch
from
+272 −0
Open

Fork test: PR #1347 - Add valid string case check (#1335) #1375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
141fcb7
Add valid string case check (#1335)
SyedIshmumAhnaf Jul 19, 2026
a633efc
Address string case review feedback
SyedIshmumAhnaf Jul 23, 2026
88eedde
Update sentence case handling, docs, and tests
ghanse Jul 24, 2026
45c32f3
Merge branch 'main' into feat/valid-string-case-check
ghanse Jul 24, 2026
db1078d
Merge branch 'main' into feat/valid-string-case-check
ghanse Jul 24, 2026
0c37062
Fix perf tests
ghanse Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/dqx/docs/reference/quality_checks.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ You can also define your own custom checks in Python (see [Creating custom check
| `is_older_than_col2_for_n_days` | Checks whether the values in one input column are at least N days older than the values in another column. | `column1`: first column to check (can be a string column name or a column expression); `column2`: second column to check (can be a string column name or a column expression); `days`: number of days; `negate`: if the condition should be negated |
| `regex_match` | Checks whether the values in the input column match a given regex. | `column`: column to check (can be a string column name or a column expression); regex: regex to check; `negate`: if the condition should be negated (true) or not |
| `is_valid_email` | Checks whether the values in the input column have valid email address format. | `column`: column to check (can be a string column name or a column expression) |
| `has_valid_string_case` | Checks whether string values match the requested letter case: `upper` requires all alphabetic characters to be uppercase; `lower` requires all alphabetic characters to be lowercase; `title` requires the first character of each space-delimited word to be uppercase; `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase. | `column`: column to check (can be a string column name or a column expression); `case`: one of `upper`, `lower`, `title`, or `sentence` |
| `is_valid_national_id` | Checks whether the values in the input column are valid national identification numbers (e.g., US Social Security Numbers) for the given country. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) |
| `is_valid_ipv4_address` | Checks whether the values in the input column have valid IPv4 address format. | `column` to check (can be a string column name or a column expression) |
| `is_ipv4_address_in_cidr` | Checks whether the values in the input column have valid IPv4 address format and fall within the given CIDR block. | `column`: column to check (can be a string column name or a column expression); `cidr_block`: CIDR block string |
Expand Down Expand Up @@ -533,6 +534,13 @@ For brevity, the `name` field in the examples is omitted and it will be auto-gen
arguments:
column: col1

# has_valid_string_case check
- criticality: error
check:
function: has_valid_string_case
arguments:
column: col1
case: lower
# is_valid_national_id check
- criticality: error
check:
Expand Down Expand Up @@ -1288,6 +1296,12 @@ checks = [
column="col1"
),

# has_valid_string_case check
DQRowRule(
criticality="error",
check_func=check_funcs.has_valid_string_case,
column="col1",
check_func_kwargs={"case": "lower"}
# is_valid_national_id check
DQRowRule(
criticality="error",
Expand Down
69 changes: 69 additions & 0 deletions src/databricks/labs/dqx/check_funcs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
_IPV4_CIDR_SUFFIX = r"(3[0-2]|[12]?\d)"
IPV4_MAX_OCTET_COUNT = 4
IPV4_BIT_LENGTH = 32
_VALID_STRING_CASES = {"upper", "lower", "title", "sentence"}
_MAX_SUBSTRING_LENGTH = 2_147_483_647

# Email helpers (RFC 5322 §3.2.3, §3.2.4 + RFC 5321 §4.1.3, §4.5.3.1).
_EMAIL_ATEXT = r"[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]"
Expand Down Expand Up @@ -265,6 +267,73 @@ def is_null_or_empty(column: str | Column, trim_strings: bool | None = False) ->
)


@register_rule("row")
def has_valid_string_case(column: str | Column, case: str) -> Column:
"""Checks whether string values match the requested letter case:

* `upper` requires all alphabetic characters to be uppercase
* `lower` requires all alphabetic characters to be lowercase
* `title` requires the first character of each space-delimited word to be uppercase
* `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase

Args:
column: column to check; can be a string column name or a column expression
case: expected case; one of *upper*, *lower*, *title*, or *sentence*

Returns:
Column object for condition

Raises:
InvalidParameterError: If *case* is not a supported string case.
"""
if not isinstance(case, str):
raise InvalidParameterError(f"'case' must be a string, got {type(case)} instead.")
if case not in _VALID_STRING_CASES:
raise InvalidParameterError(f"'case' must be one of {sorted(_VALID_STRING_CASES)}, got '{case}'")

col_str_norm, col_expr_str, col_expr = get_normalized_column_and_expr(column)
col_expr_cast = col_expr.cast("string")

if case == "upper":
normalized_case = F.upper(col_expr_cast)
elif case == "lower":
normalized_case = F.lower(col_expr_cast)
elif case == "title":
words = F.split(col_expr_cast, " ")
normalized_case = F.array_join(
F.transform(
words,
lambda word: F.concat(
F.upper(F.substring(word, 1, 1)),
F.substring(word, 2, _MAX_SUBSTRING_LENGTH),
),
),
" ",
)
else:
segments = F.split(col_expr_cast, r"\.")
normalized_case = F.array_join(
F.transform(
segments,
lambda segment: F.concat(
F.regexp_extract(segment, r"^(\s*)", 1),
F.upper(F.substring(F.regexp_replace(segment, r"^\s*", ""), 1, 1)),
F.substring(F.regexp_replace(segment, r"^\s*", ""), 2, _MAX_SUBSTRING_LENGTH),
),
),
".",
)

condition = F.when(col_expr.isNotNull(), col_expr_cast != normalized_case).otherwise(F.lit(None))
message = F.concat_ws(
"",
F.lit("Value '"),
col_expr.cast("string"),
F.lit(f"' in Column '{col_expr_str}' does not have valid '{case}' string case"),
)
return make_condition(condition, message, f"{col_str_norm}_has_invalid_{case}_string_case")


@register_rule("row")
def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column:
"""Checks whether the values in the input column are not null and present in the list of allowed values.
Expand Down
88 changes: 88 additions & 0 deletions tests/integration/test_row_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,101 @@
is_null,
is_empty,
is_null_or_empty,
has_valid_string_case,
)
from databricks.labs.dqx.pii import pii_detection_funcs
from databricks.labs.dqx.errors import InvalidParameterError

SCHEMA = "a: string, b: int"


def test_has_valid_string_case(spark):
test_df = spark.createDataFrame(
[
[
"UPPER",
"lower",
"Notes From IEEE Meeting",
"First segment. Second segment.",
{"value": "nested"},
123,
],
[
"Upper",
"Lower",
"Notes from IEEE Meeting",
"first SEGMENT. second SEGMENT",
{"value": "Nested"},
None,
],
[None, "", " ", "123!?. #$", {"value": "nested"}, 123],
[" UPPER ", " lower ", " spark sql ", " first SEGMENT. second SEGMENT ", {"value": "nested"}, 123],
["123!?", "123!?", "hello-world", "Hello 123!. Second segment", {"value": "NESTED"}, 123],
[
"UPPER",
"lower",
"Notes From The 4th IEEE Meeting",
"The first IEEE meeting was today. The 2nd IEEE meeting will be next month.",
{"value": "nested"},
123,
],
],
"upper: string, lower: string, title: string, sentence: string, nested: struct<value:string>, numeric: int",
)

actual = test_df.select(
has_valid_string_case("upper", "upper"),
has_valid_string_case("lower", "lower"),
has_valid_string_case("title", "title"),
has_valid_string_case("sentence", "sentence"),
has_valid_string_case(F.col("nested").getItem("value"), "lower"),
has_valid_string_case("numeric", "lower"),
)

def violation(value: str, column: str, case: str) -> str:
return f"Value '{value}' in Column '{column}' does not have valid '{case}' string case"

expected = spark.createDataFrame(
[
[None, None, None, None, None, None],
[
violation("Upper", "upper", "upper"),
violation("Lower", "lower", "lower"),
violation("Notes from IEEE Meeting", "title", "title"),
violation("first SEGMENT. second SEGMENT", "sentence", "sentence"),
violation("Nested", "nested['value']", "lower"),
None,
],
[None, None, None, None, None, None],
[
None,
None,
violation(" spark sql ", "title", "title"),
violation(" first SEGMENT. second SEGMENT ", "sentence", "sentence"),
None,
None,
],
[
None,
None,
violation("hello-world", "title", "title"),
None,
violation("NESTED", "nested['value']", "lower"),
None,
],
[None, None, None, None, None, None],
],
"upper_has_invalid_upper_string_case: string, "
"lower_has_invalid_lower_string_case: string, "
"title_has_invalid_title_string_case: string, "
"sentence_has_invalid_sentence_string_case: string, "
"nested_value_has_invalid_lower_string_case: string, "
"numeric_has_invalid_lower_string_case: string",
)

assertDataFrameEqual(actual, expected)


def test_col_is_not_null_and_not_empty(spark):
input_schema = "a: string, b: int, c: map<string, string>, d: array<string>, e: string"
test_df = spark.createDataFrame(
Expand Down
46 changes: 46 additions & 0 deletions tests/perf/test_apply_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2210,6 +2210,52 @@ def test_benchmark_is_valid_email(benchmark, ws, generated_email_df, column):
assert actual_count == EXPECTED_ROWS


@pytest.mark.parametrize(
"case, generated_string_df",
[
pytest.param(
"upper",
{"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "VALID UPPER CASE"},
id="upper",
),
pytest.param(
"lower",
{"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "valid lower case"},
id="lower",
),
pytest.param(
"title",
{"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "Notes From IEEE Meeting"},
id="title",
),
pytest.param(
"sentence",
{"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "First segment. Second segment."},
id="sentence",
),
],
indirect=["generated_string_df"],
)
@pytest.mark.benchmark(group="test_benchmark_has_valid_string_case")
def test_benchmark_has_valid_string_case(benchmark, ws, generated_string_df, case):
columns, df, _ = generated_string_df
column = columns[0]
dq_engine = DQEngine(workspace_client=ws, extra_params=EXTRA_PARAMS)
checks = [
DQRowRule(
name=f"{column}_has_invalid_{case}_string_case",
criticality="warn",
check_func=check_funcs.has_valid_string_case,
column=column,
check_func_kwargs={"case": case},
),
]
benchmark.group += f" {case}"
checked = dq_engine.apply_checks(df, checks)
actual_count = benchmark(lambda: checked.count())
assert actual_count == EXPECTED_ROWS


@pytest.mark.parametrize(
"column",
[
Expand Down
7 changes: 7 additions & 0 deletions tests/resources/all_row_checks.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,13 @@
arguments:
column: col_email

# has_valid_string_case check
- criticality: error
check:
function: has_valid_string_case
arguments:
column: col1
case: lower
# is_valid_national_id check
- criticality: error
check:
Expand Down
29 changes: 29 additions & 0 deletions tests/unit/test_build_rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
foreign_key,
is_valid_ipv4_address,
is_valid_email,
has_valid_string_case,
is_ipv4_address_in_cidr,
is_not_less_than,
is_not_greater_than,
Expand Down Expand Up @@ -773,6 +774,13 @@ def test_build_rules_by_metadata():
"criticality": "error",
"check": {"function": "is_valid_email", "arguments": {"column": "a"}},
},
{
"criticality": "error",
"check": {
"function": "has_valid_string_case",
"arguments": {"column": "a", "case": "lower"},
},
},
]

actual_rules = deserialize_checks(checks)
Expand Down Expand Up @@ -1045,6 +1053,13 @@ def test_build_rules_by_metadata():
check_func=is_valid_email,
column="a",
),
DQRowRule(
name="a_has_invalid_lower_string_case",
criticality="error",
check_func=has_valid_string_case,
column="a",
check_func_kwargs={"case": "lower"},
),
]

assert pprint.pformat(actual_rules) == pprint.pformat(expected_rules)
Expand Down Expand Up @@ -1380,6 +1395,12 @@ def test_convert_dq_rules_to_metadata():
criticality="error", check_func=is_valid_date, column="b", check_func_kwargs={"date_format": "yyyy-MM-dd"}
),
DQRowRule(criticality="error", check_func=is_valid_json, column="col_json_str"),
DQRowRule(
criticality="error",
check_func=has_valid_string_case,
column="col1",
check_func_kwargs={"case": "lower"},
),
DQRowRule(
criticality="error",
check_func=has_json_keys,
Expand Down Expand Up @@ -1595,6 +1616,14 @@ def test_convert_dq_rules_to_metadata():
"arguments": {"column": "col_json_str"},
},
},
{
"name": "col1_has_invalid_lower_string_case",
"criticality": "error",
"check": {
"function": "has_valid_string_case",
"arguments": {"column": "col1", "case": "lower"},
},
},
{
"name": "col_json_str_does_not_have_json_keys",
"criticality": "error",
Expand Down
Loading
Loading