diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 4e999e512..320478133 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -60,6 +60,7 @@ You can also define your own custom checks in Python (see [Creating custom check | `is_older_than_col2_for_n_days` | Checks whether the values in one input column are at least N days older than the values in another column. | `column1`: first column to check (can be a string column name or a column expression); `column2`: second column to check (can be a string column name or a column expression); `days`: number of days; `negate`: if the condition should be negated | | `regex_match` | Checks whether the values in the input column match a given regex. | `column`: column to check (can be a string column name or a column expression); regex: regex to check; `negate`: if the condition should be negated (true) or not | | `is_valid_email` | Checks whether the values in the input column have valid email address format. | `column`: column to check (can be a string column name or a column expression) | +| `has_valid_string_case` | Checks whether string values match the requested letter case: `upper` requires all alphabetic characters to be uppercase; `lower` requires all alphabetic characters to be lowercase; `title` requires the first character of each space-delimited word to be uppercase; `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase. | `column`: column to check (can be a string column name or a column expression); `case`: one of `upper`, `lower`, `title`, or `sentence` | | `is_valid_national_id` | Checks whether the values in the input column are valid national identification numbers (e.g., US Social Security Numbers) for the given country. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) | | `is_valid_ipv4_address` | Checks whether the values in the input column have valid IPv4 address format. | `column` to check (can be a string column name or a column expression) | | `is_ipv4_address_in_cidr` | Checks whether the values in the input column have valid IPv4 address format and fall within the given CIDR block. | `column`: column to check (can be a string column name or a column expression); `cidr_block`: CIDR block string | @@ -533,6 +534,13 @@ For brevity, the `name` field in the examples is omitted and it will be auto-gen arguments: column: col1 +# has_valid_string_case check +- criticality: error + check: + function: has_valid_string_case + arguments: + column: col1 + case: lower # is_valid_national_id check - criticality: error check: @@ -1288,6 +1296,12 @@ checks = [ column="col1" ), + # has_valid_string_case check + DQRowRule( + criticality="error", + check_func=check_funcs.has_valid_string_case, + column="col1", + check_func_kwargs={"case": "lower"} # is_valid_national_id check DQRowRule( criticality="error", diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index 523d7d706..4835133f2 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -31,6 +31,8 @@ _IPV4_CIDR_SUFFIX = r"(3[0-2]|[12]?\d)" IPV4_MAX_OCTET_COUNT = 4 IPV4_BIT_LENGTH = 32 +_VALID_STRING_CASES = {"upper", "lower", "title", "sentence"} +_MAX_SUBSTRING_LENGTH = 2_147_483_647 # Email helpers (RFC 5322 §3.2.3, §3.2.4 + RFC 5321 §4.1.3, §4.5.3.1). _EMAIL_ATEXT = r"[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]" @@ -265,6 +267,73 @@ def is_null_or_empty(column: str | Column, trim_strings: bool | None = False) -> ) +@register_rule("row") +def has_valid_string_case(column: str | Column, case: str) -> Column: + """Checks whether string values match the requested letter case: + + * `upper` requires all alphabetic characters to be uppercase + * `lower` requires all alphabetic characters to be lowercase + * `title` requires the first character of each space-delimited word to be uppercase + * `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase + + Args: + column: column to check; can be a string column name or a column expression + case: expected case; one of *upper*, *lower*, *title*, or *sentence* + + Returns: + Column object for condition + + Raises: + InvalidParameterError: If *case* is not a supported string case. + """ + if not isinstance(case, str): + raise InvalidParameterError(f"'case' must be a string, got {type(case)} instead.") + if case not in _VALID_STRING_CASES: + raise InvalidParameterError(f"'case' must be one of {sorted(_VALID_STRING_CASES)}, got '{case}'") + + col_str_norm, col_expr_str, col_expr = get_normalized_column_and_expr(column) + col_expr_cast = col_expr.cast("string") + + if case == "upper": + normalized_case = F.upper(col_expr_cast) + elif case == "lower": + normalized_case = F.lower(col_expr_cast) + elif case == "title": + words = F.split(col_expr_cast, " ") + normalized_case = F.array_join( + F.transform( + words, + lambda word: F.concat( + F.upper(F.substring(word, 1, 1)), + F.substring(word, 2, _MAX_SUBSTRING_LENGTH), + ), + ), + " ", + ) + else: + segments = F.split(col_expr_cast, r"\.") + normalized_case = F.array_join( + F.transform( + segments, + lambda segment: F.concat( + F.regexp_extract(segment, r"^(\s*)", 1), + F.upper(F.substring(F.regexp_replace(segment, r"^\s*", ""), 1, 1)), + F.substring(F.regexp_replace(segment, r"^\s*", ""), 2, _MAX_SUBSTRING_LENGTH), + ), + ), + ".", + ) + + condition = F.when(col_expr.isNotNull(), col_expr_cast != normalized_case).otherwise(F.lit(None)) + message = F.concat_ws( + "", + F.lit("Value '"), + col_expr.cast("string"), + F.lit(f"' in Column '{col_expr_str}' does not have valid '{case}' string case"), + ) + return make_condition(condition, message, f"{col_str_norm}_has_invalid_{case}_string_case") + + @register_rule("row") def is_not_null_and_is_in_list(column: str | Column, allowed: list, case_sensitive: bool = True) -> Column: """Checks whether the values in the input column are not null and present in the list of allowed values. diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index 052589c43..81e0311c3 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -40,6 +40,7 @@ is_null, is_empty, is_null_or_empty, + has_valid_string_case, ) from databricks.labs.dqx.pii import pii_detection_funcs from databricks.labs.dqx.errors import InvalidParameterError @@ -47,6 +48,93 @@ SCHEMA = "a: string, b: int" +def test_has_valid_string_case(spark): + test_df = spark.createDataFrame( + [ + [ + "UPPER", + "lower", + "Notes From IEEE Meeting", + "First segment. Second segment.", + {"value": "nested"}, + 123, + ], + [ + "Upper", + "Lower", + "Notes from IEEE Meeting", + "first SEGMENT. second SEGMENT", + {"value": "Nested"}, + None, + ], + [None, "", " ", "123!?. #$", {"value": "nested"}, 123], + [" UPPER ", " lower ", " spark sql ", " first SEGMENT. second SEGMENT ", {"value": "nested"}, 123], + ["123!?", "123!?", "hello-world", "Hello 123!. Second segment", {"value": "NESTED"}, 123], + [ + "UPPER", + "lower", + "Notes From The 4th IEEE Meeting", + "The first IEEE meeting was today. The 2nd IEEE meeting will be next month.", + {"value": "nested"}, + 123, + ], + ], + "upper: string, lower: string, title: string, sentence: string, nested: struct, numeric: int", + ) + + actual = test_df.select( + has_valid_string_case("upper", "upper"), + has_valid_string_case("lower", "lower"), + has_valid_string_case("title", "title"), + has_valid_string_case("sentence", "sentence"), + has_valid_string_case(F.col("nested").getItem("value"), "lower"), + has_valid_string_case("numeric", "lower"), + ) + + def violation(value: str, column: str, case: str) -> str: + return f"Value '{value}' in Column '{column}' does not have valid '{case}' string case" + + expected = spark.createDataFrame( + [ + [None, None, None, None, None, None], + [ + violation("Upper", "upper", "upper"), + violation("Lower", "lower", "lower"), + violation("Notes from IEEE Meeting", "title", "title"), + violation("first SEGMENT. second SEGMENT", "sentence", "sentence"), + violation("Nested", "nested['value']", "lower"), + None, + ], + [None, None, None, None, None, None], + [ + None, + None, + violation(" spark sql ", "title", "title"), + violation(" first SEGMENT. second SEGMENT ", "sentence", "sentence"), + None, + None, + ], + [ + None, + None, + violation("hello-world", "title", "title"), + None, + violation("NESTED", "nested['value']", "lower"), + None, + ], + [None, None, None, None, None, None], + ], + "upper_has_invalid_upper_string_case: string, " + "lower_has_invalid_lower_string_case: string, " + "title_has_invalid_title_string_case: string, " + "sentence_has_invalid_sentence_string_case: string, " + "nested_value_has_invalid_lower_string_case: string, " + "numeric_has_invalid_lower_string_case: string", + ) + + assertDataFrameEqual(actual, expected) + + def test_col_is_not_null_and_not_empty(spark): input_schema = "a: string, b: int, c: map, d: array, e: string" test_df = spark.createDataFrame( diff --git a/tests/perf/test_apply_checks.py b/tests/perf/test_apply_checks.py index 0c025f1b5..dd78fe888 100644 --- a/tests/perf/test_apply_checks.py +++ b/tests/perf/test_apply_checks.py @@ -2210,6 +2210,52 @@ def test_benchmark_is_valid_email(benchmark, ws, generated_email_df, column): assert actual_count == EXPECTED_ROWS +@pytest.mark.parametrize( + "case, generated_string_df", + [ + pytest.param( + "upper", + {"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "VALID UPPER CASE"}, + id="upper", + ), + pytest.param( + "lower", + {"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "valid lower case"}, + id="lower", + ), + pytest.param( + "title", + {"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "Notes From IEEE Meeting"}, + id="title", + ), + pytest.param( + "sentence", + {"n_rows": DEFAULT_ROWS, "n_columns": 1, "template": "First segment. Second segment."}, + id="sentence", + ), + ], + indirect=["generated_string_df"], +) +@pytest.mark.benchmark(group="test_benchmark_has_valid_string_case") +def test_benchmark_has_valid_string_case(benchmark, ws, generated_string_df, case): + columns, df, _ = generated_string_df + column = columns[0] + dq_engine = DQEngine(workspace_client=ws, extra_params=EXTRA_PARAMS) + checks = [ + DQRowRule( + name=f"{column}_has_invalid_{case}_string_case", + criticality="warn", + check_func=check_funcs.has_valid_string_case, + column=column, + check_func_kwargs={"case": case}, + ), + ] + benchmark.group += f" {case}" + checked = dq_engine.apply_checks(df, checks) + actual_count = benchmark(lambda: checked.count()) + assert actual_count == EXPECTED_ROWS + + @pytest.mark.parametrize( "column", [ diff --git a/tests/resources/all_row_checks.yaml b/tests/resources/all_row_checks.yaml index 04984d132..93bd61977 100644 --- a/tests/resources/all_row_checks.yaml +++ b/tests/resources/all_row_checks.yaml @@ -484,6 +484,13 @@ arguments: column: col_email +# has_valid_string_case check +- criticality: error + check: + function: has_valid_string_case + arguments: + column: col1 + case: lower # is_valid_national_id check - criticality: error check: diff --git a/tests/unit/test_build_rules.py b/tests/unit/test_build_rules.py index 97aa48442..1d7048085 100644 --- a/tests/unit/test_build_rules.py +++ b/tests/unit/test_build_rules.py @@ -24,6 +24,7 @@ foreign_key, is_valid_ipv4_address, is_valid_email, + has_valid_string_case, is_ipv4_address_in_cidr, is_not_less_than, is_not_greater_than, @@ -773,6 +774,13 @@ def test_build_rules_by_metadata(): "criticality": "error", "check": {"function": "is_valid_email", "arguments": {"column": "a"}}, }, + { + "criticality": "error", + "check": { + "function": "has_valid_string_case", + "arguments": {"column": "a", "case": "lower"}, + }, + }, ] actual_rules = deserialize_checks(checks) @@ -1045,6 +1053,13 @@ def test_build_rules_by_metadata(): check_func=is_valid_email, column="a", ), + DQRowRule( + name="a_has_invalid_lower_string_case", + criticality="error", + check_func=has_valid_string_case, + column="a", + check_func_kwargs={"case": "lower"}, + ), ] assert pprint.pformat(actual_rules) == pprint.pformat(expected_rules) @@ -1380,6 +1395,12 @@ def test_convert_dq_rules_to_metadata(): criticality="error", check_func=is_valid_date, column="b", check_func_kwargs={"date_format": "yyyy-MM-dd"} ), DQRowRule(criticality="error", check_func=is_valid_json, column="col_json_str"), + DQRowRule( + criticality="error", + check_func=has_valid_string_case, + column="col1", + check_func_kwargs={"case": "lower"}, + ), DQRowRule( criticality="error", check_func=has_json_keys, @@ -1595,6 +1616,14 @@ def test_convert_dq_rules_to_metadata(): "arguments": {"column": "col_json_str"}, }, }, + { + "name": "col1_has_invalid_lower_string_case", + "criticality": "error", + "check": { + "function": "has_valid_string_case", + "arguments": {"column": "col1", "case": "lower"}, + }, + }, { "name": "col_json_str_does_not_have_json_keys", "criticality": "error", diff --git a/tests/unit/test_row_checks.py b/tests/unit/test_row_checks.py index 87ebd9c5c..49c220eeb 100644 --- a/tests/unit/test_row_checks.py +++ b/tests/unit/test_row_checks.py @@ -1,3 +1,4 @@ +from typing import cast import pytest from databricks.labs.dqx.utils import get_column_name_or_alias from databricks.labs.dqx.check_funcs import ( @@ -10,6 +11,7 @@ is_in_list, is_not_null_and_is_in_list, is_aggr_not_greater_than, + has_valid_string_case, is_ipv4_address_in_cidr, is_ipv6_address_in_cidr, is_valid_national_id, @@ -53,6 +55,23 @@ def test_col_is_in_list_missing_allowed_list(): is_in_list("a", allowed=[]) +@pytest.mark.parametrize( + "case, expected_message", + [ + ("camel", "'case' must be one of ['lower', 'sentence', 'title', 'upper'], got 'camel'"), + ("", "'case' must be one of ['lower', 'sentence', 'title', 'upper'], got ''"), + ("Upper", "'case' must be one of ['lower', 'sentence', 'title', 'upper'], got 'Upper'"), + (None, "'case' must be a string, got instead."), + (1, "'case' must be a string, got instead."), + ], +) +def test_has_valid_string_case_rejects_invalid_case(case: object, expected_message: str): + with pytest.raises(InvalidParameterError) as error: + has_valid_string_case("a", cast(str, case)) + + assert str(error.value) == expected_message + + def test_incorrect_aggr_type(): # With new implementation, invalid aggr_type triggers a warning (not immediate error) # The error occurs at runtime when the apply function is called