From 808a388c77d02324540a29f61e2b4b599100e962 Mon Sep 17 00:00:00 2001 From: Matt Faltyn Date: Sun, 19 Jul 2026 09:02:47 +0200 Subject: [PATCH 1/3] Fix deterministic fingerprints for set arguments --- src/databricks/labs/dqx/utils.py | 10 +++++++--- tests/unit/test_rule.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/dqx/utils.py b/src/databricks/labs/dqx/utils.py index 645ed6dda..2d304f11a 100644 --- a/src/databricks/labs/dqx/utils.py +++ b/src/databricks/labs/dqx/utils.py @@ -189,8 +189,8 @@ def normalize_bound_args(val: Any, allow_simple_expressions_only: bool = True) - """ Normalize a value or collection of values for consistent processing. - Handles primitives, dates, Decimal, and column-like objects. Lists, tuples, and sets are - recursively normalized with type preserved. + Handles primitives, dates, Decimal, and column-like objects. Collections are recursively + normalized, preserving list and tuple order while canonicalizing set and frozenset order. For Decimal values, uses a special JSON-serializable format to preserve type information for round-trip deserialization. @@ -211,7 +211,11 @@ def normalize_bound_args(val: Any, allow_simple_expressions_only: bool = True) - if val is None: return None - if isinstance(val, (list, tuple, set, frozenset)): + if isinstance(val, (set, frozenset)): + normalized = [normalize_bound_args(v, allow_simple_expressions_only) for v in val] + return sorted(normalized, key=lambda item: json.dumps(item, sort_keys=True, default=str)) + + if isinstance(val, (list, tuple)): return [normalize_bound_args(v, allow_simple_expressions_only) for v in val] if isinstance(val, dict): diff --git a/tests/unit/test_rule.py b/tests/unit/test_rule.py index 1af85718e..49853d76a 100644 --- a/tests/unit/test_rule.py +++ b/tests/unit/test_rule.py @@ -1,5 +1,7 @@ """Unit tests for rule fingerprinting, replacement, expansion, and serialization alignment.""" +from collections.abc import Iterator +from enum import Enum import re import pytest @@ -31,6 +33,29 @@ def test_compute_rule_fingerprint_same_rule_same_fingerprint(): assert _hex_sha256_pattern().match(fp1) +def test_compute_rule_fingerprint_set_arguments_order_independent(): + """Set arguments with mixed nested values produce the same fingerprint in any iteration order.""" + + class NestedValue(Enum): + DICT = {"nested": [1, "1"]} + + class IterationOrderedSet(set[object]): + def __init__(self, values: list[object]) -> None: + super().__init__(values) + self.iteration_order = values + + def __iter__(self) -> Iterator[object]: + return iter(self.iteration_order) + + values: list[object] = [NestedValue.DICT, 1, "1", ("pair", 2), frozenset({3, "3"})] + check = {"check": {"function": "is_in_list", "arguments": {"allowed": IterationOrderedSet(values)}}} + reversed_check = { + "check": {"function": "is_in_list", "arguments": {"allowed": IterationOrderedSet(list(reversed(values)))}} + } + + assert compute_rule_fingerprint(check) == compute_rule_fingerprint(reversed_check) + + def test_compute_rule_set_fingerprint_by_metadata_same_set_same_fingerprint(): """Same rule set produces the same rule_set_fingerprint (determinism).""" checks = [ From 5ce042e7327e4cf82ee27565e72d3e9cc4914854 Mon Sep 17 00:00:00 2001 From: Marcin Wojtyczka Date: Wed, 22 Jul 2026 13:16:28 +0200 Subject: [PATCH 2/3] Strengthen set-argument fingerprint test Add equivalence-to-plain-set and negative-case assertions to guard against the custom subclass masking behavior and the sort collapsing distinct inputs. Co-authored-by: Isaac --- tests/unit/test_rule.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_rule.py b/tests/unit/test_rule.py index 49853d76a..38ed32b78 100644 --- a/tests/unit/test_rule.py +++ b/tests/unit/test_rule.py @@ -52,8 +52,16 @@ def __iter__(self) -> Iterator[object]: reversed_check = { "check": {"function": "is_in_list", "arguments": {"allowed": IterationOrderedSet(list(reversed(values)))}} } + plain_set_check = {"check": {"function": "is_in_list", "arguments": {"allowed": set(values)}}} - assert compute_rule_fingerprint(check) == compute_rule_fingerprint(reversed_check) + fingerprint = compute_rule_fingerprint(check) + assert fingerprint == compute_rule_fingerprint(reversed_check) + # The custom subclass canonicalizes to the same fingerprint as a plain set of the same values. + assert fingerprint == compute_rule_fingerprint(plain_set_check) + + # A set with different values must produce a different fingerprint (sort does not collapse distinct inputs). + different_check = {"check": {"function": "is_in_list", "arguments": {"allowed": {1, "1", ("pair", 2)}}}} + assert compute_rule_fingerprint(different_check) != fingerprint def test_compute_rule_set_fingerprint_by_metadata_same_set_same_fingerprint(): From eff738776e496da077ac8d6163ddd5305d010779 Mon Sep 17 00:00:00 2001 From: Greg Hansen Date: Thu, 23 Jul 2026 12:33:11 -0400 Subject: [PATCH 3/3] Add test coverage for set and frozenset --- tests/unit/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ccd3e366c..77e4696ab 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -340,6 +340,8 @@ def test_is_simple_column_expression(column: str, expected: bool): # Collections ([1, 2, 3], [1, 2, 3]), ((4, 5, 6), [4, 5, 6]), + ({1, 2, 3}, [1, 2, 3]), + (frozenset({1, 2, 3}), [1, 2, 3]), # PySpark Column (F.col("col_name"), "col_name"), (F.col("a").alias("b"), "b"),