Fork test: PR #1317 - FEATURE: Added profile build for "has_no_outliers" check

docs/dqx/docs/reference/profiler.mdx

@@ -16,19 +16,19 @@ When running the code on a Databricks workspace, the workspace client is automat
 You only need the following code to create the workspace client if you run DQX on Databricks workspace:
 
 <Tabs>
-  <TabItem value="Python" label="Python" default>
-    ```python
-    from databricks.sdk import WorkspaceClient
-    from databricks.labs.dqx.profiler.profiler import DQProfiler
-    from databricks.labs.dqx.profiler.generator import DQGenerator
-    from databricks.labs.dqx.profiler.dlt_generator import DQDltGenerator
-
-    ws = WorkspaceClient()
-    profiler = DQProfiler(ws)
-    generator = DQGenerator(ws)
-    dlt_generator = DQDltGenerator(ws)
-    ```
-  </TabItem>
+    <TabItem value="Python" label="Python" default>
+        ```python
+        from databricks.sdk import WorkspaceClient
+        from databricks.labs.dqx.profiler.profiler import DQProfiler
+        from databricks.labs.dqx.profiler.generator import DQGenerator
+        from databricks.labs.dqx.profiler.dlt_generator import DQDltGenerator
+
+        ws = WorkspaceClient()
+        profiler = DQProfiler(ws)
+        generator = DQGenerator(ws)
+        dlt_generator = DQDltGenerator(ws)
+        ```
+    </TabItem>
 </Tabs>
 
 For external environments, such as CI servers or local machines, you can authenticate to Databricks using any method supported by the Databricks SDK. For detailed instructions, refer to the [default authentication flow](https://databricks-sdk-py.readthedocs.io/en/latest/authentication.html#default-authentication-flow).
@@ -68,6 +68,14 @@ The profiler supports extensive configuration options to customize behavior:
 | `limit`                     | `1000`        | Maximum number of records to analyze                                                                                                                                                                                                                                                                                                 |
 | `filter`                    | `None`        | Filter for the input data as a string SQL expression                                                                                                                                                                                                                                                                                 |
 | `llm_primary_key_detection` | `True`        | Detect primary keys for generation of `is_unique` rule                                                                                                                                                                                                                                                                               |
+| `outliers_ratio`                | `0.01`        | Generate `has_no_outliers` rule if the fraction of outliers (detected via MAD) is at or below this threshold                                                                                                                                                                                                                     |
+| `has_no_outliers`               | `False`       | Enable or disable `has_no_outliers` rule generation globally for all columns. Disabled by default due to performance considerations.                                                                                                                                                                                             |
+| `has_no_outliers_allow_columns` | `[]`          | Generate `has_no_outliers` rule only for the listed columns; all other columns are skipped. Mutually exclusive with `has_no_outliers_deny_columns`                                                                                                                                                                                    |
+| `has_no_outliers_deny_columns`  | `[]`          | Skip `has_no_outliers` rule generation for the listed columns; all other columns are processed. Mutually exclusive with `has_no_outliers_allow_columns`                                                                                                                                                                               |
+
+<Admonition type="note" title="Ratio thresholds are evaluated on the sampled data">
+Ratio-based decisions such as `outliers_ratio` (and the other `*_ratio` options) are evaluated on the **sampled** data (see `sample_fraction` / `limit`), not the full dataset. A `has_no_outliers` rule is therefore emitted based on the outlier fraction observed in the sample; the generated check itself still runs against the full dataset at apply time. For more representative rule generation on skewed data, increase `sample_fraction` or `limit`.
+</Admonition>
 
 ## DQProfile Structure
 
@@ -88,14 +96,15 @@ class DQProfile:
 Profiling data generates several types of `DQProfile` objects. Each profile can be used to generate a corresponding `DQRule`
 consisting of an input column, built-in DQX check function, and arguments inferred during profiling.
 
-| Profile Type                  | DQX Check Function            | Supported Data Types                          | Profiled Values                                                                                                               | Notes                                                                                                                                                                                               |
-| ----------------------------- | ----------------------------- | --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| is_not_null                   | `is_not_null`                 | All                                           |                                                                                                                               | Generated when `null_ratio <= max_null_ratio` for non-text columns, or when `null_ratio <= max_null_ratio` but `empty_ratio > max_empty_ratio` for text columns                                     |
-| is_not_null_or_empty          | `is_not_null_and_not_empty`   | `StringType`, `CharType`, or `VarcharType`    | `trim_strings`: Whether to trim whitespace before checking for empty strings                                                  | Generated for text columns when both `null_ratio <= max_null_ratio` and `empty_ratio <= max_empty_ratio`                                                                                            |
-| is_not_empty                  | `is_not_empty`                | `StringType`, `CharType`, or `VarcharType`    | `trim_strings`: Whether to trim whitespace before checking for empty strings                                                  | Generated for text columns when `null_ratio > max_null_ratio` (nulls allowed) but `empty_ratio <= max_empty_ratio`                                                                                 |
-| is_in                         | `is_in_list`                  | `StringType`, `CharType`, `VarcharType`, `IntegerType`, or `LongType` | `in`: List of distinct values from the profiled data                                                                          | List of values checks are inferred based on the `max_in_count` and `distinct_ratio` in the profiler options                                                                                         |
-| min_max                       | `is_in_range`                 | `NumericType`, `DateType`, `TimestampType`, or `TimestampNTZType` | `min`: Minimum value from the profiled data; `max`: Maximum value from the profiled data                                      | Values can be rounded by setting `"round": True` in the profiler options; Outlier values may be ignored based on the profiler options (e.g. `remove_outliers`, `num_sigmas`, and `outlier_columns`) |
-| is_unique                     | `is_unique`                   | All                                           | `reasoning`: Reason for inferring that a column is a primary key; `confidence`: Confidence score of the LLM-generated result  | Requires installation of [LLM Extensions](/docs/installation#optional-dependencies); Set `llm_primary_key_detection: True` in the profiler options to enable primary key detection                  |
+| Profile Type         | DQX Check Function          | Supported Data Types                                                  | Profiled Values                                                                                                              | Notes                                                                                                                                                                                               |
+|----------------------|-----------------------------|-----------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| is_not_null          | `is_not_null`               | All                                                                   |                                                                                                                              | Generated when `null_ratio <= max_null_ratio` for non-text columns, or when `null_ratio <= max_null_ratio` but `empty_ratio > max_empty_ratio` for text columns                                     |
+| is_not_null_or_empty | `is_not_null_and_not_empty` | `StringType`, `CharType`, or `VarcharType`                            | `trim_strings`: Whether to trim whitespace before checking for empty strings                                                 | Generated for text columns when both `null_ratio <= max_null_ratio` and `empty_ratio <= max_empty_ratio`                                                                                            |
+| is_not_empty         | `is_not_empty`              | `StringType`, `CharType`, or `VarcharType`                            | `trim_strings`: Whether to trim whitespace before checking for empty strings                                                 | Generated for text columns when `null_ratio > max_null_ratio` (nulls allowed) but `empty_ratio <= max_empty_ratio`                                                                                  |
+| is_in                | `is_in_list`                | `StringType`, `CharType`, `VarcharType`, `IntegerType`, or `LongType` | `in`: List of distinct values from the profiled data                                                                         | List of values checks are inferred based on the `max_in_count` and `distinct_ratio` in the profiler options                                                                                         |
+| min_max              | `is_in_range`               | `NumericType`, `DateType`, `TimestampType`, or `TimestampNTZType`     | `min`: Minimum value from the profiled data; `max`: Maximum value from the profiled data                                     | Values can be rounded by setting `"round": True` in the profiler options; Outlier values may be ignored based on the profiler options (e.g. `remove_outliers`, `num_sigmas`, and `outlier_columns`) |
+| has_no_outliers      | `has_no_outliers`           | `NumericType` (all numeric types)                                     |                                                                                                                              | Generated when the fraction of outliers detected via the Median Absolute Deviation (MAD) method is at or below `outliers_ratio`; outliers are values outside `median ± 3.5 × MAD`                   |
+| is_unique            | `is_unique`                 | All                                                                   | `reasoning`: Reason for inferring that a column is a primary key; `confidence`: Confidence score of the LLM-generated result | Requires installation of [LLM Extensions](/docs/installation#optional-dependencies); Set `llm_primary_key_detection: True` in the profiler options to enable primary key detection                  |
 
 ## DQGenerator methods
 
@@ -116,5 +125,6 @@ The `DQDltGenerator` class creates Delta Live Tables expectation statements from
 | `generate_dlt_rules` | Generates Delta Live Table rules in the specified language.        | `rules`: List of DQProfile objects; `action`: Optional violation action ("drop", "fail", or None); `language`: Target language ("SQL", "Python", or "Python_Dict").     | Yes                      |
 
 <Admonition type="info" title="Complete Profiling Guide">
-For comprehensive examples, advanced options, and best practices, see the [Data Profiling Guide](/docs/guide/data_profiling).
+    For comprehensive examples, advanced options, and best practices, see the [Data Profiling
+    Guide](/docs/guide/data_profiling).
 </Admonition> 

src/databricks/labs/dqx/check_funcs.py

@@ -14,6 +14,8 @@
 from pyspark.sql import types
 from pyspark.sql import Column, DataFrame, SparkSession
 from pyspark.sql.window import Window
+
+from databricks.labs.dqx.profiling_utils import calculate_median_absolute_deviation_bounds
 from databricks.labs.dqx.rule import register_rule, register_for_original_columns_preselection
 from databricks.labs.dqx.utils import (
     get_column_name_or_alias,
@@ -1311,13 +1313,9 @@ def apply(df: DataFrame) -> DataFrame:
         # Validate and compile the filter (rejecting unsafe SQL); keep None when no filter is given so
         # the MAD calculation stays a true no-op instead of filtering on F.lit(True).
         filter_condition = safe_filter_expr(row_filter) if row_filter else None
-        median, mad = _calculate_median_absolute_deviation(df, col_expr_str, filter_condition)
-        if median is not None and mad is not None:
-            median = float(median)
-            mad = float(mad)
-            # Create outlier condition
-            lower_bound = median - (3.5 * mad)
-            upper_bound = median + (3.5 * mad)
+        bounds = calculate_median_absolute_deviation_bounds(df, col_expr_str, filter_condition)
+        if bounds is not None:
+            lower_bound, upper_bound = bounds
             lower_bound_expr = get_limit_expr(lower_bound)
             upper_bound_expr = get_limit_expr(upper_bound)
 
@@ -4206,38 +4204,3 @@ def _validate_sql_query_params(query: str, merge_columns: list[str] | None) -> N
     invalid_columns = [col for col in merge_columns if not isinstance(col, str) or not col]
     if invalid_columns:
         raise InvalidParameterError("'merge_columns' entries must be non-empty strings.")
-
-
-def _calculate_median_absolute_deviation(
-    df: DataFrame, column: str, filter_condition: Column | None
-) -> tuple[Any, Any]:
-    """
-    Calculate the Median Absolute Deviation (MAD) for a numeric column.
-
-    The MAD is a robust measure of variability based on the median, calculated as:
-    MAD = median(|X_i - median(X)|)
-
-    This is useful for outlier detection as it is more robust to outliers than
-    standard deviation.
-
-    Args:
-        df: PySpark DataFrame
-        column: Name of the numeric column to calculate MAD for
-        filter_condition: Filter to apply before calculation (optional)
-
-    Returns:
-        The Median and Absolute Deviation values
-    """
-    if filter_condition is not None:
-        df = df.filter(filter_condition)
-
-    # Step 1: Calculate the median of the column
-    median_value = df.agg(F.percentile_approx(column, 0.5)).collect()[0][0]
-
-    # Step 2: Calculate absolute deviations from the median
-    df_with_deviations = df.select(F.abs(F.col(column) - F.lit(median_value)).alias("absolute_deviation"))
-
-    # Step 3: Calculate the median of absolute deviations
-    mad = df_with_deviations.agg(F.percentile_approx("absolute_deviation", 0.5)).collect()[0][0]
-
-    return median_value, mad

src/databricks/labs/dqx/config.py

@@ -92,6 +92,7 @@ class ProfilerConfig:
     # Override profiler default thresholds
     max_null_ratio: float | None = None
     max_empty_ratio: float | None = None
+    outliers_ratio: float | None = None
 
 
 @dataclass

src/databricks/labs/dqx/profiler/generator.py

@@ -443,11 +443,37 @@ def dq_generate_is_unique(column: str, criticality: str = "error", **params: dic
             "user_metadata": user_metadata,
         }
 
+    @staticmethod
+    def dq_generate_has_no_outliers(column: str, criticality: str = "error", **params: dict):
+        """Generates a data quality rule to check if a column's values contain no outliers.
+
+        Uses the MAD (Median Absolute Deviation) method via the *has_no_outliers* check function.
+        Values outside median ± 3.5 * MAD are flagged as outliers.
+
+        Args:
+            column: The name of the column to check.
+            criticality: The criticality of the rule as "warn" or "error" (default is "error").
+            params: Additional parameters (unused; left for compatibility with other functions).
+
+        Returns:
+            A dictionary representing the data quality rule.
+        """
+        params = params or {}  # consume the shared **params dispatch arg (see _checks_mapping call site)
+        return {
+            "check": {
+                "function": "has_no_outliers",
+                "arguments": {"column": column},
+            },
+            "name": f"{column}_has_no_outliers",
+            "criticality": criticality,
+        }
+
     _checks_mapping = {
         "is_not_null": dq_generate_is_not_null,
         "is_in": dq_generate_is_in,
         "min_max": dq_generate_min_max,
         "is_not_null_or_empty": dq_generate_is_not_null_or_empty,
         "is_not_empty": dq_generate_is_not_empty,
         "is_unique": dq_generate_is_unique,
+        "has_no_outliers": dq_generate_has_no_outliers,
     }