Warn when imbalance handling adds validation samples

supervised/base_automl.py

@@ -5,6 +5,7 @@
 import time
 import types
 import uuid
+import warnings
 from abc import ABC
 from copy import deepcopy
 
@@ -642,9 +643,11 @@ def _handle_drastic_imbalance(
             min_samples_per_class = max(
                 min_samples_per_class, self._validation_strategy.get("k_folds", 0)
             )
+        total_append_samples = 0
         for i in range(len(classes)):
             if cnts[i] < min_samples_per_class:
                 append_samples = min_samples_per_class - cnts[i]
+                total_append_samples += append_samples
                 new_X = (
                     X[y == classes[i]]
                     .sample(n=append_samples, replace=True, random_state=1)
@@ -673,6 +676,15 @@ def _handle_drastic_imbalance(
                         sensitive_features.loc[
                             sensitive_features.shape[0]
                         ] = new_sensitive_features.loc[j]
+        if total_append_samples > 0:
+            warnings.warn(
+                f"AutoML added {total_append_samples} duplicated sample(s) "
+                "before validation because some classes have fewer than "
+                f"{min_samples_per_class} samples. Validation folds and "
+                "out-of-fold predictions can include these duplicated rows.",
+                UserWarning,
+                stacklevel=2,
+            )
 
     def _save_data_info(self, X, y, sample_weight=None, sensitive_features=None):
         target_is_numeric = pd.api.types.is_numeric_dtype(y)

tests/tests_automl/test_handle_imbalance.py

@@ -1,5 +1,6 @@
 import shutil
 import unittest
+import warnings
 
 import numpy as np
 import pandas as pd
@@ -51,11 +52,16 @@ def test_handle_drastic_imbalance(self):
         y[10:12] = 2
         y = pd.Series(np.array(y), name="target")
         a._ml_task = MULTICLASS_CLASSIFICATION
-        a._handle_drastic_imbalance(X, y)
+        with warnings.catch_warnings(record=True) as record:
+            warnings.simplefilter("always")
+            a._handle_drastic_imbalance(X, y)
 
         self.assertEqual(X.shape[0], 130)
         self.assertEqual(X.shape[1], 3)
         self.assertEqual(y.shape[0], 130)
+        self.assertEqual(len(record), 1)
+        self.assertIn("AutoML added 30 duplicated sample(s)", str(record[0].message))
+        self.assertIn("out-of-fold predictions", str(record[0].message))
 
     def test_handle_drastic_imbalance_sample_weight(self):
         a = AutoML(
@@ -88,11 +94,16 @@ def test_handle_drastic_imbalance_sample_weight(self):
 
         y = pd.Series(np.array(y), name="target")
         a._ml_task = MULTICLASS_CLASSIFICATION
-        a._handle_drastic_imbalance(X, y, sample_weight)
+        with warnings.catch_warnings(record=True) as record:
+            warnings.simplefilter("always")
+            a._handle_drastic_imbalance(X, y, sample_weight)
 
         self.assertEqual(X.shape[0], 138)
         self.assertEqual(X.shape[1], 3)
         self.assertEqual(y.shape[0], 138)
+        self.assertEqual(len(record), 1)
+        self.assertIn("AutoML added 38 duplicated sample(s)", str(record[0].message))
+        self.assertIn("out-of-fold predictions", str(record[0].message))
 
         self.assertEqual(np.sum(sample_weight[100:119]), 0)
         self.assertEqual(np.sum(sample_weight[119:138]), 19 * 10)
@@ -126,7 +137,16 @@ def test_imbalance_dont_change_data_after_fit(self):
         y[10:12] = 2
         sample_weight = np.ones(rows)
 
-        a.fit(X, y, sample_weight=sample_weight)
+        with warnings.catch_warnings(record=True) as record:
+            warnings.simplefilter("always")
+            a.fit(X, y, sample_weight=sample_weight)
+
+        self.assertTrue(
+            any(
+                "AutoML added 30 duplicated sample(s)" in str(w.message)
+                for w in record
+            )
+        )
 
         # original data **without** inserted samples to handle imbalance
         self.assertEqual(X.shape[0], rows)