FairGBM openml API implementation

openml-h2o/src/main/java/com/feedzai/openml/h2o/H2OFeatureImportanceCreator.java

@@ -97,7 +97,7 @@ public VarImp calculateFeatureImportance(final Dataset dataset,
             final Path datasetPath = H2OUtils.writeDatasetToDisk(dataset);
             final Model model = this.h2OApp.train(gbm.getAlgorithmDescriptor(), datasetPath, dataset.getSchema(), params, random.nextLong());
             // safe cast as long as using H2OAlgorithm.GRADIENT_BOOSTING_MACHINE
-            return ((GBMModel) model)._output._varimp;
+            return model._output._varimp;
         } catch (final IOException e) {
             final String errorMessage = "An error occurred while calculating feature importance.";
             logger.error(errorMessage, e);

openml-h2o/src/main/java/com/feedzai/openml/h2o/H2OUtils.java

@@ -70,7 +70,7 @@ private H2OUtils() { }
      */
     public static Path writeDatasetToDisk(final Dataset dataset) throws IOException {
 
-        final Path datasetPath = Files.createTempFile(FEEDZAI_H2O_PREFIX + UUID.randomUUID().toString(), ".dataset");
+        final Path datasetPath = Files.createTempFile(FEEDZAI_H2O_PREFIX + UUID.randomUUID(), ".dataset");
         logger.info("Writing dataset to disk: {}", datasetPath);
 
         final DatasetSchema schema = dataset.getSchema();

openml-h2o/src/main/java/com/feedzai/openml/h2o/server/H2OApp.java

@@ -78,7 +78,7 @@ public class H2OApp<M extends Model> {
      * The lock used to initialize the instance.
      * @since 0.1.0
      */
-    private final static Object instanceLock = new Object();
+    private static final Object instanceLock = new Object();
 
     /**
      * Gets the {@link H2OApp} singleton instance.
@@ -159,11 +159,11 @@ public Model train(final MLAlgorithmDescriptor algorithm,
      */
     public Path export(final Model model,
                        final Path exportDir) throws IOException, ModelTrainingException {
-        logger.info("Exporting model {} to {}", model._output._job._result.toString(), exportDir.toAbsolutePath().toString());
+        logger.info("Exporting model {} to {}", model._output._job._result.toString(), exportDir.toAbsolutePath());
         if (model.haveMojo()) {
             // this workaround prevents an H2O bug where this parameter deserialization fails when it is deserialized as an integer value.
-            if (model instanceof IsolationForestModel && ((IsolationForestModel) model)._output._min_path_length == Long.MAX_VALUE) {
-                ((IsolationForestModel) model)._output._min_path_length = Integer.MAX_VALUE;
+            if (model instanceof IsolationForestModel && model._output._min_path_length == Long.MAX_VALUE) {
+                model._output._min_path_length = Integer.MAX_VALUE;
             }
             new MojoExported().save(exportDir, model);
         } else {

openml-lightgbm/lightgbm-builder/pom.xml

@@ -15,21 +15,28 @@
 
     <groupId>com.feedzai.openml.lightgbm</groupId>
     <artifactId>lightgbm-lib</artifactId>
-    <version>3.0.1-with_model_locale_fix_for_java_and_streaming</version>
+    <version>v3.2.1-fairgbm-alpha</version>
 
     <packaging>jar</packaging>
     <name>Openml LightGBM lib</name>
     <description>
         LightGBM build for Java generated with make-lightgbm.
     </description>
     <url>https://github.com/feedzai/make-lightgbm</url>
-
     <properties>
-        <!-- Microsoft hasn't merged our model-locale-fix patch yet. -->
-        <!--<lightgbm.repo.url>https://github.com/microsoft/LightGBM</lightgbm.repo.url>-->
-        <lightgbm.repo.url>https://github.com/feedzai/LightGBM.git</lightgbm.repo.url>
-        <lightgbmlib.version>3.0.1-with_model_locale_fix_for_java_and_streaming</lightgbmlib.version>
-        <lightgbm.version>v3.0.1-with_model_locale_fix_for_java_and_streaming</lightgbm.version>
+        <!-- Microsoft LightGBM -->
+        <!-- <lightgbm.repo.url>https://github.com/microsoft/LightGBM</lightgbm.repo.url> -->
+
+        <!-- Feedzai's custom LightGBM -->
+        <!-- <lightgbm.repo.url>https://github.com/feedzai/LightGBM.git</lightgbm.repo.url> -->
+
+        <!-- Feedzai's FairGBM! -->
+        <lightgbm.repo.url>https://github.com/feedzai/fairgbm.git</lightgbm.repo.url>
+
+<!--        <lightgbm.version>main-fairgbm</lightgbm.version>-->
+<!--        <lightgbm.version>python-api-fairgbm-update</lightgbm.version>-->
+        <lightgbm.version>fairgbm-python-api</lightgbm.version>
+        <lightgbmlib.version>v3.2.1-fairgbm-alpha</lightgbmlib.version>
     </properties>
 
     <build>

openml-lightgbm/lightgbm-provider/pom.xml

@@ -22,10 +22,10 @@
     <groupId>com.feedzai</groupId>
     <artifactId>openml-lightgbm</artifactId>
 
-    <description>OpenML Microsoft LightGBM Machine Learning Model and Classifier provider</description>
+    <description>OpenML LightGBM Machine Learning Model and Classifier provider</description>
 
     <properties>
-        <lightgbmlib.version>3.0.1-with_model_locale_fix_for_java_and_streaming</lightgbmlib.version>
+        <lightgbmlib.version>v3.2.1-fairgbm-alpha</lightgbmlib.version>
     </properties>
 
     <dependencies>

openml-lightgbm/lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/AlgoDescriptorUtil.java

@@ -0,0 +1,45 @@
+package com.feedzai.openml.provider.lightgbm;
+
+import com.feedzai.openml.provider.descriptor.fieldtype.NumericFieldType;
+
+public abstract class AlgoDescriptorUtil {
+
+    /**
+     * An alias to ease the readability of parameters' configuration that are not mandatory.
+     */
+    protected static final boolean NOT_MANDATORY = false;
+
+    /**
+     * An alias to ease the readability of parameters' configuration that are not mandatory.
+     */
+    protected static final boolean MANDATORY = true;
+
+    /**
+     * Helper method to return a range of type DOUBLE.
+     *
+     * @param minValue Minimum allowed value.
+     * @param maxValue Maximum allowed value.
+     * @param defaultValue Default value.
+     * @return Double range with the specs above.
+     */
+    protected static NumericFieldType doubleRange(final double minValue,
+                                                  final double maxValue,
+                                                  final double defaultValue) {
+        return NumericFieldType.range(minValue, maxValue, NumericFieldType.ParameterConfigType.DOUBLE, defaultValue);
+    }
+
+    /**
+     * Helper method to return a range of type INT.
+     *
+     * @param minValue Minimum allowed value.
+     * @param maxValue Maximum allowed value.
+     * @param defaultValue Default value.
+     * @return Integer range with the specs above.
+     */
+    protected static NumericFieldType intRange(final int minValue,
+                                               final int maxValue,
+                                               final int defaultValue) {
+        return NumericFieldType.range(minValue, maxValue, NumericFieldType.ParameterConfigType.INT, defaultValue);
+    }
+
+}

openml-lightgbm/lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/FairGBMDescriptorUtil.java

@@ -0,0 +1,203 @@
+/*
+ * Copyright 2022 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.feedzai.openml.provider.lightgbm;
+
+import com.feedzai.openml.provider.descriptor.ModelParameter;
+import com.feedzai.openml.provider.descriptor.fieldtype.ChoiceFieldType;
+import com.feedzai.openml.provider.descriptor.fieldtype.FreeTextFieldType;
+import com.feedzai.openml.provider.descriptor.fieldtype.NumericFieldType;
+import com.google.common.collect.ImmutableSet;
+
+import com.google.common.collect.Sets;
+import java.util.Set;
+
+/**
+ * Utility to organize all the necessary Machine Learning Hyper-Parameters for configuring the training of LightGBM.
+ *
+ * @author Andre Cruz (andre.cruz@feedzai.com)
+ * @since 1.2.1 // TODO: check what's the current version
+ */
+public class FairGBMDescriptorUtil extends LightGBMDescriptorUtil {
+
+    public static final String CONSTRAINT_GROUP_COLUMN_PARAMETER_NAME = "constraint_group_column";
+
+    /**
+     * Defines the set of model parameters accepted by the FairGBM algorithm.
+     *
+     * Parameters are sorted such that commonly changed parameters come first, and parameters that should seldom be
+     * changed come last.
+     */
+    static final Set<ModelParameter> PARAMS = Sets.union(ImmutableSet.of(
+            // The single parameter that will change for every different dataset
+            new ModelParameter(
+                    CONSTRAINT_GROUP_COLUMN_PARAMETER_NAME,
+                    "(Fairness) Sensitive group column",
+                    "Fairness constraints will be enforced over this column.\n"
+                            + "This column must be in categorical format.\n"
+                            + "Start this string with `name:` to use the name of a column here, \n"
+                            + "e.g., `name:age_group` for a column named `age_group`.",
+                    MANDATORY,
+                    new FreeTextFieldType("")
+//                    new FreeTextFieldType("", ".+")       # TODO: https://github.com/feedzai/feedzai-openml/issues/68
+            ),
+
+            new ModelParameter(
+                    "constraint_type",
+                    "(Fairness) Constraint type",
+                    "Enforces group-wise parity on the given target metric for the selected group column. "
+                            + "In general, FPR can be used for most fraud detection settings, "
+                            + "as we want to equalize the negative outcomes on legitimate individuals "
+                            + "(false positives).",
+                    NOT_MANDATORY,
+                    new ChoiceFieldType(ImmutableSet.of("FPR", "FNR", "FPR,FNR"), "FPR")
+            ),
+
+            // Parameters related to global constraints
+            new ModelParameter(
+                    "global_constraint_type",
+                    "(Fairness) Global constraint type",
+                    "Type of global constraint to enforce during training of fairness constraints. "
+                            + "For instance, if you want to deploy your model on 5% FPR you should use a global "
+                            + "constraint on 5% FPR, in order for the fairness constraints to be trained with this "
+                            + "FPR constraint in mind. Otherwise, fairness may not generalize well when you change "
+                            + "the model's operating point.",
+                    NOT_MANDATORY,
+                    new ChoiceFieldType(ImmutableSet.of("FPR", "FNR", "FPR,FNR"), "FPR,FNR")
+            ),
+            new ModelParameter(
+                    "global_target_fpr",
+                    "(Fairness) Global target FPR",
+                    "This is an inequality constraint: inactive when FPR is lower than the target. "
+                            + "If you have some (approximate) value of FPR in mind for deploying the model, then "
+                            + "set it here as well so that fairness constraints can better adapt to that value. "
+                            + "Oftentimes, some tension is required between global FPR and FNR constraints in order to "
+                            + "achieve the target values (a global constraint on FPR and FNR simultaneously).",
+                    NOT_MANDATORY,
+                    doubleRange(0.0, 1.0, 0.05)
+            ),
+            new ModelParameter(
+                    "global_target_fnr",
+                    "(Fairness) Global target FNR",
+                    "This is an inequality constraint: inactive when FNR is lower than the target. "
+                            + "If you have some (approximate) value of FNR in mind for deploying the model, then "
+                            + "set it here as well so that fairness constraints can better adapt to that value. "
+                            + "Oftentimes, some tension is required between global FPR and FNR constraints in order to "
+                            + "achieve the target values (a global constraint on FPR and FNR simultaneously).",
+                    NOT_MANDATORY,
+                    doubleRange(0.0, 1.0, 0.5)
+            ),
+
+            new ModelParameter(
+                    "objective",
+                    "(Fairness) Objective function",
+                    "For FairGBM you must use a constrained optimization function. "
+                            + "Currently, `constrained_cross_entropy` is recommended for most cases.",
+                    NOT_MANDATORY,
+                    new ChoiceFieldType(
+                            ImmutableSet.of("constrained_cross_entropy", "constrained_recall_objective"),
+                            "constrained_cross_entropy")
+            ),
+
+            // Slack on the fairness constraints
+            new ModelParameter(
+                    "constraint_fpr_threshold",
+                    "(Fairness) FPR slack for fairness",
+                    "The slack when fulfilling fairness FPR constraints. "
+                            + "The allowed difference between group-wise FPR. "
+                            + "When using the value 0.0 this will enforce group-wise FPR to be *exactly* equal. "
+                            + "Higher values lead to less strict fairness enforcement.",
+                    NOT_MANDATORY,
+                    doubleRange(0.0, 1.0, 0.0)
+            ),
+            new ModelParameter(
+                    "constraint_fnr_threshold",
+                    "(Fairness) FNR slack for fairness",
+                    "The slack when fulfilling fairness FNR constraints. "
+                            + "The allowed difference between group-wise FNR. "
+                            + "When using the value 0.0 this will enforce group-wise FNR to be *exactly* equal. "
+                            + "Higher values lead to less strict fairness enforcement.",
+                    NOT_MANDATORY,
+                    doubleRange(0.0, 1.0, 0.0)
+            ),
+
+            // Eventually we want this parameter to not depend as much on the size of the dataset
+            // But currently this needs to be changed for each dataset considering its size (larger for larger datasets)
+            // See: https://github.com/feedzai/fairgbm/issues/7
+            new ModelParameter(
+                    "multiplier_learning_rate",
+                    "(Fairness) Multipliers' learning rate",
+                    "The Lagrangian multipliers control how strong the constraint enforcement is.",
+                    NOT_MANDATORY,
+                    NumericFieldType.min(Float.MIN_VALUE, NumericFieldType.ParameterConfigType.DOUBLE, 1e3)
+            ), // NOTE: I'm using Float.MIN_VALUE here because the minimum value of a double in C++ depends on the architecture it's ran on, using float here is more conservative
+            new ModelParameter(
+                    "init_multipliers",
+                    "(Fairness) Initial multipliers",
+                    "The Lagrangian multipliers control how strong the constraint enforcement is. "
+                            + "The default value is starting with zero `0` for each constraint.",
+                    NOT_MANDATORY,
+                    new FreeTextFieldType("")
+//                    new FreeTextFieldType("", "^((\\d+(\\.\\d*)?,)*(\\d+(\\.\\d*)?))?$")  # TODO: https://github.com/feedzai/feedzai-openml/issues/68
+            ),
+
+            // These parameters probably shouldn't be changed in 90% of cases
+            new ModelParameter(
+                    "constraint_stepwise_proxy",
+                    "(Fairness) Stepwise proxy for fairness constraints",
+                    "The type of proxy function to use for the fairness constraint. "
+                            + "We need to use a differentiable proxy function, as FPR and FNR have discontinuous gradients.",
+                    NOT_MANDATORY,
+                    new ChoiceFieldType(ImmutableSet.of("cross_entropy", "quadratic", "hinge"), "cross_entropy")
+            ),
+            new ModelParameter(
+                    "objective_stepwise_proxy",
+                    "(Fairness) Stepwise proxy for global constraints",
+                    "The proxy function to use for the objective function. "
+                            + "Only used when explicitly optimizing for Recall (or any other metric of the "
+                            + "confusion matrix). Leave blank when using standard objectives such as cross-entropy.",
+                    NOT_MANDATORY,
+                    new ChoiceFieldType(ImmutableSet.of("cross_entropy", "quadratic", "hinge", ""), "")
+            )
+
+            // TODO: assess whether these parameters would ever be useful
+//            // These parameters probably shouldn't be changed in 99% of cases
+//            new ModelParameter(
+//                    "stepwise_proxy_margin",
+//                    "",
+//                    "",
+//                    NOT_MANDATORY,
+//                    new FreeTextFieldType("")
+//            ),
+//            new ModelParameter(
+//                    "score_threshold",
+//                    "",
+//                    "",
+//                    NOT_MANDATORY,
+//                    new FreeTextFieldType("")
+//            ),
+//            new ModelParameter(
+//                    "global_score_threshold",
+//                    "",
+//                    "",
+//                    NOT_MANDATORY,
+//                    new FreeTextFieldType("")
+//            )
+
+    ), LightGBMDescriptorUtil.PARAMS);
+
+}

openml-lightgbm/lightgbm-provider/src/main/java/com/feedzai/openml/provider/lightgbm/LightGBMAlgorithms.java

@@ -35,11 +35,21 @@ public enum LightGBMAlgorithms implements MLAlgorithmEnum {
      * LightGBM binary classifier algorithm.
      */
     LIGHTGBM_BINARY_CLASSIFIER(createDescriptor(
-            "LightGBM Binary Classifier",
+            "LightGBM",
             LightGBMDescriptorUtil.PARAMS,
             MachineLearningAlgorithmType.SUPERVISED_BINARY_CLASSIFICATION,
             "https://lightgbm.readthedocs.io/"
     )),
+
+    /**
+     * FairGBM binary classifier algorithm.
+     */
+    FAIRGBM_BINARY_CLASSIFIER(createDescriptor(
+            "FairGBM (LightGBM with Fairness)",
+            FairGBMDescriptorUtil.PARAMS,
+            MachineLearningAlgorithmType.SUPERVISED_BINARY_CLASSIFICATION,
+            "https://lightgbm.readthedocs.io/"  // TODO: link to our documentation
+    )),
     ;
 
     /**