Adjustment for Slurm operation

boosted_decision_tree.py

@@ -0,0 +1,69 @@
+import os
+# Keep using Keras 2
+os.environ['TF_USE_LEGACY_KERAS'] = '1'
+import locale
+# locale.setlocale(locale.LC_ALL, 'en_US')
+
+import tensorflow_decision_forests as tfdf
+
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+import tf_keras
+import math
+
+dataset_df = pd.read_csv("extracted_param_pairType_2.csv")
+
+# Display the first 3 examples.
+print(dataset_df.head(3))
+
+label = "cat"
+
+classes = dataset_df[label].unique().tolist()
+print(f"Label classes: {classes}")
+
+dataset_df[label] = dataset_df[label].map(classes.index)
+
+def split_dataset(dataset, test_ratio=0.20):
+  """Splits a panda dataframe in two."""
+  test_indices = np.random.rand(len(dataset)) < test_ratio
+  return dataset[~test_indices], dataset[test_indices]
+
+
+train_ds_pd, test_ds_pd = split_dataset(dataset_df)
+print("{} examples in training, {} examples for testing.".format(
+    len(train_ds_pd), len(test_ds_pd)))
+
+train_ds = tfdf.keras.pd_dataframe_to_tf_dataset(train_ds_pd, label=label)
+test_ds = tfdf.keras.pd_dataframe_to_tf_dataset(test_ds_pd, label=label)
+
+
+feature_1 = tfdf.keras.FeatureUsage(name="dau1_tauIdVSjet", semantic=tfdf.keras.FeatureSemantic.CATEGORICAL)
+feature_2 = tfdf.keras.FeatureUsage(name="dau1_pt")
+feature_3 = tfdf.keras.FeatureUsage(name="dau2_tauIdVSjet")
+feature_4 = tfdf.keras.FeatureUsage(name="dau2_pt")
+feature_5 = tfdf.keras.FeatureUsage(name="pNet_sum")
+feature_6 = tfdf.keras.FeatureUsage(name="bjet1_pt")
+feature_7 = tfdf.keras.FeatureUsage(name="bjet2_pt")
+feature_8 = tfdf.keras.FeatureUsage(name="bjet1_eta")
+feature_9 = tfdf.keras.FeatureUsage(name="bjet2_eta")
+all_features = [feature_1, feature_2, feature_3, feature_4, feature_5, feature_6, feature_7, feature_8, feature_9]
+
+# Specify the model.
+model_1 = tfdf.keras.GradientBoostedTreesModel(growing_strategy="LOCAL", num_trees=1000, max_depth=5,
+    split_axis="SPARSE_OBLIQUE",
+    categorical_algorithm="RANDOM",)
+
+# Train the model.
+model_1.fit(train_ds)
+
+model_1.compile(metrics=["accuracy"])
+evaluation = model_1.evaluate(test_ds, return_dict=True)
+print()
+
+for name, value in evaluation.items():
+  print(f"{name}: {value:.4f}")
+
+model_1.save("my_saved_model")
+
+tfdf.model_plotter.plot_model_in_colab(model_1, tree_idx=0, max_depth=3)

data_mutau_standard.json

@@ -0,0 +1,12 @@
+{
+    "vals": {
+        "mutau": [
+            47075385.0
+        ]
+    },
+    "errs": {
+        "mutau": [
+            0.0
+        ]
+    }
+}

dnn.py

@@ -0,0 +1,65 @@
+import os
+# Keep using Keras 2
+os.environ['TF_USE_LEGACY_KERAS'] = '1'
+import locale
+# locale.setlocale(locale.LC_ALL, 'en_US')
+
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+import tf_keras
+import math
+
+dataset_df = pd.read_csv("extracted_param_pairType_2.csv")
+
+# Display the first 3 examples.
+print(dataset_df.head(3))
+
+label = "cat"
+
+classes = dataset_df[label].unique().tolist()
+print(f"Label classes: {classes}")
+
+dataset_df[label] = dataset_df[label].map(classes.index)
+
+def split_dataset(dataset, test_ratio=0.20):
+  """Splits a panda dataframe in two."""
+  test_indices = np.random.rand(len(dataset)) < test_ratio
+  return dataset[~test_indices], dataset[test_indices]
+
+normalize = tf.keras.layers.Normalization()
+
+
+
+
+loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+
+train_ds_pd, test_ds_pd = split_dataset(dataset_df)
+train_features = train_ds_pd.copy()
+train_labels = train_features.pop('cat')
+print("{} examples in training, {} examples for testing.".format(
+    len(train_ds_pd), len(test_ds_pd)))
+
+train_features = np.array(train_features)
+normalize.adapt(train_features)
+model = tf.keras.Sequential([
+    normalize,
+    tf.keras.layers.Dense(256, activation='relu'),
+    tf.keras.layers.Dense(256, activation='relu'),
+    tf.keras.layers.Dense(4)
+])
+# Train the model.
+model.compile(loss = loss_fn,
+              optimizer = tf.keras.optimizers.Adam(), 
+              metrics=['accuracy'])
+
+model.fit(train_features, train_labels, epochs=20)
+# evaluation = model_1.evaluate(test_ds, return_dict=True)
+# print()
+
+# for name, value in evaluation.items():
+#   print(f"{name}: {value:.4f}")
+
+# model_1.save("my_saved_model")
+
+# tfdf.model_plotter.plot_model_in_colab(model_1, tree_idx=0, max_depth=3)

inclusion/DeepJetWPs.py

@@ -0,0 +1,37 @@
+#! /usr/bin/env python
+# Example of how to read the a correctionlib JSON file
+
+# This is a minimal version of a tau POG example:
+# For more information, see the README in
+# https://gitlab.cern.ch/cms-nanoAOD/jsonpog-integration/-/tree/master/POG/TAU
+#import sys; sys.path.insert(0,"correctionlib") # add correctionlib to path
+from correctionlib import _core
+
+# This corrections file is documented in https://cms-nanoaod-integration.web.cern.ch/commonJSONSFs/
+# qhere the general correctionlib documentation is kept
+fname = "/cvmfs/cms.cern.ch/rsync/cms-nanoAOD/jsonpog-integration/POG/BTV/2022_Summer22/btagging.json.gz"
+
+# Load CorrectionSet
+if fname.endswith(".json.gz"):
+  import gzip
+  with gzip.open(fname,'rt') as file:
+    #data = json.load(file)
+    data = file.read().strip()
+  cset = _core.CorrectionSet.from_string(data)
+else:
+  cset = _core.CorrectionSet.from_file(fname)
+
+# Load Correction objects that can be evaluated
+corr = cset["deepJet_wp_values"]
+
+btagWPl = corr.evaluate("L")
+btagWPm = corr.evaluate("M")
+btagWPt = corr.evaluate("T")
+btagWPxt = corr.evaluate("XT")
+btagWPxxt = corr.evaluate("XXT")
+
+print("L WP =", btagWPl)
+print("M WP =", btagWPm)
+print("T WP =", btagWPt)
+print("XT WP =", btagWPxt)
+print("XXT WP =", btagWPxxt)

inclusion/condor/closure.py

@@ -47,18 +47,27 @@ def closure(args)                    :
     jw.add_string('echo "{} for channel ${{1}} and single trigger ${{2}} done."'.format(script))
 
     #### Write submission file
-    jw.write_condor(filename=outs_submit,
+    if main.machine == "slurm": 
+            jw.write_batch(filename=outs_submit,
+                    real_exec=utils.build_script_path(script),
+                    shell_exec=outs_job,
+                    outfile=outs_check,
+                    logfile=outs_log,
+                    queue=main.queue,
+                    machine=main.machine)
+    else:
+        jw.write_condor(filename=outs_submit,
                     real_exec=utils.build_script_path(script),
                     shell_exec=outs_job,
                     outfile=outs_check,
                     logfile=outs_log,
                     queue=main.queue,
                     machine=main.machine)
 
-    qlines = []
-    for chn in args.channels:
-        for trig in args.closure_single_triggers:
-            qlines.append('  {},{}'.format(chn,trig))
+        qlines = []
+        for chn in args.channels:
+            for trig in args.closure_single_triggers:
+                qlines.append('  {},{}'.format(chn,trig))
 
-    jw.write_queue( qvars=('channel', 'closure_single_trigger'),
-                    qlines=qlines )
+        jw.write_queue( qvars=('channel', 'closure_single_trigger'),
+                        qlines=qlines )

inclusion/condor/discriminator.py

@@ -46,14 +46,23 @@ def discriminator(args):
         jw.add_string('echo "Script {} with channel {} done."'.format(script, args.channels[i]))
 
         #### Write submission file
-        jw.write_condor(filename=outs_submit[i],
-                        real_exec=utils.build_script_path(script),
-                        shell_exec=outs_job[i],
-                        outfile=outs_check[i],
-                        logfile=outs_log[i],
-                        queue=main.queue,
-                        machine=main.machine)
-        jw.write_queue()
+        if main.machine == "slurm": 
+            jw.write_batch(filename=outs_submit[i],
+                            real_exec=utils.build_script_path(script),
+                            shell_exec=outs_job[i],
+                            outfile=outs_check[i],
+                            logfile=outs_log[i],
+                            queue=main.queue,
+                            machine=main.machine)
+        else:
+            jw.write_condor(filename=outs_submit[i],
+                            real_exec=utils.build_script_path(script),
+                            shell_exec=outs_job[i],
+                            outfile=outs_check[i],
+                            logfile=outs_log[i],
+                            queue=main.queue,
+                            machine=main.machine)
+            jw.write_queue()
 
 # -- Parse options
 if __name__ == '__main__':

inclusion/condor/eff_and_sf.py

@@ -56,26 +56,49 @@ def eff_and_sf(args):
     jw.add_string('echo "{} done."'.format(script))
 
     #### Write submission file
-    jw.write_condor(filename=outs_submit,
-                    real_exec=utils.build_script_path(script),
-                    shell_exec=outs_job,
-                    outfile=outs_check,
-                    logfile=outs_log,
-                    queue=main.queue,
-                    machine=main.machine)
+    if main.machine == "slurm": 
+            cfg = importlib.import_module(args.configuration)
+            input_params = []
+            for chn in args.channels:
+                if chn == args.channels[0]:
+                    triggercomb = utils.generate_trigger_combinations(chn, cfg.triggers,
+                                                                    cfg.exclusive)
+                else:
+                    triggercomb += utils.generate_trigger_combinations(chn, cfg.triggers,
+                                                                    cfg.exclusive)
+
+            for tcomb in set(triggercomb):
+                input_params.append('"{}"'.format( utils.join_name_trigger_intersection(tcomb)) )
 
-    cfg = importlib.import_module(args.configuration)
-    qlines = []
-    for chn in args.channels:
-        if chn == args.channels[0]:
-            triggercomb = utils.generate_trigger_combinations(chn, cfg.triggers,
-                                                              cfg.exclusive)
-        else:
-            triggercomb += utils.generate_trigger_combinations(chn, cfg.triggers,
-                                                               cfg.exclusive)
-
-    for tcomb in set(triggercomb):
-        qlines.append('  {}'.format( utils.join_name_trigger_intersection(tcomb)) )
+            jw.write_batch(filename=outs_submit,
+                        real_exec=utils.build_script_path(script),
+                        shell_exec=outs_job,
+                        outfile=outs_check,
+                        logfile=outs_log,
+                        queue=main.queue,
+                        machine=main.machine,
+                        input_output_params=input_params )
+    else:
+        jw.write_condor(filename=outs_submit,
+                        real_exec=utils.build_script_path(script),
+                        shell_exec=outs_job,
+                        outfile=outs_check,
+                        logfile=outs_log,
+                        queue=main.queue,
+                        machine=main.machine)
 
-    jw.write_queue( qvars=('triggercomb',),
-                    qlines=qlines )
+        cfg = importlib.import_module(args.configuration)
+        qlines = []
+        for chn in args.channels:
+            if chn == args.channels[0]:
+                triggercomb = utils.generate_trigger_combinations(chn, cfg.triggers,
+                                                                cfg.exclusive)
+            else:
+                triggercomb += utils.generate_trigger_combinations(chn, cfg.triggers,
+                                                                cfg.exclusive)
+
+        for tcomb in set(triggercomb):
+            qlines.append('  {}'.format( utils.join_name_trigger_intersection(tcomb)) )
+
+        jw.write_queue( qvars=('triggercomb',),
+                        qlines=qlines )

inclusion/condor/eff_and_sf_aggr.py

@@ -41,17 +41,31 @@ def eff_and_sf_aggr(args):
     jw.add_string('echo "{} done."'.format(script))
 
     #### Write submission file
-    jw.write_condor(filename=outs_submit,
-                    real_exec=utils.build_script_path(script),
-                    shell_exec=outs_job,
-                    outfile=outs_check,
-                    logfile=outs_log,
-                    queue=main.queue,
-                    machine=main.machine)
-
-    qlines = []
-    for chn in args.channels:
-        qlines.append('  {}'.format(chn))
-
-    jw.write_queue( qvars=('channel',),
-                    qlines=qlines )
+    if main.machine == "slurm": 
+            input_params = []
+            for chn in args.channels:
+                input_params.append('"{}"'.format(chn))
+
+            jw.write_batch(filename=outs_submit,
+                        real_exec=utils.build_script_path(script),
+                        shell_exec=outs_job,
+                        outfile=outs_check,
+                        logfile=outs_log,
+                        queue=main.queue,
+                        machine=main.machine,
+                        input_output_params=input_params)
+    else:
+        jw.write_condor(filename=outs_submit,
+                        real_exec=utils.build_script_path(script),
+                        shell_exec=outs_job,
+                        outfile=outs_check,
+                        logfile=outs_log,
+                        queue=main.queue,
+                        machine=main.machine)
+
+        qlines = []
+        for chn in args.channels:
+            qlines.append('  {}'.format(chn))
+
+        jw.write_queue( qvars=('channel',),
+                        qlines=qlines )