From 05fe242cdcc9b327e4c65ad261d0b7b7fd608b69 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:11:43 +0100 Subject: [PATCH 01/19] add EventPreprocessor --- src/ctapipe/io/__init__.py | 3 + src/ctapipe/io/event_preprocessor.py | 197 ++++++++++++++++++ .../io/tests/test_event_preprocessor.py | 114 ++++++++++ 3 files changed, 314 insertions(+) create mode 100644 src/ctapipe/io/event_preprocessor.py create mode 100644 src/ctapipe/io/tests/test_event_preprocessor.py diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index ef85cd7ddfc..2ffa708d600 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -8,6 +8,7 @@ from .astropy_helpers import read_table, write_table # noqa: I001 from .datalevels import DataLevel from .dl2_tables_preprocessing import DL2EventPreprocessor, DL2EventLoader +from .event_preprocessor import EventPreprocessor, PreprocessorFeatureSet from .eventsource import EventSource from .eventseeker import EventSeeker from .tableio import TableReader, TableWriter @@ -46,4 +47,6 @@ "DL2EventPreprocessor", "DL2EventLoader", "get_hdf5_monitoring_types", + "EventPreprocessor", + "PreprocessorFeatureSet", ] diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py new file mode 100644 index 00000000000..66ecb81c9a1 --- /dev/null +++ b/src/ctapipe/io/event_preprocessor.py @@ -0,0 +1,197 @@ +"""Module containing classes related to event loading and preprocessing""" + +from enum import StrEnum, auto + +from astropy.coordinates import angular_separation +from traitlets import default + +from ..coordinates import altaz_to_fov +from ..core import ( + Component, + FeatureGenerator, + QualityQuery, + ToolConfigurationError, + traits, +) + +__all__ = ["EventPreprocessor"] + + +class PreprocessorFeatureSet(StrEnum): + """Pre-defined configurations for DL2EventPreprocessor for specific use cases.""" + + custom = auto() #: use user-supplied configuration + dl2_irf = auto() #: support IRF preprocessing use case + + +class EventPreprocessor(Component): + """ + Selects or generates features and filters tables of events. + + In normal use, one only has to specify the ``feature_set`` option, which + will generate features supports standard use cases. For advanced usage, you + can set ``feature_set=custom`` and pass in a configured + `~ctapipe.core.FeatureGenerator` and set the ``features`` property of this + class with the columns you to retain in the output table. + + In the `~ctapipe.core.FeatureGenerator`` used internally, you have access to several + additional functions useful for DL2 processing: + + - `~astropy.coordinates.angular_separation` + - `~ctapipe.coordinates.alt_az_to_fov` + """ + + energy_reconstructor = traits.Unicode( + default_value="RandomForestRegressor", + help="Prefix of the reco `_energy` column", + ).tag(config=True) + + geometry_reconstructor = traits.Unicode( + default_value="HillasReconstructor", + help="Prefix of the `_alt` and `_az` reco geometry columns", + ).tag(config=True) + + gammaness_reconstructor = traits.Unicode( + default_value="RandomForestClassifier", + help="Prefix of the classifier `_prediction` column", + ).tag(config=True) + + feature_set = traits.UseEnum( + PreprocessorFeatureSet, + default_value=PreprocessorFeatureSet.dl2_irf, + help=( + "Set up the FeatureGenerator.features, output features, and quality criteria " + "based on standard use cases." + "Specify 'custom' if you want to set your own in your config file. If this is set to " + "any value other than 'custom', the feature properties of the configuration " + "file you pass in will be overridden." + ), + ) + + features = traits.List( + traits.Unicode(), + help=( + "Features (columns) to retain in the output. " + "These can include columns generated by the FeatureGenerator. " + "If you set these, make sure feature_set=custom." + ), + ).tag(config=True) + + def __init__(self, config=None, parent=None, **kwargs): + super().__init__(config=config, parent=parent, **kwargs) + if PreprocessorFeatureSet(self.feature_set) == PreprocessorFeatureSet.custom: + self.feature_generator = FeatureGenerator(parent=self) + self.quality_query = QualityQuery(parent=self) + else: + self.feature_generator = FeatureGenerator( + features=self._get_predefined_features_to_generate() + ) + self.quality_query = QualityQuery( + quality_criteria=self._get_predefined_quality_criteria() + ) + # sanity checks: + if len(self.features) == 0: + raise ToolConfigurationError( + "DL2EventPreprocessor has no output features configured." + "You have set `feature_set=custom`, but did not provide the list " + "of features in the configuration (DL2EventPreprocessor.features)." + ) + + def __call__(self, table): + """Return new table with only the columns in features.""" + + # generate new features, which includes renaming columns: + generated = self.feature_generator( + table, angular_separation=angular_separation, altaz_to_fov=altaz_to_fov + ) + + # apply event selection on the resulting table + + selected_mask = self.quality_query.get_table_mask(generated) + + # return only the columns specified in `self.features`, and rows in + # `selected_mask` + return generated[self.features][selected_mask] + + def _get_predefined_features_to_generate(self) -> list[tuple]: + """Return a default list of FeatureGenerator features.""" + if self.feature_set == PreprocessorFeatureSet.dl2_irf: + # Default features for DL2/Subarray events + return [ + ("reco_energy", f"{self.energy_reconstructor}_energy"), + ("reco_alt", f"{self.geometry_reconstructor}_alt"), + ("reco_az", f"{self.geometry_reconstructor}_az"), + ("gh_score", f"{self.gammaness_reconstructor}_prediction"), + ("theta", "angular_separation(reco_az, reco_alt, true_az, true_alt)"), + ( + "reco_fov_coord", + "altaz_to_fov(reco_az, reco_alt, subarray_pointing_lon, subarray_pointing_lat)", + ), + ("reco_fov_lon", "reco_fov_coord[:,0]"), + ("reco_fov_lat", "reco_fov_coord[:,1]"), + ( + "true_fov_coord", + "altaz_to_fov(true_az, true_alt, subarray_pointing_lon, subarray_pointing_lat)", + ), + ("true_fov_lon", "true_fov_coord[:,0]"), + ("true_fov_lat", "true_fov_coord[:,1]"), + ( + "true_fov_offset", + "angular_separation(reco_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", + ), + ( + "reco_fov_offset", + "angular_separation(true_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", + ), + ( + "multiplicity", + f"np.count_nonzero({self.gammaness_reconstructor}_telescopes,axis=1)", + ), + ] + else: + raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") + + def _get_predefined_quality_criteria(self) -> list[tuple]: + """ + Set the quality criteria for a DL2FeatureSet. + + Here you can use any columns in the input table, or any that are + specified in the FeatureGenerator. + """ + if self.feature_set == PreprocessorFeatureSet.dl2_irf: + return [ + ("Valid geometry", f"{self.geometry_reconstructor}_is_valid"), + ("valid energy", f"{self.energy_reconstructor}_is_valid"), + ("valid gammaness", f"{self.gammaness_reconstructor}_is_valid"), + ("sufficient multiplicity", "multiplicity >= 4"), + ] + else: + raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") + + @default("features") + def default_features(self): + """Set the columns to output, for a given FeatureSet.""" + if self.feature_set == PreprocessorFeatureSet.dl2_irf: + return [ + "event_id", + "obs_id", + "reco_energy", + "reco_alt", + "reco_az", + "gh_score", + "true_energy", + "true_alt", + "true_az", + "true_fov_offset", + "reco_fov_offset", + "theta", + "reco_fov_lat", + "true_fov_lat", + "reco_fov_lon", + "true_fov_lon", + "multiplicity", + ] + elif self.feature_set == PreprocessorFeatureSet.custom: + return [] + else: + raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") diff --git a/src/ctapipe/io/tests/test_event_preprocessor.py b/src/ctapipe/io/tests/test_event_preprocessor.py new file mode 100644 index 00000000000..d3b9c631a26 --- /dev/null +++ b/src/ctapipe/io/tests/test_event_preprocessor.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +import numpy as np +import pytest +from astropy import units as u +from astropy.table import QTable + +from ctapipe.io import PreprocessorFeatureSet + + +@pytest.fixture(scope="function") +def minimal_dl2_table(): + """A dunmmy DL2 table for testing DL2EventPreprocessor""" + return QTable( + dict( + obs_id=[10, 10, 10, 10], + event_id=[1, 2, 3, 4], + true_energy=[100.0, 50.0, 2.0, 30.0] * u.TeV, + RandomForestRegressor_energy=[100.1, 49.2, 2.6, 40.0] * u.TeV, + RandomForestRegressor_is_valid=[True, True, True, True], + HillasReconstructor_az=[271.0, 271.6, 271.4, 268.1] * u.deg, + HillasReconstructor_alt=[70.1, 68.2, 69.3, 70.8] * u.deg, + HillasReconstructor_is_valid=[True, True, True, False], + RandomForestClassifier_prediction=[0.9, 0.5, 0.1, 0.3], + RandomForestClassifier_is_valid=[True, True, True, False], + true_alt=[70.0, 70.0, 70.0, 70.0] * u.deg, + true_az=[270.0, 270.0, 270.0, 270.0] * u.deg, + subarray_pointing_lat=[70.0, 70.0, 70.0, 70.0] * u.deg, + subarray_pointing_lon=[270.0, 270.0, 270.0, 270.0] * u.deg, + RandomForestClassifier_telescopes=np.array( + [ + [False, True, True, True], + [True, True, True, True], + [True, True, False, True], + [True, True, True, True], + ] + ), + ) + ) + + +@pytest.mark.parametrize("feature_set", list(PreprocessorFeatureSet)) +def test_event_preprocessing(feature_set, minimal_dl2_table): + from traitlets.config import Config + + from ctapipe.io import EventPreprocessor + + # set some custom features for the case where the feature_set==custom. + # These will be ignored in other feature_sets. + custom_config = Config() + custom_config.EventPreprocessor.features = ["obs_id", "event_id"] + table = minimal_dl2_table + + # process the table: + preprocess = EventPreprocessor(config=custom_config, feature_set=feature_set) + table_processed = preprocess(table) + + for feature in preprocess.features: + assert feature in table_processed.columns + + # check that the qualityquery worked + assert len(table_processed) <= len(table) + + +def test_no_output(): + """Check error is raised if no columns are specified for output.""" + from ctapipe.core import ToolConfigurationError + from ctapipe.io import EventPreprocessor, PreprocessorFeatureSet + + with pytest.raises(ToolConfigurationError): + EventPreprocessor(feature_set=PreprocessorFeatureSet.custom) + + +def test_nondefault_reconstructors(minimal_dl2_table): + """Check that using a different constructor than default still works""" + + from ctapipe.io import EventPreprocessor + + # define some new reconstructors, and add those columns to the test table: + geom = "ExampleGeometryReconstructor" + energy = "ExampleEnergyRegressor" + gammaness = "ExampleGammnessClassifier" + table = minimal_dl2_table + + table[f"{geom}_alt"] = ([71.1, 62.2, 61.3, 75.8] * u.deg,) + table[f"{geom}_az"] = [231.0, 231.6, 231.4, 238.1] * u.deg + table[f"{geom}_is_valid"] = [True, False, True, True] + + table[f"{energy}_energy"] = [20.0, 1.0, 0.5, 0.1] * u.TeV + table[f"{energy}_is_valid"] = [True, False, True, True] + + table[f"{gammaness}_prediction"] = [0.1, 0.8, 0.9, 0.7] + table[f"{gammaness}_is_valid"] = [True, False, True, True] + table[f"{gammaness}_telescopes"] = table["RandomForestClassifier_telescopes"] + + preprocess = EventPreprocessor( + feature_set=PreprocessorFeatureSet.dl2_irf, + geometry_reconstructor=geom, + energy_reconstructor=energy, + gammaness_reconstructor=gammaness, + ) + + table_processed = preprocess(table) + + # check that the processing worked. In this case, we check that the + # requested columns are renamed correctly and that the filtered values match + # the original values. + + mask = table["event_id"] == table_processed["event_id"] + masked = table[mask] # so that we just compare values after filtering + + assert np.allclose(table_processed["reco_energy"], masked[f"{energy}_energy"]) + assert np.allclose(table_processed["reco_az"], masked[f"{geom}_az"]) + assert np.allclose(table_processed["gh_score"], masked[f"{gammaness}_prediction"]) From 25675844f6fc6af741550e871c642cb9af535567 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:13:28 +0100 Subject: [PATCH 02/19] added altaz_to_fov helper --- src/ctapipe/coordinates/tests/test_utils.py | 15 ++++++++++++ src/ctapipe/coordinates/utils.py | 26 ++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/ctapipe/coordinates/tests/test_utils.py b/src/ctapipe/coordinates/tests/test_utils.py index 1ebd419c679..52615a3c81d 100644 --- a/src/ctapipe/coordinates/tests/test_utils.py +++ b/src/ctapipe/coordinates/tests/test_utils.py @@ -53,3 +53,18 @@ def test_single_telescope(subarray_prod5_paranal): # 10 km is around the shower maximum, should be around 1 degree from the source with pytest.warns(MissingFrameAttributeWarning): assert u.isclose(source.separation(point), 1.0 * u.deg, atol=0.1 * u.deg) + + +def test_altaz_to_fov(): + from ctapipe.coordinates import altaz_to_fov + + column = altaz_to_fov( + az=[220.0, 220.2] * u.deg, + alt=[80.0, 79.2] * u.deg, + pointing_az=[220.0, 220.0] * u.deg, + pointing_alt=[80.0, 80.0] * u.deg, + ) + + assert column.unit == u.deg + assert np.allclose(column[0].value, 0) + assert np.allclose(column[1].value, [-0.03747984, -0.79993558]) diff --git a/src/ctapipe/coordinates/utils.py b/src/ctapipe/coordinates/utils.py index e8c2310b776..a2514bb681c 100644 --- a/src/ctapipe/coordinates/utils.py +++ b/src/ctapipe/coordinates/utils.py @@ -1,14 +1,16 @@ import astropy.units as u import numpy as np -from astropy.coordinates import AltAz +from astropy.coordinates import AltAz, SkyCoord from erfa.ufunc import p2s as cartesian_to_spherical from erfa.ufunc import s2p as spherical_to_cartesian from .ground_frames import _get_xyz +from .nominal_frame import NominalFrame __all__ = [ "altaz_to_righthanded_cartesian", "get_point_on_shower_axis", + "altaz_to_fov", ] @@ -80,3 +82,25 @@ def get_point_on_shower_axis(core_x, core_y, alt, az, telescope_position, distan cartesian = point[np.newaxis, :] - _get_xyz(telescope_position).T lon, lat, _ = cartesian_to_spherical(cartesian) return AltAz(alt=lat, az=-lon, copy=False) + + +def altaz_to_fov(az, alt, pointing_az, pointing_alt) -> u.Quantity[2]: + """ + Compute FOV coordinates from alt/az coordinates. + + This can be used in an FeatureGenerator or ExpressionEngine to get a single + column with fov_lon, fov_lat coordinates. + + Returns + ------- + u.Quantity[2]: + 2D array of coordinates with 2 columns: fov_lon, fov_lat + """ + pointing_coord = SkyCoord(az=pointing_az, alt=pointing_alt, frame="altaz") + event_coord = SkyCoord(az=az, alt=alt, frame="altaz", origin=pointing_coord) + nominal_coord = event_coord.transform_to(NominalFrame) + + # note the minus sign for the fov_lon, which is to match GADF conventions + return u.Quantity( + np.column_stack((-nominal_coord.fov_lon.deg, nominal_coord.fov_lat.deg)), u.deg + ) From 4a553a16f5f6c8b43babdd011c561a341d40042f Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:28:33 +0100 Subject: [PATCH 03/19] added changelog --- docs/changes/2928.feature.rst | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/changes/2928.feature.rst diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst new file mode 100644 index 00000000000..c8d993066fe --- /dev/null +++ b/docs/changes/2928.feature.rst @@ -0,0 +1,36 @@ +Introduces the `~ctapipe.io.EventPreprocessor` class that can generically +transform an event table by applying the following steps: + +* Generate new or rename existing columns with a `~ctapipe.core.FeatureGenerator` +* Select "good" event rows with a `~ctapipe.core.QualityQuery` +* Select which columns to output (by setting the ``features`` configuration + attribute of the `~ctapipe.io.EventPreprocessor`) + +This is useful for doing the final steps of DL2 processing, and will eventually +replace what is in `DL2EventPreprocessor` and `DL2EventLoader`, which will be +deprecated in a future release. + +The `~ctapipe.core.EventPreprocessor` also includes the ability to pre-configure +itself for specific use cases by setting the ``feature_set`` option. Currently +only two `~ctapipe.io.ProcessingFeatureSet` are implemented: +`feature_set=dl2_irf`, which defines the transforms, event selection, and output +features for processing simulated DL2 events, and `feature_set=custom`, which +has no pre-configuration and requires all parameters to be set by the user in a +config file. + +The functionality of `DL2EventLoader` can be mimicked with the following: + +.. code-block:: python + + from ctapipe.io import TableLoader, EventPreprocessor + from astropy.table import vstack + + DL2FILE = "some_dl2_file.h5" + loader = TableLoader(DL2FILE, dl2=True, simulated=True, observation_info=True) + preprocess = EventPreprocessor(feature_set="dl2_simulation") + events = vstack( + [ + preprocess(QTable(c.data)) + for c in loader.read_subarray_events_chunked(chunk_size=100_000) + ] + ) From e93db1274f5e30c722be3dca02a375374896333c Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:31:46 +0100 Subject: [PATCH 04/19] add alt_az_to_fov to init --- docs/changes/2928.feature.rst | 6 ++++++ src/ctapipe/coordinates/__init__.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index c8d993066fe..3a487c175bf 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -34,3 +34,9 @@ The functionality of `DL2EventLoader` can be mimicked with the following: for c in loader.read_subarray_events_chunked(chunk_size=100_000) ] ) + + + This also introduces a helper function `~ctapipe.coordinates.altaz_to_fov` to + convert columns of alt/az coordinates to FOV coordinates in the + `~ctapipe.coordinates.NominalFrame`, which works with the + `~ctapipe.io.FeatureGenerator`. diff --git a/src/ctapipe/coordinates/__init__.py b/src/ctapipe/coordinates/__init__.py index 910ab215edd..afa129c0c72 100644 --- a/src/ctapipe/coordinates/__init__.py +++ b/src/ctapipe/coordinates/__init__.py @@ -21,7 +21,11 @@ from .impact_distance import impact_distance, shower_impact_distance from .nominal_frame import NominalFrame from .telescope_frame import TelescopeFrame -from .utils import altaz_to_righthanded_cartesian, get_point_on_shower_axis +from .utils import ( + altaz_to_fov, + altaz_to_righthanded_cartesian, + get_point_on_shower_axis, +) __all__ = [ "TelescopeFrame", @@ -37,6 +41,7 @@ "impact_distance", "shower_impact_distance", "get_point_on_shower_axis", + "altaz_to_fov", ] From f1a15a3e7ad16b04f981089e98cd6d80ca3a48e2 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:44:45 +0100 Subject: [PATCH 05/19] add missing config=True tag --- src/ctapipe/io/event_preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index 66ecb81c9a1..b1f8e0e71d7 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -66,7 +66,7 @@ class with the columns you to retain in the output table. "any value other than 'custom', the feature properties of the configuration " "file you pass in will be overridden." ), - ) + ).tag(config=True) features = traits.List( traits.Unicode(), From 65d1f01bc8c98287d20bea30947ad87e9b87ea35 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 11:47:52 +0100 Subject: [PATCH 06/19] fix some docstring/type annotation warnings --- src/ctapipe/coordinates/utils.py | 6 +++--- src/ctapipe/io/event_preprocessor.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ctapipe/coordinates/utils.py b/src/ctapipe/coordinates/utils.py index a2514bb681c..5b8ab3b362d 100644 --- a/src/ctapipe/coordinates/utils.py +++ b/src/ctapipe/coordinates/utils.py @@ -84,16 +84,16 @@ def get_point_on_shower_axis(core_x, core_y, alt, az, telescope_position, distan return AltAz(alt=lat, az=-lon, copy=False) -def altaz_to_fov(az, alt, pointing_az, pointing_alt) -> u.Quantity[2]: +def altaz_to_fov(az, alt, pointing_az, pointing_alt) -> u.Quantity: """ Compute FOV coordinates from alt/az coordinates. - This can be used in an FeatureGenerator or ExpressionEngine to get a single + This can be used in a FeatureGenerator or ExpressionEngine to get a single column with fov_lon, fov_lat coordinates. Returns ------- - u.Quantity[2]: + u.Quantity: 2D array of coordinates with 2 columns: fov_lon, fov_lat """ pointing_coord = SkyCoord(az=pointing_az, alt=pointing_alt, frame="altaz") diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index b1f8e0e71d7..ca555cbcd2e 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -38,7 +38,7 @@ class with the columns you to retain in the output table. additional functions useful for DL2 processing: - `~astropy.coordinates.angular_separation` - - `~ctapipe.coordinates.alt_az_to_fov` + - `~ctapipe.coordinates.altaz_to_fov` """ energy_reconstructor = traits.Unicode( From 6bea57c7d1dec92fac4e3bea2fe2576235bbd0b1 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 13:44:56 +0100 Subject: [PATCH 07/19] Don't use GADF FOV convention by default FOV coordinates are now in the ctapipe nominal frame. The conversion to GADF with negative lon, should be done explicitly on export if needed. --- docs/changes/2928.feature.rst | 2 +- src/ctapipe/coordinates/__init__.py | 4 ++-- src/ctapipe/coordinates/utils.py | 8 ++++---- src/ctapipe/io/event_preprocessor.py | 22 +++++++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index 3a487c175bf..562401cf728 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -36,7 +36,7 @@ The functionality of `DL2EventLoader` can be mimicked with the following: ) - This also introduces a helper function `~ctapipe.coordinates.altaz_to_fov` to + This also introduces a helper function `~ctapipe.coordinates.altaz_to_nominal` to convert columns of alt/az coordinates to FOV coordinates in the `~ctapipe.coordinates.NominalFrame`, which works with the `~ctapipe.io.FeatureGenerator`. diff --git a/src/ctapipe/coordinates/__init__.py b/src/ctapipe/coordinates/__init__.py index afa129c0c72..7d6ed4ea864 100644 --- a/src/ctapipe/coordinates/__init__.py +++ b/src/ctapipe/coordinates/__init__.py @@ -22,7 +22,7 @@ from .nominal_frame import NominalFrame from .telescope_frame import TelescopeFrame from .utils import ( - altaz_to_fov, + altaz_to_nominal, altaz_to_righthanded_cartesian, get_point_on_shower_axis, ) @@ -41,7 +41,7 @@ "impact_distance", "shower_impact_distance", "get_point_on_shower_axis", - "altaz_to_fov", + "altaz_to_nominal", ] diff --git a/src/ctapipe/coordinates/utils.py b/src/ctapipe/coordinates/utils.py index 5b8ab3b362d..04a2475c2a2 100644 --- a/src/ctapipe/coordinates/utils.py +++ b/src/ctapipe/coordinates/utils.py @@ -10,7 +10,7 @@ __all__ = [ "altaz_to_righthanded_cartesian", "get_point_on_shower_axis", - "altaz_to_fov", + "altaz_to_nominal", ] @@ -84,9 +84,9 @@ def get_point_on_shower_axis(core_x, core_y, alt, az, telescope_position, distan return AltAz(alt=lat, az=-lon, copy=False) -def altaz_to_fov(az, alt, pointing_az, pointing_alt) -> u.Quantity: +def altaz_to_nominal(az, alt, pointing_az, pointing_alt) -> u.Quantity: """ - Compute FOV coordinates from alt/az coordinates. + Compute nominal (FOV) coordinates from alt/az coordinates. This can be used in a FeatureGenerator or ExpressionEngine to get a single column with fov_lon, fov_lat coordinates. @@ -102,5 +102,5 @@ def altaz_to_fov(az, alt, pointing_az, pointing_alt) -> u.Quantity: # note the minus sign for the fov_lon, which is to match GADF conventions return u.Quantity( - np.column_stack((-nominal_coord.fov_lon.deg, nominal_coord.fov_lat.deg)), u.deg + np.column_stack((nominal_coord.fov_lon.deg, nominal_coord.fov_lat.deg)), u.deg ) diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index ca555cbcd2e..af6fa2e9945 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -5,7 +5,7 @@ from astropy.coordinates import angular_separation from traitlets import default -from ..coordinates import altaz_to_fov +from ..coordinates import altaz_to_nominal from ..core import ( Component, FeatureGenerator, @@ -38,7 +38,7 @@ class with the columns you to retain in the output table. additional functions useful for DL2 processing: - `~astropy.coordinates.angular_separation` - - `~ctapipe.coordinates.altaz_to_fov` + - `~ctapipe.coordinates.altaz_to_nominal` """ energy_reconstructor = traits.Unicode( @@ -102,7 +102,9 @@ def __call__(self, table): # generate new features, which includes renaming columns: generated = self.feature_generator( - table, angular_separation=angular_separation, altaz_to_fov=altaz_to_fov + table, + angular_separation=angular_separation, + altaz_to_nominal=altaz_to_nominal, ) # apply event selection on the resulting table @@ -125,15 +127,21 @@ def _get_predefined_features_to_generate(self) -> list[tuple]: ("theta", "angular_separation(reco_az, reco_alt, true_az, true_alt)"), ( "reco_fov_coord", - "altaz_to_fov(reco_az, reco_alt, subarray_pointing_lon, subarray_pointing_lat)", + "altaz_to_nominal(reco_az, reco_alt, subarray_pointing_lon, subarray_pointing_lat)", ), - ("reco_fov_lon", "reco_fov_coord[:,0]"), + ( + "reco_fov_lon", + "reco_fov_coord[:,0]", + ), # note: GADF IRFs use the negative of this ("reco_fov_lat", "reco_fov_coord[:,1]"), ( "true_fov_coord", - "altaz_to_fov(true_az, true_alt, subarray_pointing_lon, subarray_pointing_lat)", + "altaz_to_nominal(true_az, true_alt, subarray_pointing_lon, subarray_pointing_lat)", ), - ("true_fov_lon", "true_fov_coord[:,0]"), + ( + "true_fov_lon", + "true_fov_coord[:,0]", + ), # note: GADF IRFs use the negative of this ("true_fov_lat", "true_fov_coord[:,1]"), ( "true_fov_offset", From 6e97c93ba8e4b944264cbd01cf24009cf067151c Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 13:51:05 +0100 Subject: [PATCH 08/19] rename function in test too --- src/ctapipe/coordinates/tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/coordinates/tests/test_utils.py b/src/ctapipe/coordinates/tests/test_utils.py index 52615a3c81d..0fd18db95d0 100644 --- a/src/ctapipe/coordinates/tests/test_utils.py +++ b/src/ctapipe/coordinates/tests/test_utils.py @@ -56,9 +56,9 @@ def test_single_telescope(subarray_prod5_paranal): def test_altaz_to_fov(): - from ctapipe.coordinates import altaz_to_fov + from ctapipe.coordinates import altaz_to_nominal - column = altaz_to_fov( + column = altaz_to_nominal( az=[220.0, 220.2] * u.deg, alt=[80.0, 79.2] * u.deg, pointing_az=[220.0, 220.0] * u.deg, From 87b43dc6883287ba1ddd0c21624ac3b590bc77b8 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 14:43:46 +0100 Subject: [PATCH 09/19] fix test after GADF -> Nominal change --- src/ctapipe/coordinates/tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/coordinates/tests/test_utils.py b/src/ctapipe/coordinates/tests/test_utils.py index 0fd18db95d0..6d41dfda4f0 100644 --- a/src/ctapipe/coordinates/tests/test_utils.py +++ b/src/ctapipe/coordinates/tests/test_utils.py @@ -55,7 +55,7 @@ def test_single_telescope(subarray_prod5_paranal): assert u.isclose(source.separation(point), 1.0 * u.deg, atol=0.1 * u.deg) -def test_altaz_to_fov(): +def test_altaz_to_nominal(): from ctapipe.coordinates import altaz_to_nominal column = altaz_to_nominal( @@ -67,4 +67,4 @@ def test_altaz_to_fov(): assert column.unit == u.deg assert np.allclose(column[0].value, 0) - assert np.allclose(column[1].value, [-0.03747984, -0.79993558]) + assert np.allclose(column[1].value, [0.03747984, -0.79993558]) From 5da834ae907b4e0afa2e72bdacf29e8d7f367b12 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 16:03:39 +0100 Subject: [PATCH 10/19] fix links in changelog --- docs/changes/2928.feature.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index 562401cf728..d79e53854d0 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -7,18 +7,18 @@ transform an event table by applying the following steps: attribute of the `~ctapipe.io.EventPreprocessor`) This is useful for doing the final steps of DL2 processing, and will eventually -replace what is in `DL2EventPreprocessor` and `DL2EventLoader`, which will be +replace what is in `~ctapipe.io.DL2EventPreprocessor` and `~ctapipe.io.DL2EventLoader`, which will be deprecated in a future release. -The `~ctapipe.core.EventPreprocessor` also includes the ability to pre-configure +The `~ctapipe.io.EventPreprocessor` also includes the ability to pre-configure itself for specific use cases by setting the ``feature_set`` option. Currently only two `~ctapipe.io.ProcessingFeatureSet` are implemented: -`feature_set=dl2_irf`, which defines the transforms, event selection, and output -features for processing simulated DL2 events, and `feature_set=custom`, which +``feature_set=dl2_irf``, which defines the transforms, event selection, and output +features for processing simulated DL2 events, and ``feature_set=custom``, which has no pre-configuration and requires all parameters to be set by the user in a config file. -The functionality of `DL2EventLoader` can be mimicked with the following: +The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the following: .. code-block:: python From 096fd64176e4f923e6f5e7895dfd18171713c72a Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 16:04:32 +0100 Subject: [PATCH 11/19] pass parent to predefined QualityQuery --- src/ctapipe/io/event_preprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index af6fa2e9945..ce5786f647d 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -84,10 +84,10 @@ def __init__(self, config=None, parent=None, **kwargs): self.quality_query = QualityQuery(parent=self) else: self.feature_generator = FeatureGenerator( - features=self._get_predefined_features_to_generate() + parent=self, features=self._get_predefined_features_to_generate() ) self.quality_query = QualityQuery( - quality_criteria=self._get_predefined_quality_criteria() + parent=self, quality_criteria=self._get_predefined_quality_criteria() ) # sanity checks: if len(self.features) == 0: From 3fd0513115085f44a1471f6c79c9ac40b54e818c Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 16:06:10 +0100 Subject: [PATCH 12/19] remove old comment --- src/ctapipe/coordinates/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ctapipe/coordinates/utils.py b/src/ctapipe/coordinates/utils.py index 04a2475c2a2..52a993357ff 100644 --- a/src/ctapipe/coordinates/utils.py +++ b/src/ctapipe/coordinates/utils.py @@ -100,7 +100,6 @@ def altaz_to_nominal(az, alt, pointing_az, pointing_alt) -> u.Quantity: event_coord = SkyCoord(az=az, alt=alt, frame="altaz", origin=pointing_coord) nominal_coord = event_coord.transform_to(NominalFrame) - # note the minus sign for the fov_lon, which is to match GADF conventions return u.Quantity( np.column_stack((nominal_coord.fov_lon.deg, nominal_coord.fov_lat.deg)), u.deg ) From fa6de2a2ede1d8c423d9e749e0b76e49d449cfa5 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 16:07:13 +0100 Subject: [PATCH 13/19] remove unnecessary conversion --- src/ctapipe/io/event_preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index ce5786f647d..a3e7deb2e63 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -79,7 +79,7 @@ class with the columns you to retain in the output table. def __init__(self, config=None, parent=None, **kwargs): super().__init__(config=config, parent=parent, **kwargs) - if PreprocessorFeatureSet(self.feature_set) == PreprocessorFeatureSet.custom: + if self.feature_set == PreprocessorFeatureSet.custom: self.feature_generator = FeatureGenerator(parent=self) self.quality_query = QualityQuery(parent=self) else: From fdffce3e98077f970c3c9a1651880e0fb04145b3 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 17:07:22 +0100 Subject: [PATCH 14/19] fix links in changelog --- docs/changes/2928.feature.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index d79e53854d0..4cf26495dc4 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -12,7 +12,7 @@ deprecated in a future release. The `~ctapipe.io.EventPreprocessor` also includes the ability to pre-configure itself for specific use cases by setting the ``feature_set`` option. Currently -only two `~ctapipe.io.ProcessingFeatureSet` are implemented: +only two `~ctapipe.io.PreprocessorFeatureSet` are implemented: ``feature_set=dl2_irf``, which defines the transforms, event selection, and output features for processing simulated DL2 events, and ``feature_set=custom``, which has no pre-configuration and requires all parameters to be set by the user in a @@ -36,7 +36,7 @@ The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the follo ) - This also introduces a helper function `~ctapipe.coordinates.altaz_to_nominal` to - convert columns of alt/az coordinates to FOV coordinates in the - `~ctapipe.coordinates.NominalFrame`, which works with the - `~ctapipe.io.FeatureGenerator`. +This also introduces a helper function `~ctapipe.coordinates.altaz_to_nominal` +to convert columns of alt/az coordinates to FOV coordinates in the +`~ctapipe.coordinates.NominalFrame`, which works with the +`~ctapipe.io.FeatureGenerator`. From 88657f59ad3d0c6c6fe8629c84ba8ea76908e026 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Wed, 28 Jan 2026 17:09:45 +0100 Subject: [PATCH 15/19] fix docstring typo and attribute --- docs/changes/2928.feature.rst | 2 +- src/ctapipe/io/event_preprocessor.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index 4cf26495dc4..d9fe943f1bc 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -39,4 +39,4 @@ The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the follo This also introduces a helper function `~ctapipe.coordinates.altaz_to_nominal` to convert columns of alt/az coordinates to FOV coordinates in the `~ctapipe.coordinates.NominalFrame`, which works with the -`~ctapipe.io.FeatureGenerator`. +`~ctapipe.core.FeatureGenerator`. diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index a3e7deb2e63..24b5c5fa400 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -34,11 +34,11 @@ class EventPreprocessor(Component): `~ctapipe.core.FeatureGenerator` and set the ``features`` property of this class with the columns you to retain in the output table. - In the `~ctapipe.core.FeatureGenerator`` used internally, you have access to several - additional functions useful for DL2 processing: + In the `~ctapipe.core.FeatureGenerator` used internally, you have access to + several additional functions useful for DL2 processing: - - `~astropy.coordinates.angular_separation` - - `~ctapipe.coordinates.altaz_to_nominal` + - `~astropy.coordinates.angular_separation` + - `~ctapipe.coordinates.altaz_to_nominal` """ energy_reconstructor = traits.Unicode( @@ -177,7 +177,7 @@ def _get_predefined_quality_criteria(self) -> list[tuple]: raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") @default("features") - def default_features(self): + def _features(self): """Set the columns to output, for a given FeatureSet.""" if self.feature_set == PreprocessorFeatureSet.dl2_irf: return [ From eba8c25609a19d1f934d0ae9711ecf3faae18ce5 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Thu, 19 Feb 2026 15:45:15 +0100 Subject: [PATCH 16/19] fix wrong inputs for angular_separation --- src/ctapipe/io/event_preprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index 24b5c5fa400..3fab4e2a68d 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -145,11 +145,11 @@ def _get_predefined_features_to_generate(self) -> list[tuple]: ("true_fov_lat", "true_fov_coord[:,1]"), ( "true_fov_offset", - "angular_separation(reco_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", + "angular_separation(true_fov_lon, true_fov_lat, 0*u.deg, 0*u.deg)", ), ( "reco_fov_offset", - "angular_separation(true_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", + "angular_separation(reco_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", ), ( "multiplicity", From 1f7bbd8ac1b00e546ac3bfb9b044f98ae007ce3a Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Thu, 19 Feb 2026 16:09:34 +0100 Subject: [PATCH 17/19] use a FeatureSetRegistry for FeatureSets --- src/ctapipe/io/__init__.py | 3 +- src/ctapipe/io/event_preprocessor.py | 218 +++++++++--------- .../io/tests/test_event_preprocessor.py | 10 +- 3 files changed, 121 insertions(+), 110 deletions(-) diff --git a/src/ctapipe/io/__init__.py b/src/ctapipe/io/__init__.py index 2ffa708d600..c9f4b041ffc 100644 --- a/src/ctapipe/io/__init__.py +++ b/src/ctapipe/io/__init__.py @@ -8,7 +8,7 @@ from .astropy_helpers import read_table, write_table # noqa: I001 from .datalevels import DataLevel from .dl2_tables_preprocessing import DL2EventPreprocessor, DL2EventLoader -from .event_preprocessor import EventPreprocessor, PreprocessorFeatureSet +from .event_preprocessor import EventPreprocessor from .eventsource import EventSource from .eventseeker import EventSeeker from .tableio import TableReader, TableWriter @@ -48,5 +48,4 @@ "DL2EventLoader", "get_hdf5_monitoring_types", "EventPreprocessor", - "PreprocessorFeatureSet", ] diff --git a/src/ctapipe/io/event_preprocessor.py b/src/ctapipe/io/event_preprocessor.py index 3fab4e2a68d..d311035815d 100644 --- a/src/ctapipe/io/event_preprocessor.py +++ b/src/ctapipe/io/event_preprocessor.py @@ -1,9 +1,6 @@ """Module containing classes related to event loading and preprocessing""" -from enum import StrEnum, auto - from astropy.coordinates import angular_separation -from traitlets import default from ..coordinates import altaz_to_nominal from ..core import ( @@ -17,11 +14,113 @@ __all__ = ["EventPreprocessor"] -class PreprocessorFeatureSet(StrEnum): - """Pre-defined configurations for DL2EventPreprocessor for specific use cases.""" +from typing import Callable + + +class FeatureSetRegistry: + """Registry for custom feature set configurations.""" - custom = auto() #: use user-supplied configuration - dl2_irf = auto() #: support IRF preprocessing use case + _registry = {} + + @classmethod + def register(cls, name: str): + """Register a feature set configuration. + + Examples + -------- + >>> @FeatureSetRegistry.register("my_analysis") + ... def my_config(preprocessor): + ... return { + ... "features_to_generate": [("custom", "col_a / col_b")], + ... "quality_criteria": [("cut", "custom > 0.5")], + ... "output_features": ["event_id", "custom"] + ... } + """ + + def decorator(func: Callable): + cls._registry[name] = func + return func + + return decorator + + @classmethod + def get(cls, name: str): + """Get a registered configuration function.""" + return cls._registry.get(name) + + @classmethod + def list_available(cls): + """List all registered feature set names.""" + return list(cls._registry.keys()) + + +@FeatureSetRegistry.register("dl2_irf") +def _dl2_irf_config(preprocessor): + """Built-in configuration for DL2 IRF generation.""" + return { + "features_to_generate": [ + ("reco_energy", f"{preprocessor.energy_reconstructor}_energy"), + ("reco_alt", f"{preprocessor.geometry_reconstructor}_alt"), + ("reco_az", f"{preprocessor.geometry_reconstructor}_az"), + ("gh_score", f"{preprocessor.gammaness_reconstructor}_prediction"), + ("theta", "angular_separation(reco_az, reco_alt, true_az, true_alt)"), + ( + "reco_fov_coord", + "altaz_to_nominal(reco_az, reco_alt, subarray_pointing_lon, subarray_pointing_lat)", + ), + ( + "reco_fov_lon", + "reco_fov_coord[:,0]", + ), # note: GADF IRFs use the negative of this + ("reco_fov_lat", "reco_fov_coord[:,1]"), + ( + "true_fov_coord", + "altaz_to_nominal(true_az, true_alt, subarray_pointing_lon, subarray_pointing_lat)", + ), + ( + "true_fov_lon", + "true_fov_coord[:,0]", + ), # note: GADF IRFs use the negative of this + ("true_fov_lat", "true_fov_coord[:,1]"), + ( + "true_fov_offset", + "angular_separation(true_fov_lon, true_fov_lat, 0*u.deg, 0*u.deg)", + ), + ( + "reco_fov_offset", + "angular_separation(reco_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", + ), + ( + "multiplicity", + f"np.count_nonzero({preprocessor.gammaness_reconstructor}_telescopes,axis=1)", + ), + ], + "quality_criteria": [ + ("Valid geometry", f"{preprocessor.geometry_reconstructor}_is_valid"), + ("valid energy", f"{preprocessor.energy_reconstructor}_is_valid"), + ("valid gammaness", f"{preprocessor.gammaness_reconstructor}_is_valid"), + ("sufficient multiplicity", "multiplicity >= 4"), + ], + "output_features": [ + "event_id", + "obs_id", + "reco_energy", + "reco_alt", + "reco_az", + "gh_score", + "true_energy", + "true_alt", + "true_az", + "true_fov_offset", + "reco_fov_offset", + "theta", + "reco_fov_lat", + "true_fov_lat", + "reco_fov_lon", + "true_fov_lon", + "multiplicity", + ], + } class EventPreprocessor(Component): @@ -56,9 +155,9 @@ class with the columns you to retain in the output table. help="Prefix of the classifier `_prediction` column", ).tag(config=True) - feature_set = traits.UseEnum( - PreprocessorFeatureSet, - default_value=PreprocessorFeatureSet.dl2_irf, + feature_set = traits.CaselessStrEnum( + ["custom"] + FeatureSetRegistry.list_available(), + default_value="custom", help=( "Set up the FeatureGenerator.features, output features, and quality criteria " "based on standard use cases." @@ -79,16 +178,18 @@ class with the columns you to retain in the output table. def __init__(self, config=None, parent=None, **kwargs): super().__init__(config=config, parent=parent, **kwargs) - if self.feature_set == PreprocessorFeatureSet.custom: + if self.feature_set == "custom": self.feature_generator = FeatureGenerator(parent=self) self.quality_query = QualityQuery(parent=self) - else: + else: # use a pre-registered feature set + feature_set = FeatureSetRegistry.get(self.feature_set)(self) self.feature_generator = FeatureGenerator( - parent=self, features=self._get_predefined_features_to_generate() + parent=self, features=feature_set["features_to_generate"] ) self.quality_query = QualityQuery( - parent=self, quality_criteria=self._get_predefined_quality_criteria() + parent=self, quality_criteria=feature_set["quality_criteria"] ) + self.features = feature_set["output_features"] # sanity checks: if len(self.features) == 0: raise ToolConfigurationError( @@ -114,92 +215,3 @@ def __call__(self, table): # return only the columns specified in `self.features`, and rows in # `selected_mask` return generated[self.features][selected_mask] - - def _get_predefined_features_to_generate(self) -> list[tuple]: - """Return a default list of FeatureGenerator features.""" - if self.feature_set == PreprocessorFeatureSet.dl2_irf: - # Default features for DL2/Subarray events - return [ - ("reco_energy", f"{self.energy_reconstructor}_energy"), - ("reco_alt", f"{self.geometry_reconstructor}_alt"), - ("reco_az", f"{self.geometry_reconstructor}_az"), - ("gh_score", f"{self.gammaness_reconstructor}_prediction"), - ("theta", "angular_separation(reco_az, reco_alt, true_az, true_alt)"), - ( - "reco_fov_coord", - "altaz_to_nominal(reco_az, reco_alt, subarray_pointing_lon, subarray_pointing_lat)", - ), - ( - "reco_fov_lon", - "reco_fov_coord[:,0]", - ), # note: GADF IRFs use the negative of this - ("reco_fov_lat", "reco_fov_coord[:,1]"), - ( - "true_fov_coord", - "altaz_to_nominal(true_az, true_alt, subarray_pointing_lon, subarray_pointing_lat)", - ), - ( - "true_fov_lon", - "true_fov_coord[:,0]", - ), # note: GADF IRFs use the negative of this - ("true_fov_lat", "true_fov_coord[:,1]"), - ( - "true_fov_offset", - "angular_separation(true_fov_lon, true_fov_lat, 0*u.deg, 0*u.deg)", - ), - ( - "reco_fov_offset", - "angular_separation(reco_fov_lon, reco_fov_lat, 0*u.deg, 0*u.deg)", - ), - ( - "multiplicity", - f"np.count_nonzero({self.gammaness_reconstructor}_telescopes,axis=1)", - ), - ] - else: - raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") - - def _get_predefined_quality_criteria(self) -> list[tuple]: - """ - Set the quality criteria for a DL2FeatureSet. - - Here you can use any columns in the input table, or any that are - specified in the FeatureGenerator. - """ - if self.feature_set == PreprocessorFeatureSet.dl2_irf: - return [ - ("Valid geometry", f"{self.geometry_reconstructor}_is_valid"), - ("valid energy", f"{self.energy_reconstructor}_is_valid"), - ("valid gammaness", f"{self.gammaness_reconstructor}_is_valid"), - ("sufficient multiplicity", "multiplicity >= 4"), - ] - else: - raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") - - @default("features") - def _features(self): - """Set the columns to output, for a given FeatureSet.""" - if self.feature_set == PreprocessorFeatureSet.dl2_irf: - return [ - "event_id", - "obs_id", - "reco_energy", - "reco_alt", - "reco_az", - "gh_score", - "true_energy", - "true_alt", - "true_az", - "true_fov_offset", - "reco_fov_offset", - "theta", - "reco_fov_lat", - "true_fov_lat", - "reco_fov_lon", - "true_fov_lon", - "multiplicity", - ] - elif self.feature_set == PreprocessorFeatureSet.custom: - return [] - else: - raise NotImplementedError(f"unsupported feature_set: {self.feature_set}") diff --git a/src/ctapipe/io/tests/test_event_preprocessor.py b/src/ctapipe/io/tests/test_event_preprocessor.py index d3b9c631a26..378e0d4ac65 100644 --- a/src/ctapipe/io/tests/test_event_preprocessor.py +++ b/src/ctapipe/io/tests/test_event_preprocessor.py @@ -5,7 +5,7 @@ from astropy import units as u from astropy.table import QTable -from ctapipe.io import PreprocessorFeatureSet +from ctapipe.io.event_preprocessor import FeatureSetRegistry @pytest.fixture(scope="function") @@ -39,7 +39,7 @@ def minimal_dl2_table(): ) -@pytest.mark.parametrize("feature_set", list(PreprocessorFeatureSet)) +@pytest.mark.parametrize("feature_set", FeatureSetRegistry.list_available()) def test_event_preprocessing(feature_set, minimal_dl2_table): from traitlets.config import Config @@ -65,10 +65,10 @@ def test_event_preprocessing(feature_set, minimal_dl2_table): def test_no_output(): """Check error is raised if no columns are specified for output.""" from ctapipe.core import ToolConfigurationError - from ctapipe.io import EventPreprocessor, PreprocessorFeatureSet + from ctapipe.io import EventPreprocessor with pytest.raises(ToolConfigurationError): - EventPreprocessor(feature_set=PreprocessorFeatureSet.custom) + EventPreprocessor(feature_set="custom") def test_nondefault_reconstructors(minimal_dl2_table): @@ -94,7 +94,7 @@ def test_nondefault_reconstructors(minimal_dl2_table): table[f"{gammaness}_telescopes"] = table["RandomForestClassifier_telescopes"] preprocess = EventPreprocessor( - feature_set=PreprocessorFeatureSet.dl2_irf, + feature_set="dl2_irf", geometry_reconstructor=geom, energy_reconstructor=energy, gammaness_reconstructor=gammaness, From 243ca62bd80f0a042515347ae728c55ae62eb0d7 Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Thu, 19 Feb 2026 16:11:17 +0100 Subject: [PATCH 18/19] update changelog --- docs/changes/2928.feature.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index d9fe943f1bc..a3f2f41ac4d 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -7,16 +7,15 @@ transform an event table by applying the following steps: attribute of the `~ctapipe.io.EventPreprocessor`) This is useful for doing the final steps of DL2 processing, and will eventually -replace what is in `~ctapipe.io.DL2EventPreprocessor` and `~ctapipe.io.DL2EventLoader`, which will be -deprecated in a future release. +replace what is in `~ctapipe.io.DL2EventPreprocessor` and `~ctapipe.io.DL2EventLoader`, which will be deprecated in a future release. The `~ctapipe.io.EventPreprocessor` also includes the ability to pre-configure itself for specific use cases by setting the ``feature_set`` option. Currently -only two `~ctapipe.io.PreprocessorFeatureSet` are implemented: -``feature_set=dl2_irf``, which defines the transforms, event selection, and output -features for processing simulated DL2 events, and ``feature_set=custom``, which -has no pre-configuration and requires all parameters to be set by the user in a -config file. +only two are implemented: ``feature_set=dl2_irf``, which defines the transforms, +event selection, and output features for processing simulated DL2 events, and +``feature_set=custom``, which has no pre-configuration and requires all +parameters to be set by the user in a config file. More can be added by adding +to the registry. The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the following: @@ -27,7 +26,7 @@ The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the follo DL2FILE = "some_dl2_file.h5" loader = TableLoader(DL2FILE, dl2=True, simulated=True, observation_info=True) - preprocess = EventPreprocessor(feature_set="dl2_simulation") + preprocess = EventPreprocessor(feature_set="dl2_irf") events = vstack( [ preprocess(QTable(c.data)) From 9b32d86f1c6a3cbce5ad00eaca3fd987d4f2665e Mon Sep 17 00:00:00 2001 From: Karl Kosack Date: Thu, 19 Feb 2026 16:35:28 +0100 Subject: [PATCH 19/19] show better example --- docs/changes/2928.feature.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/changes/2928.feature.rst b/docs/changes/2928.feature.rst index a3f2f41ac4d..c3742044879 100644 --- a/docs/changes/2928.feature.rst +++ b/docs/changes/2928.feature.rst @@ -25,14 +25,14 @@ The functionality of `~ctapipe.io.DL2EventLoader` can be mimicked with the follo from astropy.table import vstack DL2FILE = "some_dl2_file.h5" - loader = TableLoader(DL2FILE, dl2=True, simulated=True, observation_info=True) - preprocess = EventPreprocessor(feature_set="dl2_irf") - events = vstack( - [ - preprocess(QTable(c.data)) - for c in loader.read_subarray_events_chunked(chunk_size=100_000) - ] - ) + with TableLoader(DL2FILE, dl2=True, simulated=True, observation_info=True) as loader: + preprocess = EventPreprocessor(feature_set="dl2_irf") + events = vstack( + [ + preprocess(QTable(c.data)) + for c in loader.read_subarray_events_chunked(chunk_size=100_000) + ] + ) This also introduces a helper function `~ctapipe.coordinates.altaz_to_nominal`