diff --git a/application/backend/app/execution/common/geti_config_converter.py b/application/backend/app/execution/common/geti_config_converter.py index 29f6d24777b..30b5ff6016a 100644 --- a/application/backend/app/execution/common/geti_config_converter.py +++ b/application/backend/app/execution/common/geti_config_converter.py @@ -156,8 +156,14 @@ class TransformsUpdater: "max_shear_degree": "shear", } + # Augmentations that cannot be combined with the tiling pipeline. Mosaic and + # MixUp stitch/blend multiple full images together, which is fundamentally + # incompatible with splitting an image into tiles, so they must never be added + # to a tiling recipe (see the tile recipe NOTE about the AugmentationScheduler). + TILING_INCOMPATIBLE_AUGMENTATIONS: ClassVar[set[str]] = {"mosaic", "mixup"} + @classmethod - def update(cls, augmentation_params: dict, config: dict) -> None: # noqa: C901 + def update(cls, augmentation_params: dict, config: dict) -> None: # noqa: C901, PLR0912 """Update augmentations in the config based on Geti model template. For each augmentation in augmentation_params: @@ -186,9 +192,12 @@ def update(cls, augmentation_params: dict, config: dict) -> None: # noqa: C901 is_ultralytics = config.get("backend") == "ultralytics" for aug_name, aug_value in augmentation_params.items(): + if tiling and aug_name in cls.TILING_INCOMPATIBLE_AUGMENTATIONS: + logger.info("Augmentation '{}' is incompatible with the Tiling pipeline and will be skipped", aug_name) + continue if aug_name not in cls.AUGMENTATION_REGISTRY: if tiling: - logger.info("Augmentation '%s' is not applicable in Tiling pipeline", aug_name) + logger.info("Augmentation '{}' is not applicable in Tiling pipeline", aug_name) continue msg = f"Unknown augmentation: '{aug_name}'. Available: {list(cls.AUGMENTATION_REGISTRY.keys())}" raise ValueError(msg) @@ -1047,16 +1056,22 @@ def _update_params(config: dict, param_dict: dict) -> None: deim_framework = augmentation_params.pop("deim_framework", None) training_parameters = param_dict.get("training", {}) - # Update tiling (always applied regardless of DEIM state) + # Update tiling first so downstream augmentation handling can react to it. TransformsUpdater.update_tiling(tiling, config) - - # When DEIM is enabled, the AugmentationSchedulerCallback owns the pipeline; - # user augmentation overrides must be ignored. - deim_enabled = deim_framework is True + tile_enabled = bool(config["data"].get("tile_config", {}).get("enable_tiler", False)) + + # The DEIM adaptive augmentation scheduling framework (mosaic/mixup based) + # is incompatible with tiling. The tile recipe intentionally omits the + # AugmentationSchedulerCallback and ships a static tiling-friendly pipeline, + # so DEIM must be treated as disabled whenever tiling is enabled, regardless + # of the requested value. Otherwise the converter would silently ignore the + # user's augmentations (deim=True) or add incompatible ones for a tiling run. + deim_enabled = deim_framework is True and not tile_enabled if not deim_enabled: TransformsUpdater.update(augmentation_params, config) - if deim_framework is False: - # User explicitly disabled DEIM -> remove the scheduler callback + if deim_framework is False or tile_enabled: + # DEIM disabled by the user, or forced off because tiling is on -> + # remove the scheduler callback and fall back to the static pipeline. GetiConfigConverter._disable_deim_framework(config) # Update training hyperparameters diff --git a/application/backend/app/execution/training/getitune_trainer.py b/application/backend/app/execution/training/getitune_trainer.py index 3836704ca39..adeace258d8 100644 --- a/application/backend/app/execution/training/getitune_trainer.py +++ b/application/backend/app/execution/training/getitune_trainer.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: from getitune import TaskType - from getitune.config.data import SubsetConfig + from getitune.config.data import SubsetConfig, TileConfig from getitune.data.dataset.base import VisionDataset from getitune.engine.engine import Engine @@ -90,6 +90,7 @@ class DatasetInfo: getitune_validation_subset_config: SubsetConfig getitune_testing_subset_config: SubsetConfig revision_id: UUID + tile_config: TileConfig @dataclass(frozen=True) @@ -243,7 +244,7 @@ def prepare_training_configuration( return training_config, getitune_training_config @step("Prepare Training Dataset", 10) - def prepare_training_dataset( + def prepare_training_dataset( # noqa: PLR0915 self, project_id: UUID, task: Task, @@ -258,7 +259,9 @@ def prepare_training_dataset( Otherwise, it creates a new dataset from the current items in the database with user-verified annotations. """ - from getitune.config.data import SamplerConfig, SubsetConfig + from getitune import TaskType + from getitune.config.data import SamplerConfig, SubsetConfig, TileConfig + from getitune.data.dataset.tile import TileDatasetFactory from getitune.data.entity.utils import detect_storage_dtype from getitune.data.factory import TransformLibFactory @@ -349,6 +352,30 @@ def build_subset_config(subset_name: str) -> SubsetConfig: transforms=test_subset_config.transforms, # pyrefly: ignore[missing-attribute,bad-argument-type] ) + # Build a TileConfig, and wrap each subset dataset with the tiling factory (if tiling is enabled). + tile_cfg_data = getitune_training_config["data"].get("tile_config", {}) + tile_config = TileConfig(**tile_cfg_data) + logger.info("Tiling is set to {}", tile_config.enable_tiler) + if tile_config.enable_tiler and getitune_task_type in (TaskType.DETECTION, TaskType.INSTANCE_SEGMENTATION): + logger.info("TileConfig for training: {}", tile_config) + logger.info("Wrapping subset datasets with TileDatasetFactory") + getitune_training_dataset = TileDatasetFactory.create( + dataset=getitune_training_dataset, tile_config=tile_config + ) + getitune_validation_dataset = TileDatasetFactory.create( + dataset=getitune_validation_dataset, tile_config=tile_config + ) + getitune_testing_dataset = TileDatasetFactory.create( + dataset=getitune_testing_dataset, tile_config=tile_config + ) + + logger.info( + "Training dataset size: {}, Validation dataset size: {}, Testing dataset size: {}", + len(getitune_training_dataset), + len(getitune_validation_dataset), + len(getitune_testing_dataset), + ) + logger.info("Augmentations applied {}", getitune_training_config["data"]) return DatasetInfo( getitune_training_dataset=getitune_training_dataset, getitune_validation_dataset=getitune_validation_dataset, @@ -357,6 +384,7 @@ def build_subset_config(subset_name: str) -> SubsetConfig: getitune_validation_subset_config=val_subset_config, getitune_testing_subset_config=test_subset_config, revision_id=dataset_revision_id, + tile_config=tile_config, ) @step("Prepare Model") @@ -416,6 +444,8 @@ def train_model( val_subset=dataset_info.getitune_validation_subset_config, test_subset=dataset_info.getitune_testing_subset_config, ) + # Ensure the datamodule (and downstream model) uses the resolved tiling configuration. + datamodule.tile_config = dataset_info.tile_config logger.info("Instantiating model for training (model_id={})", model_id) model_cfg = training_config["model"] diff --git a/application/backend/app/models/training_configuration/augmentation.py b/application/backend/app/models/training_configuration/augmentation.py index cf75e95d1d4..7cd55207749 100644 --- a/application/backend/app/models/training_configuration/augmentation.py +++ b/application/backend/app/models/training_configuration/augmentation.py @@ -564,7 +564,9 @@ class AugmentationParameters(BaseModel): "training time. When disabled, a simpler static augmentation pipeline " "is used instead, allowing direct control over individual augmentations. " "WARNING: when the DEIM framework is enabled, all other augmentation " - "parameters below are ignored and will have no effect on training." + "parameters below are ignored and will have no effect on training. " + "Note: the DEIM framework is not supported when Tiling algorithm is " + "enabled and will be automatically disabled in that case." ), ) random_zoom_out: RandomZoomOut | None = Field( @@ -588,7 +590,10 @@ class AugmentationParameters(BaseModel): mosaic: Mosaic | None = Field( default=None, title="Mosaic", - description="Combines 4 images into one mosaic for augmentation. Applied before resize.", + description=( + "Combines 4 images into one mosaic for augmentation. Applied before resize. " + "Note: this augmentation is not supported when Tiling algorithm is enabled." + ), json_schema_extra={"depends_on": {"deim_framework": [False, None]}}, ) random_resize_crop: RandomResizeCrop | None = Field( @@ -613,7 +618,10 @@ class AugmentationParameters(BaseModel): mixup: Mixup | None = Field( default=None, title="Mixup", - description="Blends two images and their labels for augmentation. Applied before resize.", + description=( + "Blends two images and their labels for augmentation. Applied before resize. " + "Note: this augmentation is not supported when Tiling algorithm is enabled." + ), json_schema_extra={"depends_on": {"deim_framework": [False, None]}}, ) random_horizontal_flip: RandomHorizontalFlip | None = Field( diff --git a/application/backend/app/supported_models/manifests/detection/dfine_l.yaml b/application/backend/app/supported_models/manifests/detection/dfine_l.yaml index 83d0715eee7..aa614eb99dc 100644 --- a/application/backend/app/supported_models/manifests/detection/dfine_l.yaml +++ b/application/backend/app/supported_models/manifests/detection/dfine_l.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 54.7 coco_map_50: 72.4 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0005 diff --git a/application/backend/app/supported_models/manifests/detection/dfine_m.yaml b/application/backend/app/supported_models/manifests/detection/dfine_m.yaml index bbc39562fa2..5e857725a45 100644 --- a/application/backend/app/supported_models/manifests/detection/dfine_m.yaml +++ b/application/backend/app/supported_models/manifests/detection/dfine_m.yaml @@ -13,9 +13,6 @@ stats: benchmark_metrics: coco_map_50_95: 52.7 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0004 diff --git a/application/backend/app/supported_models/manifests/detection/dfine_x.yaml b/application/backend/app/supported_models/manifests/detection/dfine_x.yaml index cef8266eb85..76bea6778f7 100644 --- a/application/backend/app/supported_models/manifests/detection/dfine_x.yaml +++ b/application/backend/app/supported_models/manifests/detection/dfine_x.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 56.5 coco_map_50: 74.0 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0005 diff --git a/application/backend/app/supported_models/manifests/detection/dinov3_detr_l.yaml b/application/backend/app/supported_models/manifests/detection/dinov3_detr_l.yaml index 6d0a3cdecc9..2e97c204806 100644 --- a/application/backend/app/supported_models/manifests/detection/dinov3_detr_l.yaml +++ b/application/backend/app/supported_models/manifests/detection/dinov3_detr_l.yaml @@ -13,9 +13,6 @@ stats: benchmark_metrics: coco_map_50_95: 56.0 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0005 diff --git a/application/backend/app/supported_models/manifests/detection/dinov3_detr_m.yaml b/application/backend/app/supported_models/manifests/detection/dinov3_detr_m.yaml index bedd92b1c34..394a4f3f5c1 100644 --- a/application/backend/app/supported_models/manifests/detection/dinov3_detr_m.yaml +++ b/application/backend/app/supported_models/manifests/detection/dinov3_detr_m.yaml @@ -13,9 +13,6 @@ stats: benchmark_metrics: coco_map_50_95: 53.0 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0004 diff --git a/application/backend/app/supported_models/manifests/detection/dinov3_detr_s.yaml b/application/backend/app/supported_models/manifests/detection/dinov3_detr_s.yaml index dbc180502a5..5e6d1eb808e 100644 --- a/application/backend/app/supported_models/manifests/detection/dinov3_detr_s.yaml +++ b/application/backend/app/supported_models/manifests/detection/dinov3_detr_s.yaml @@ -13,9 +13,6 @@ stats: benchmark_metrics: coco_map_50_95: 50.9 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0004 diff --git a/application/backend/app/supported_models/manifests/detection/rfdetr_l.yaml b/application/backend/app/supported_models/manifests/detection/rfdetr_l.yaml index 4a592039002..c3c98dd21a0 100644 --- a/application/backend/app/supported_models/manifests/detection/rfdetr_l.yaml +++ b/application/backend/app/supported_models/manifests/detection/rfdetr_l.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 56.5 coco_map_50: 75.1 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/detection/rfdetr_m.yaml b/application/backend/app/supported_models/manifests/detection/rfdetr_m.yaml index 8944cee9758..dce9e23ad4d 100644 --- a/application/backend/app/supported_models/manifests/detection/rfdetr_m.yaml +++ b/application/backend/app/supported_models/manifests/detection/rfdetr_m.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 54.7 coco_map_50: 73.6 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.00021 diff --git a/application/backend/app/supported_models/manifests/detection/rfdetr_n.yaml b/application/backend/app/supported_models/manifests/detection/rfdetr_n.yaml index ce2a4b0bd54..0198a37e531 100644 --- a/application/backend/app/supported_models/manifests/detection/rfdetr_n.yaml +++ b/application/backend/app/supported_models/manifests/detection/rfdetr_n.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 48.4 coco_map_50: 67.6 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.00021 diff --git a/application/backend/app/supported_models/manifests/detection/rfdetr_s.yaml b/application/backend/app/supported_models/manifests/detection/rfdetr_s.yaml index 5d715e29c24..678781984d4 100644 --- a/application/backend/app/supported_models/manifests/detection/rfdetr_s.yaml +++ b/application/backend/app/supported_models/manifests/detection/rfdetr_s.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 53.0 coco_map_50: 72.1 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.00021 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_2xl.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_2xl.yaml index 17f333daa0e..bfbac7443e7 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_2xl.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_2xl.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 49.9 coco_map_50: 73.1 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_l.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_l.yaml index 149236d8f3f..9d7c1c272e2 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_l.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_l.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 47.1 coco_map_50: 70.5 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_m.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_m.yaml index cd934e40682..2c73d8cf82a 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_m.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_m.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 45.3 coco_map_50: 68.4 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_n.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_n.yaml index b7b27b029c1..cf05f2dd200 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_n.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_n.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 40.3 coco_map_50: 63.0 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_s.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_s.yaml index feb1ae98966..7702ea3e5ac 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_s.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_s.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 43.1 coco_map_50: 66.2 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_xl.yaml b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_xl.yaml index f3c53491c41..0c7052a7bad 100644 --- a/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_xl.yaml +++ b/application/backend/app/supported_models/manifests/instance_segmentation/rfdetr_xl.yaml @@ -14,9 +14,6 @@ stats: coco_map_50_95: 48.8 coco_map_50: 72.2 -capabilities: - tiling: false - hyperparameters: training: learning_rate: 0.0001 diff --git a/application/backend/tests/integration/services/test_training_configuration_service.py b/application/backend/tests/integration/services/test_training_configuration_service.py index 3d8c3095d19..35c520cef28 100644 --- a/application/backend/tests/integration/services/test_training_configuration_service.py +++ b/application/backend/tests/integration/services/test_training_configuration_service.py @@ -148,9 +148,9 @@ def test_get_default_by_model_architecture_strips_tiling_when_unsupported( """Tiling parameters should be removed from the default config when the architecture doesn't support tiling.""" TrainingConfigurationService.get_default_by_model_architecture.cache_clear() - # D-FINE-M does not support tiling (capabilities.tiling: false in its manifest) + # YOLO11-N does not support tiling (capabilities.tiling: false in its manifest) result = TrainingConfigurationService.get_default_by_model_architecture( - model_architecture_id="object-detection-dfine-m", + model_architecture_id="object-detection-yolo11-n", ) assert result.algo_level_parameters.dataset_preparation.augmentation.tiling is None @@ -183,7 +183,7 @@ def test_get_by_model_architecture_strips_tiling_when_unsupported( algo_level_config = TrainingConfigurationDB( id=str(uuid4()), project_id=fxt_project.id, - model_architecture_id="object-detection-dfine-m", + model_architecture_id="object-detection-yolo11-n", configuration_data=algo_level_data, ) db_session.add(algo_level_config) @@ -191,7 +191,7 @@ def test_get_by_model_architecture_strips_tiling_when_unsupported( result = fxt_training_configuration_service.get_by_model_architecture( project_id=UUID(fxt_project.id), - model_architecture_id="object-detection-dfine-m", + model_architecture_id="object-detection-yolo11-n", ) assert result.algo_level_parameters.dataset_preparation.augmentation.tiling is None diff --git a/application/backend/tests/unit/api/schemas/test_training_configuration_view.py b/application/backend/tests/unit/api/schemas/test_training_configuration_view.py index 25ea0b39553..8056a8688a8 100644 --- a/application/backend/tests/unit/api/schemas/test_training_configuration_view.py +++ b/application/backend/tests/unit/api/schemas/test_training_configuration_view.py @@ -702,7 +702,8 @@ def fxt_training_configuration_view_json() -> dict: "key": "mosaic", "name": "Mosaic", "description": ( - "Combines 4 images into one mosaic for augmentation. Applied before resize." + "Combines 4 images into one mosaic for augmentation. Applied before resize. " + "Note: this augmentation is not supported when Tiling algorithm is enabled." ), "depends_on": {"deim_framework": [False, None]}, "parameters": [ @@ -882,7 +883,8 @@ def fxt_training_configuration_view_json() -> dict: "key": "mixup", "name": "Mixup", "description": ( - "Blends two images and their labels for augmentation. Applied before resize." + "Blends two images and their labels for augmentation. Applied before resize. " + "Note: this augmentation is not supported when Tiling algorithm is enabled." ), "depends_on": {"deim_framework": [False, None]}, "parameters": [ diff --git a/application/backend/tests/unit/execution/common/test_geti_config_converter.py b/application/backend/tests/unit/execution/common/test_geti_config_converter.py index 45e76df40ca..ecd10b6aa78 100644 --- a/application/backend/tests/unit/execution/common/test_geti_config_converter.py +++ b/application/backend/tests/unit/execution/common/test_geti_config_converter.py @@ -22,6 +22,14 @@ "class_path": "lightning.pytorch.callbacks.ModelCheckpoint", "init_args": {"dirpath": "", "monitor": "val/map_50"}, } +AUG_SCHEDULER_CLASS_PATH = "getitune.backend.lightning.callbacks.aug_scheduler.AugmentationSchedulerCallback" + +AUG_SCHEDULER_CALLBACK = { + "class_path": AUG_SCHEDULER_CLASS_PATH, + "init_args": { + "data_aug_switch": {"class_path": "getitune.backend.lightning.callbacks.aug_scheduler.DataAugSwitch"} + }, +} def _make_getitune_config(**overrides: Any) -> dict: @@ -250,6 +258,69 @@ def test_convert_removes_early_stopping_when_disabled(self) -> None: idx = GetiConfigConverter.get_callback_idx(result["callbacks"], EARLY_STOPPING_CLASS_PATH) assert idx == -1 + def test_convert_excludes_deim_framework_when_tiling_enabled(self) -> None: + """DEIM is incompatible with tiling: enabling tiling must exclude the DEIM scheduler callback.""" + getitune_cfg = _make_getitune_config( + callbacks=[ + copy.deepcopy(EARLY_STOPPING_CALLBACK), + copy.deepcopy(CHECKPOINT_CALLBACK), + copy.deepcopy(AUG_SCHEDULER_CALLBACK), + ] + ) + geti_cfg = _make_geti_config( + model_manifest_id="object-detection-dfine-m", + hyper_parameters={ + "dataset_preparation": { + "augmentation": { + "deim_framework": True, + "tiling": { + "enable": True, + "enable_adaptive_tiling": True, + "tile_size": 512, + "tile_overlap": 0.3, + }, + } + } + }, + ) + + with patch("getitune.tools.auto_configurator.AutoConfigurator") as MockAutoConfigurator: + MockAutoConfigurator.return_value.config = getitune_cfg + result = GetiConfigConverter.convert(geti_cfg) + + # Tiling is enabled ... + assert result["data"]["tile_config"]["enable_tiler"] is True + # ... and the DEIM augmentation scheduler callback has been excluded. + assert GetiConfigConverter.get_callback_idx(result["callbacks"], AUG_SCHEDULER_CLASS_PATH) == -1 + + def test_convert_keeps_deim_framework_when_tiling_disabled(self) -> None: + """When tiling is disabled and DEIM is enabled, the DEIM scheduler callback is retained.""" + getitune_cfg = _make_getitune_config( + callbacks=[ + copy.deepcopy(EARLY_STOPPING_CALLBACK), + copy.deepcopy(CHECKPOINT_CALLBACK), + copy.deepcopy(AUG_SCHEDULER_CALLBACK), + ] + ) + geti_cfg = _make_geti_config( + model_manifest_id="object-detection-dfine-m", + hyper_parameters={ + "dataset_preparation": { + "augmentation": { + "deim_framework": True, + "tiling": {"enable": False}, + } + } + }, + ) + + with patch("getitune.tools.auto_configurator.AutoConfigurator") as MockAutoConfigurator: + MockAutoConfigurator.return_value.config = getitune_cfg + result = GetiConfigConverter.convert(geti_cfg) + + assert result["data"]["tile_config"]["enable_tiler"] is False + assert GetiConfigConverter.get_callback_idx(result["callbacks"], AUG_SCHEDULER_CLASS_PATH) >= 0 + def test_convert_applies_input_size(self) -> None: getitune_cfg = _make_getitune_config() geti_cfg = _make_geti_config(hyper_parameters={"training": {"input_size_height": 640, "input_size_width": 640}}) @@ -590,6 +661,48 @@ def test_tiling_update_disabled(self) -> None: tc = config["data"]["tile_config"] assert tc["enable_tiler"] is False + def test_mosaic_skipped_when_tiling_enabled(self) -> None: + """Mosaic must never be added to a tiling pipeline (it is incompatible).""" + config = _make_getitune_config() + config["data"]["tile_config"]["enable_tiler"] = True + TransformsUpdater.update( + {"mosaic": {"enable": True, "probability": 1.0}}, + config, + ) + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert not any("CachedMosaic" in a["class_path"] for a in cpu_augs) + + def test_mixup_skipped_when_tiling_enabled(self) -> None: + """MixUp must never be added to a tiling pipeline (it is incompatible).""" + config = _make_getitune_config() + config["data"]["tile_config"]["enable_tiler"] = True + TransformsUpdater.update( + {"mixup": {"enable": True, "probability": 0.5, "alpha": 1.5}}, + config, + ) + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert not any("CachedMixUp" in a["class_path"] for a in cpu_augs) + + def test_mosaic_added_when_tiling_disabled(self) -> None: + """Sanity check: mosaic is still added when tiling is off.""" + config = _make_getitune_config() + TransformsUpdater.update( + {"mosaic": {"enable": True, "probability": 1.0}}, + config, + ) + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert any("CachedMosaic" in a["class_path"] for a in cpu_augs) + + def test_mixup_added_when_tiling_disabled(self) -> None: + """Sanity check: mixup is still added when tiling is off.""" + config = _make_getitune_config() + TransformsUpdater.update( + {"mixup": {"enable": True, "probability": 0.5, "alpha": 1.5}}, + config, + ) + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert any("CachedMixUp" in a["class_path"] for a in cpu_augs) + def test_gaussian_noise_sigma_renamed_to_std(self) -> None: config = _make_getitune_config() TransformsUpdater.update( @@ -629,6 +742,91 @@ def test_not_found(self) -> None: assert GetiConfigConverter.get_callback_idx(callbacks, "missing.Class") == -1 +AUG_SCHEDULER_CLASS_PATH = "getitune.backend.lightning.callbacks.aug_scheduler.AugmentationSchedulerCallback" + + +def _make_deim_getitune_config() -> dict: + """A getitune config that includes the DEIM AugmentationSchedulerCallback.""" + config = _make_getitune_config() + config["callbacks"].append({"class_path": AUG_SCHEDULER_CLASS_PATH, "init_args": {}}) + return config + + +class TestUpdateParamsDeimTiling: + """Tests for the DEIM framework / tiling interaction in _update_params. + + Tiling is incompatible with the DEIM adaptive augmentation scheduling + framework (mosaic/mixup based). When tiling is enabled the converter must: + * force DEIM off (remove the AugmentationSchedulerCallback), + * apply the tiling-compatible user augmentations, + * never add tiling-incompatible augmentations (mosaic/mixup). + """ + + def _has_scheduler(self, config: dict) -> bool: + return GetiConfigConverter.get_callback_idx(config["callbacks"], AUG_SCHEDULER_CLASS_PATH) > -1 + + def test_deim_enabled_no_tiling_keeps_scheduler_and_ignores_augs(self) -> None: + config = _make_deim_getitune_config() + param_dict = { + "dataset_preparation": { + "augmentation": { + "deim_framework": True, + "tiling": {"enable": False, "enable_adaptive_tiling": False, "tile_size": 256, "tile_overlap": 0.5}, + "random_vertical_flip": {"enable": True, "probability": 0.3}, + } + } + } + GetiConfigConverter._update_params(config, param_dict) + + # Scheduler kept (DEIM owns the pipeline) and user aug override ignored. + assert self._has_scheduler(config) + gpu_augs = config["data"]["train_subset"]["augmentations_gpu"] + assert not any("VerticalFlip" in a["class_path"] for a in gpu_augs) + + def test_deim_enabled_with_tiling_disables_scheduler(self) -> None: + config = _make_deim_getitune_config() + param_dict = { + "dataset_preparation": { + "augmentation": { + "deim_framework": True, + "tiling": {"enable": True, "enable_adaptive_tiling": True, "tile_size": 256, "tile_overlap": 0.5}, + "mosaic": {"enable": True, "probability": 1.0}, + "random_vertical_flip": {"enable": True, "probability": 0.3}, + } + } + } + GetiConfigConverter._update_params(config, param_dict) + + # Tiling forces DEIM off -> scheduler removed. + assert not self._has_scheduler(config) + # Tiling enabled in the data config. + assert config["data"]["tile_config"]["enable_tiler"] is True + # Mosaic (incompatible) must not be added. + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert not any("CachedMosaic" in a["class_path"] for a in cpu_augs) + # Compatible augmentations are applied. + gpu_augs = config["data"]["train_subset"]["augmentations_gpu"] + assert any("VerticalFlip" in a["class_path"] for a in gpu_augs) + + def test_deim_disabled_with_tiling_disables_scheduler_and_skips_mosaic(self) -> None: + config = _make_deim_getitune_config() + param_dict = { + "dataset_preparation": { + "augmentation": { + "deim_framework": False, + "tiling": {"enable": True, "enable_adaptive_tiling": True, "tile_size": 256, "tile_overlap": 0.5}, + "mixup": {"enable": True, "probability": 0.5, "alpha": 1.5}, + } + } + } + GetiConfigConverter._update_params(config, param_dict) + + assert not self._has_scheduler(config) + assert config["data"]["tile_config"]["enable_tiler"] is True + cpu_augs = config["data"]["train_subset"]["augmentations_cpu"] + assert not any("CachedMixUp" in a["class_path"] for a in cpu_augs) + + class TestFullConfigRoundTrip: """Test with a config dict resembling a real Geti model_dump output.""" diff --git a/application/backend/tests/unit/execution/training/test_getitune_trainer.py b/application/backend/tests/unit/execution/training/test_getitune_trainer.py index e670f810985..adf5f7e52e8 100644 --- a/application/backend/tests/unit/execution/training/test_getitune_trainer.py +++ b/application/backend/tests/unit/execution/training/test_getitune_trainer.py @@ -542,9 +542,13 @@ def test_prepare_training_dataset_success( mock_dataset_class = Mock() mock_dataset_class.__name__ = "DetectionDataset" - mock_getitune_training_dataset = Mock() - mock_getitune_validation_dataset = Mock() - mock_getitune_testing_dataset = Mock() + # Datasets need a working __len__ because prepare_training_dataset logs their sizes. + mock_getitune_training_dataset = MagicMock() + mock_getitune_training_dataset.__len__.return_value = 10 + mock_getitune_validation_dataset = MagicMock() + mock_getitune_validation_dataset.__len__.return_value = 5 + mock_getitune_testing_dataset = MagicMock() + mock_getitune_testing_dataset.__len__.return_value = 3 mock_dataset_class.side_effect = [ mock_getitune_training_dataset, diff --git a/library/benchmark_manifest.yaml b/library/benchmark_manifest.yaml index 75fd961fb34..b508537fa5d 100644 --- a/library/benchmark_manifest.yaml +++ b/library/benchmark_manifest.yaml @@ -12,133 +12,118 @@ defaults: rotation: extended_groups: 2 experiments: - classification/multi_class_cls: - models: - - name: dino_v2 - priority: core - recipe: classification/multi_class_cls/dino_v2.yaml - - name: vit_tiny - priority: core - recipe: classification/multi_class_cls/vit_tiny.yaml - - name: mobilenet_v3_large - priority: core - recipe: classification/multi_class_cls/mobilenet_v3_large.yaml - - name: efficientnet_b3 - priority: extended - recipe: classification/multi_class_cls/efficientnet_b3.yaml - - name: efficientnet_v2 - priority: extended - recipe: classification/multi_class_cls/efficientnet_v2.yaml - - name: efficientnet_b0 - priority: extended - recipe: classification/multi_class_cls/efficientnet_b0.yaml - datasets: - - flowers102 - criteria: - accuracy_metric: accuracy - thresholds: - "training:val/{metric}": { compare: ">=", margin: 0.10 } - "torch:test/{metric}": { compare: ">=", margin: 0.10 } - "export:test/{metric}": { compare: ">=", margin: 0.10 } - "training:train/iter_time": { compare: "<=", margin: 0.10 } - "training:gpu_mem": { compare: "<=", margin: 0.10 } - "torch:test/latency": { compare: "<=", margin: 0.15 } - classification/multi_label_cls: - models: - - name: dino_v2_ml - priority: core - recipe: classification/multi_label_cls/dino_v2.yaml - - name: mobilenet_v3_large_ml - priority: core - recipe: classification/multi_label_cls/mobilenet_v3_large.yaml - - name: vit_tiny_ml - priority: core - recipe: classification/multi_label_cls/vit_tiny.yaml - - name: efficientnet_b0_ml - priority: extended - recipe: classification/multi_label_cls/efficientnet_b0.yaml - - name: efficientnet_b3_ml - priority: extended - recipe: classification/multi_label_cls/efficientnet_b3.yaml - - name: efficientnet_v2_ml - priority: extended - recipe: classification/multi_label_cls/efficientnet_v2.yaml - datasets: - - aid_multilabel - criteria: - accuracy_metric: accuracy - thresholds: - "training:val/{metric}": { compare: ">=", margin: 0.10 } - "torch:test/{metric}": { compare: ">=", margin: 0.10 } - "export:test/{metric}": { compare: ">=", margin: 0.10 } - "training:train/iter_time": { compare: "<=", margin: 0.10 } - "training:gpu_mem": { compare: "<=", margin: 0.10 } - "torch:test/latency": { compare: "<=", margin: 0.15 } detection: models: - - name: yolox_s - priority: core - recipe: detection/yolox_s.yaml - - name: deim_dfine_l - priority: core - recipe: detection/deim_dfine_l.yaml - - name: deim_dfine_m - priority: core - recipe: detection/deim_dfine_m.yaml + # --- tiling disabled (base recipes) --- - name: atss_mobilenetv2 - priority: extended + priority: core recipe: detection/atss_mobilenetv2.yaml - - name: yolox_s_tile - priority: extended - recipe: detection/yolox_tiny_tile.yaml - - name: dfine_x - priority: extended - recipe: detection/dfine_x.yaml - name: ssd_mobilenetv2 - priority: extended + priority: core recipe: detection/ssd_mobilenetv2.yaml - name: yolox_tiny - priority: extended + priority: core recipe: detection/yolox_tiny.yaml + - name: yolox_s + priority: core + recipe: detection/yolox_s.yaml - name: yolox_l - priority: extended + priority: core recipe: detection/yolox_l.yaml - name: yolox_x - priority: extended + priority: core recipe: detection/yolox_x.yaml - name: rtdetr_50 - priority: extended + priority: core recipe: detection/rtdetr_50.yaml + - name: deim_dfine_m + priority: core + recipe: detection/deim_dfine_m.yaml + - name: deim_dfine_l + priority: core + recipe: detection/deim_dfine_l.yaml + - name: deim_dfine_x + priority: core + recipe: detection/deim_dfine_x.yaml + - name: deimv2_s + priority: core + recipe: detection/deimv2_s.yaml + - name: deimv2_m + priority: core + recipe: detection/deimv2_m.yaml + - name: deimv2_l + priority: core + recipe: detection/deimv2_l.yaml + - name: rfdetr_nano + priority: core + recipe: detection/rfdetr_nano.yaml - name: rfdetr_small - priority: extended + priority: core recipe: detection/rfdetr_small.yaml - name: rfdetr_medium - priority: extended + priority: core recipe: detection/rfdetr_medium.yaml - name: rfdetr_large - priority: extended + priority: core recipe: detection/rfdetr_large.yaml - - name: deim_dfine_x - priority: extended - recipe: detection/deim_dfine_x.yaml - - name: deimv2_l - priority: extended - recipe: detection/deimv2_l.yaml - - name: deimv2_m - priority: extended - recipe: detection/deimv2_m.yaml - - name: deimv2_s - priority: extended - recipe: detection/deimv2_s.yaml - - name: yolo26_n - priority: extended - recipe: detection/yolo26_n.yaml - - name: yolo26_s - priority: extended - recipe: detection/yolo26_s.yaml - - name: yolo26_m - priority: extended - recipe: detection/yolo26_m.yaml + - name: dfine_x + priority: core + recipe: detection/dfine_x.yaml + # --- tiling enabled (_tile recipes) --- + - name: atss_mobilenetv2_tile + priority: core + recipe: detection/atss_mobilenetv2_tile.yaml + - name: ssd_mobilenetv2_tile + priority: core + recipe: detection/ssd_mobilenetv2_tile.yaml + - name: yolox_tiny_tile + priority: core + recipe: detection/yolox_tiny_tile.yaml + - name: yolox_s_tile + priority: core + recipe: detection/yolox_s_tile.yaml + - name: yolox_l_tile + priority: core + recipe: detection/yolox_l_tile.yaml + - name: yolox_x_tile + priority: core + recipe: detection/yolox_x_tile.yaml + - name: rtdetr_50_tile + priority: core + recipe: detection/rtdetr_50_tile.yaml + - name: deim_dfine_m_tile + priority: core + recipe: detection/deim_dfine_m_tile.yaml + - name: deim_dfine_l_tile + priority: core + recipe: detection/deim_dfine_l_tile.yaml + - name: deim_dfine_x_tile + priority: core + recipe: detection/deim_dfine_x_tile.yaml + - name: deimv2_s_tile + priority: core + recipe: detection/deimv2_s_tile.yaml + - name: deimv2_m_tile + priority: core + recipe: detection/deimv2_m_tile.yaml + - name: deimv2_l_tile + priority: core + recipe: detection/deimv2_l_tile.yaml + - name: rfdetr_nano_tile + priority: core + recipe: detection/rfdetr_nano_tile.yaml + - name: rfdetr_small_tile + priority: core + recipe: detection/rfdetr_small_tile.yaml + - name: rfdetr_medium_tile + priority: core + recipe: detection/rfdetr_medium_tile.yaml + - name: rfdetr_large_tile + priority: core + recipe: detection/rfdetr_large_tile.yaml + - name: dfine_x_tile + priority: core + recipe: detection/dfine_x_tile.yaml datasets: - bccd criteria: @@ -187,39 +172,68 @@ experiments: "torch:test/latency": { compare: "<=", margin: 0.15 } instance_segmentation: models: + # --- tiling disabled (base recipes) --- + - name: maskrcnn_efficientnetb2b + priority: core + recipe: instance_segmentation/maskrcnn_efficientnetb2b.yaml + - name: maskrcnn_r50 + priority: core + recipe: instance_segmentation/maskrcnn_r50.yaml + - name: maskrcnn_swint + priority: core + recipe: instance_segmentation/maskrcnn_swint.yaml + - name: rtmdet_inst_tiny + priority: core + recipe: instance_segmentation/rtmdet_inst_tiny.yaml + - name: rfdetr_seg_nano + priority: core + recipe: instance_segmentation/rfdetr_seg_nano.yaml - name: rfdetr_seg_small priority: core recipe: instance_segmentation/rfdetr_seg_small.yaml - name: rfdetr_seg_medium priority: core recipe: instance_segmentation/rfdetr_seg_medium.yaml + - name: rfdetr_seg_large + priority: core + recipe: instance_segmentation/rfdetr_seg_large.yaml - name: rfdetr_seg_xlarge priority: core recipe: instance_segmentation/rfdetr_seg_xlarge.yaml - - name: maskrcnn_r50 - priority: extended - recipe: instance_segmentation/maskrcnn_r50.yaml - - name: rtmdet_inst_tiny - priority: extended - recipe: instance_segmentation/rtmdet_inst_tiny.yaml - - name: maskrcnn_efficientnetb2b - priority: extended - recipe: instance_segmentation/maskrcnn_efficientnetb2b.yaml - - name: maskrcnn_swint - priority: extended - recipe: instance_segmentation/maskrcnn_swint.yaml - - name: rfdetr_seg_large - priority: extended - recipe: instance_segmentation/rfdetr_seg_large.yaml - - name: yolo26_n_seg - priority: extended - recipe: instance_segmentation/yolo26_n_seg.yaml - - name: yolo26_s_seg - priority: extended - recipe: instance_segmentation/yolo26_s_seg.yaml - - name: yolo26_m_seg - priority: extended - recipe: instance_segmentation/yolo26_m_seg.yaml + - name: rfdetr_seg_2xlarge + priority: core + recipe: instance_segmentation/rfdetr_seg_2xlarge.yaml + # --- tiling enabled (_tile recipes) --- + - name: maskrcnn_efficientnetb2b_tile + priority: core + recipe: instance_segmentation/maskrcnn_efficientnetb2b_tile.yaml + - name: maskrcnn_r50_tile + priority: core + recipe: instance_segmentation/maskrcnn_r50_tile.yaml + - name: maskrcnn_swint_tile + priority: core + recipe: instance_segmentation/maskrcnn_swint_tile.yaml + - name: rtmdet_inst_tiny_tile + priority: core + recipe: instance_segmentation/rtmdet_inst_tiny_tile.yaml + - name: rfdetr_seg_nano_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_nano_tile.yaml + - name: rfdetr_seg_small_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_small_tile.yaml + - name: rfdetr_seg_medium_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_medium_tile.yaml + - name: rfdetr_seg_large_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_large_tile.yaml + - name: rfdetr_seg_xlarge_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_xlarge_tile.yaml + - name: rfdetr_seg_2xlarge_tile + priority: core + recipe: instance_segmentation/rfdetr_seg_2xlarge_tile.yaml datasets: - wgisd criteria: diff --git a/library/src/getitune/backend/lightning/callbacks/gpu_augmentation.py b/library/src/getitune/backend/lightning/callbacks/gpu_augmentation.py index 9983a9280f2..afe8c1288d1 100644 --- a/library/src/getitune/backend/lightning/callbacks/gpu_augmentation.py +++ b/library/src/getitune/backend/lightning/callbacks/gpu_augmentation.py @@ -15,6 +15,7 @@ from getitune.data.augmentation import GPUAugmentationPipeline from getitune.data.entity.sample import SampleBatch +from getitune.data.entity.tile import TileBatchData from getitune.types.task import TaskType if TYPE_CHECKING: @@ -129,7 +130,7 @@ def _update_model_normalization(self, pl_module: LightningModule) -> None: def _apply_pipeline( self, pipeline: GPUAugmentationPipeline, - batch: SampleBatch, + batch: SampleBatch | TileBatchData, ) -> None: """Apply GPU augmentation pipeline to batch (in-place). @@ -138,8 +139,17 @@ def _apply_pipeline( Args: pipeline: GPUAugmentationPipeline to apply. - batch: SampleBatch to transform. + batch: SampleBatch (or TileBatchData when tiling is enabled) to transform. """ + # Tile batches (produced during validation/testing when tiling is enabled) + # store their images per-tile in ``batch_tiles`` and keep annotations at the + # original-image level, so they do not expose ``batch.images``. Only image + # transforms such as normalization are meaningful for the individual tiles; + # their predictions are merged back to the original image afterwards. + if isinstance(batch, TileBatchData): + self._apply_pipeline_to_tiles(pipeline, batch) + return + # Move pipeline to same device as batch device = batch.images.device if hasattr(batch.images, "device") else None if device is not None: @@ -212,6 +222,35 @@ def _apply_pipeline( batch.keypoints = typing.cast("list[torch.Tensor] | None", restored_keypoints) + def _apply_pipeline_to_tiles( + self, + pipeline: GPUAugmentationPipeline, + batch: TileBatchData, + ) -> None: + """Apply image-only GPU augmentations (e.g. normalization) to each tile in-place. + + During validation/testing with tiling, the batch is a ``TileBatchData`` whose + images are stored per-tile in ``batch_tiles`` (annotations are kept at the + original-image level and used only for tile-merging afterwards). The standard + ``SampleBatch`` path does not apply here since there is no ``batch.images``. + + Each tile image is passed through the pipeline on its own (as a single-image + batch). Only image transforms such as normalization are relevant for tiles; + geometric annotation handling is intentionally skipped because tiles carry no + per-tile annotations at this stage. + + Args: + pipeline: GPUAugmentationPipeline to apply. + batch: TileBatchData whose tiles are transformed in-place. + """ + for tiles in batch.batch_tiles: + for tile_idx, tile in enumerate(tiles): + device = tile.device if hasattr(tile, "device") else None + tile_pipeline = pipeline.to(device) if device is not None else pipeline + augmented = tile_pipeline.apply_image_only(tile.unsqueeze(0)).squeeze(0) + # Re-wrap as an Image tv_tensor so downstream tile handling keeps its type. + tiles[tile_idx] = tv_tensors.Image(augmented) # type: ignore[no-matching-overload] + def on_train_batch_start( self, trainer: Trainer, @@ -229,7 +268,7 @@ def on_validation_batch_start( self, trainer: Trainer, pl_module: LightningModule, - batch: SampleBatch, + batch: SampleBatch | TileBatchData, batch_idx: int, dataloader_idx: int = 0, ) -> None: @@ -243,7 +282,7 @@ def on_test_batch_start( self, trainer: Trainer, pl_module: LightningModule, - batch: SampleBatch, + batch: SampleBatch | TileBatchData, batch_idx: int, dataloader_idx: int = 0, ) -> None: diff --git a/library/src/getitune/backend/lightning/models/common/rfdetr_mixin.py b/library/src/getitune/backend/lightning/models/common/rfdetr_mixin.py index 3c56ee6f606..464ff427428 100644 --- a/library/src/getitune/backend/lightning/models/common/rfdetr_mixin.py +++ b/library/src/getitune/backend/lightning/models/common/rfdetr_mixin.py @@ -9,6 +9,7 @@ from __future__ import annotations +import logging import os from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, cast @@ -37,6 +38,9 @@ from rfdetr.models.lwdetr import LWDETR +logger = logging.getLogger(__name__) + + def _get_param_dict(args: SimpleNamespace, model_without_ddp: torch.nn.Module) -> list[dict[str, Any]]: if not isinstance(model_without_ddp.backbone, Joiner): msg = f"Expected backbone to be Joiner, got {type(model_without_ddp.backbone).__name__}" @@ -179,6 +183,34 @@ def _customize_inputs( # pyrefly: ignore[bad-override] device = getattr(entity.images, "device", torch.device("cpu")) scaled_bboxes = torch.zeros((0, 4), device=device, dtype=torch.float32) + # Defensive safeguard: the RF-DETR criterion (and the Hungarian + # matcher) concatenate boxes, labels and masks per target and + # require their per-image counts to match. The data pipeline + # should already guarantee this, but geometric transforms + # (cropping, tiling, box sanitization) can occasionally drop one + # annotation type without the others. If counts ever diverge we + # align boxes/labels/masks to their common length rather than + # crash deep inside the matcher. This runs unconditionally so it + # also protects the detection path (boxes vs labels), not only + # instance segmentation (boxes vs labels vs masks). + n_boxes = int(scaled_bboxes.shape[0]) + n_labels = int(ll.shape[0]) + n_masks = int(mm.shape[0]) if mm is not None else n_boxes + if not (n_boxes == n_labels == n_masks): + n = min(n_boxes, n_labels, n_masks) + logger.warning( + "RF-DETR target has mismatched annotation counts " + "(boxes=%d, labels=%d, masks=%d); aligning to %d.", + n_boxes, + n_labels, + n_masks, + n, + ) + scaled_bboxes = scaled_bboxes[:n] + ll = ll[:n] + if mm is not None: + mm = mm[:n] + target: dict[str, Any] = { "boxes": scaled_bboxes, "labels": ll, diff --git a/library/src/getitune/backend/lightning/models/common/target_utils.py b/library/src/getitune/backend/lightning/models/common/target_utils.py new file mode 100644 index 00000000000..834d69a48d2 --- /dev/null +++ b/library/src/getitune/backend/lightning/models/common/target_utils.py @@ -0,0 +1,99 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for preparing model training targets. + +This module provides defensive utilities used by the detection and instance +segmentation models when converting a :class:`~getitune.data.entity.sample.SampleBatch` +into model-specific training targets. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from getitune.data.entity.sample import SampleBatch + +logger = logging.getLogger(__name__) + + +def align_sample_batch_annotations(entity: SampleBatch) -> SampleBatch: + """Align per-image annotation counts (bboxes/labels/masks/keypoints) in place. + + Many detection and instance-segmentation criteria concatenate a sample's + per-image ``boxes``, ``labels`` and ``masks`` and require their counts to + match: + + * DETR-style models (RF-DETR, RT-DETR, DEIM/D-FINE, DEIMv2) feed them to a + Hungarian matcher that adds per-target cost matrices element-wise. A count + mismatch raises ``RuntimeError: The size of tensor a (N) must match the + size of tensor b (M) ...`` deep inside the matcher. + * mmdet-based models (Mask R-CNN, RTMDet-Inst, ATSS, SSD, YOLOX) build + ``InstanceData`` which enforces that all instance-level fields share the + same length. + + The data pipeline should already guarantee consistent counts, but geometric + transforms (cropping, tiling, bounding-box sanitization) can occasionally + drop one annotation type without dropping the paired ones. When that + happens this helper realigns the affected image's annotations to their + common minimum length and logs a warning, so training degrades gracefully + instead of crashing inside the loss. + + The trimming is a best-effort safety net: it keeps the first ``n`` entries + of each field (annotations are built per-instance in a consistent order, so + trailing extras are the most likely divergence). It only mutates images + whose counts actually diverge, leaving the common (consistent) path + untouched. + + Args: + entity: The batch to align. Modified in place. + + Returns: + The same ``entity`` instance (for convenient chaining). + """ + bboxes = entity.bboxes + labels = entity.labels + masks = entity.masks + keypoints = entity.keypoints + + # Nothing to align if there are no instance-level annotations. + if bboxes is None and labels is None and masks is None and keypoints is None: + return entity + + for i in range(entity.batch_size): + counts: list[int] = [ + int(field[i].shape[0]) + for field in (bboxes, labels, masks, keypoints) + if field is not None and i < len(field) and field[i] is not None + ] + + if not counts: + continue + + n = min(counts) + if all(c == n for c in counts): + continue # already consistent — leave untouched + + logger.warning( + "Sample %d has mismatched annotation counts " + "(bboxes=%s, labels=%s, masks=%s, keypoints=%s); aligning to %d.", + i, + None if bboxes is None else int(bboxes[i].shape[0]), + None if labels is None else int(labels[i].shape[0]), + None if masks is None else int(masks[i].shape[0]), + None if keypoints is None else int(keypoints[i].shape[0]), + n, + ) + + if bboxes is not None and bboxes[i] is not None: + bboxes[i] = bboxes[i][:n] # pyrefly: ignore[unsupported-operation] + if labels is not None and labels[i] is not None: + labels[i] = labels[i][:n] + if masks is not None and masks[i] is not None: + masks[i] = masks[i][:n] # pyrefly: ignore[unsupported-operation] + if keypoints is not None and keypoints[i] is not None: + keypoints[i] = keypoints[i][:n] + + return entity diff --git a/library/src/getitune/backend/lightning/models/detection/base.py b/library/src/getitune/backend/lightning/models/detection/base.py index 89e2675a0fe..044c9123a5e 100644 --- a/library/src/getitune/backend/lightning/models/detection/base.py +++ b/library/src/getitune/backend/lightning/models/detection/base.py @@ -23,6 +23,7 @@ DefaultSchedulerCallable, LightningModel, ) +from getitune.backend.lightning.models.common.target_utils import align_sample_batch_annotations from getitune.backend.lightning.models.utils.utils import InstanceData from getitune.backend.lightning.schedulers import LRSchedulerListCallable from getitune.backend.lightning.tools.explain.explain_algo import feature_vector_fn @@ -190,6 +191,12 @@ def _customize_inputs( ) -> dict[str, Any]: inputs: dict[str, Any] = {} + # Defensively realign per-image annotation counts before the head/criterion + # consumes them, so a boxes/labels divergence (e.g. from tiling or crop + # augmentations) does not crash the loss during training. + if self.training: + align_sample_batch_annotations(entity) + inputs["entity"] = entity inputs["mode"] = "loss" if self.training else "predict" return inputs diff --git a/library/src/getitune/backend/lightning/models/detection/rtdetr.py b/library/src/getitune/backend/lightning/models/detection/rtdetr.py index cad2055bfb2..3e9f61c7d54 100644 --- a/library/src/getitune/backend/lightning/models/detection/rtdetr.py +++ b/library/src/getitune/backend/lightning/models/detection/rtdetr.py @@ -18,6 +18,7 @@ from getitune.backend.lightning.exporter.base import ModelExporter from getitune.backend.lightning.exporter.native import LightningModelExporter from getitune.backend.lightning.models.base import DataInputParams, DefaultOptimizerCallable, DefaultSchedulerCallable +from getitune.backend.lightning.models.common.target_utils import align_sample_batch_annotations from getitune.backend.lightning.models.detection.backbones import PResNet from getitune.backend.lightning.models.detection.base import LightningDetectionModel from getitune.backend.lightning.models.detection.detectors import DETR @@ -137,6 +138,11 @@ def _customize_inputs( pad_value: int = 0, ) -> dict[str, Any]: targets: list[dict[str, Any]] = [] + # Defensively realign per-image annotation counts so a boxes/labels + # divergence (e.g. from tiling or crop augmentations) does not crash the + # Hungarian matcher, which concatenates boxes and labels per target. + if self.training: + align_sample_batch_annotations(entity) # prepare bboxes for the model if entity.bboxes is not None and entity.labels is not None: for bb, ll in zip(entity.bboxes, entity.labels): diff --git a/library/src/getitune/backend/lightning/models/instance_segmentation/base.py b/library/src/getitune/backend/lightning/models/instance_segmentation/base.py index a0197863efb..9bf4175fe4d 100644 --- a/library/src/getitune/backend/lightning/models/instance_segmentation/base.py +++ b/library/src/getitune/backend/lightning/models/instance_segmentation/base.py @@ -25,6 +25,7 @@ DefaultSchedulerCallable, LightningModel, ) +from getitune.backend.lightning.models.common.target_utils import align_sample_batch_annotations from getitune.backend.lightning.models.instance_segmentation.segmentors.maskrcnn_tv import MaskRCNN from getitune.backend.lightning.models.instance_segmentation.segmentors.two_stage import TwoStageDetector from getitune.backend.lightning.models.utils.utils import InstanceData, load_checkpoint @@ -115,6 +116,11 @@ def _create_model(self, num_classes: int | None = None) -> nn.Module: return detector def _customize_inputs(self, entity: SampleBatch) -> dict[str, Any]: + # Defensively realign per-image boxes/labels/masks counts so a divergence + # (e.g. from tiling or crop augmentations) does not crash mmdet's + # InstanceData, which requires all instance-level fields to share length. + if self.training: + align_sample_batch_annotations(entity) if isinstance(entity.images, list): entity.images, entity.imgs_info = stack_batch(entity.images, entity.imgs_info, pad_size_divisor=32) # type: ignore[assignment,arg-type] inputs: dict[str, Any] = {} diff --git a/library/src/getitune/backend/openvino/engine.py b/library/src/getitune/backend/openvino/engine.py index 205f856e066..da16994ba49 100644 --- a/library/src/getitune/backend/openvino/engine.py +++ b/library/src/getitune/backend/openvino/engine.py @@ -261,6 +261,11 @@ def test( model = self._update_checkpoint(checkpoint) metric = metric or model.metric_callable + # Propagate the datamodule tiling configuration to the model so that tile + # predictions can be merged back to the original image during evaluation. + if getattr(datamodule, "tile_config", None) is not None: + model.tile_config = datamodule.tile_config + datamodule = self._auto_configurator.update_ov_subset_pipeline( datamodule=datamodule, subset="test", @@ -363,6 +368,10 @@ def predict( f"datamodule.label_info={self.datamodule.label_info}" ) raise ValueError(msg) + + # Propagate the datamodule tiling configuration to the model for predictions. + if getattr(datamodule, "tile_config", None) is not None: + model.tile_config = datamodule.tile_config datamodule = self._auto_configurator.update_ov_subset_pipeline( datamodule=datamodule, subset="test", diff --git a/library/src/getitune/backend/openvino/models/base.py b/library/src/getitune/backend/openvino/models/base.py index 4327af9246d..1a6ff17c0ba 100644 --- a/library/src/getitune/backend/openvino/models/base.py +++ b/library/src/getitune/backend/openvino/models/base.py @@ -21,10 +21,12 @@ from model_api.tilers import Tiler from torch import Tensor +from getitune.config.data import TileConfig from getitune.data.entity.base import ( ImageInfo, ) from getitune.data.entity.sample import PredictionBatch, SampleBatch +from getitune.data.entity.tile import TileBatchData from getitune.metrics import NullMetricCallable from getitune.types.label import LabelInfo from getitune.types.task import TaskType @@ -93,6 +95,10 @@ def __init__( self.metric_callable = metric self._label_info = self._create_label_info_from_model() self._task: TaskType | None = None + # Tile configuration used to merge tile predictions back to the original image + # during tile-based evaluation/prediction. Populated by the engine from the + # datamodule when tiling is enabled; defaults to a disabled configuration. + self.tile_config: TileConfig = TileConfig(enable_tiler=False) tile_enabled = False with contextlib.suppress(RuntimeError): if isinstance(self.model, Model): @@ -630,7 +636,7 @@ def test_step(self, data_batch: SampleBatch, metric: Metric | MetricCollection) Override in subclasses for task-specific inference logic. """ - preds = self(data_batch) + preds = self.forward_tiles(data_batch) if isinstance(data_batch, TileBatchData) else self(data_batch) metric_inputs = self.prepare_metric_inputs(preds, data_batch) if isinstance(metric_inputs, list): for metric_input in metric_inputs: @@ -643,8 +649,24 @@ def predict_step(self, data_batch: SampleBatch) -> PredictionBatch: Override in subclasses to apply task-specific post-filtering (e.g. confidence threshold). """ + if isinstance(data_batch, TileBatchData): + return self.forward_tiles(data_batch) return self(data_batch) + def forward_tiles(self, inputs: TileBatchData) -> PredictionBatch: + """Run tile-based inference and merge tile predictions to full-image predictions. + + The datamodule produces a ``TileBatchData`` entity (a batch of tiles wrapping + the original images) when tiling is enabled. This method unbinds the tiles, + runs inference on each tile batch and merges the per-tile predictions back to + the original image coordinate space, mirroring the Lightning ``forward_tiles`` + behaviour. + + Must be overridden by task-specific subclasses that support tiling. + """ + msg = f"Tile-based inference is not supported for {type(self).__name__}." + raise NotImplementedError(msg) + def __call__(self, *args, **kwds): """Call the model for inference. diff --git a/library/src/getitune/backend/openvino/models/detection.py b/library/src/getitune/backend/openvino/models/detection.py index aae1c94b9a0..d5c4f43723b 100644 --- a/library/src/getitune/backend/openvino/models/detection.py +++ b/library/src/getitune/backend/openvino/models/detection.py @@ -11,9 +11,11 @@ from model_api.tilers import DetectionTiler from torchvision import tv_tensors +from getitune.backend.lightning.tools.tile_merge import DetectionTileMerge from getitune.backend.openvino.models.base import OVModel from getitune.backend.openvino.models.utils import rescale_bboxes_to_original from getitune.data.entity.sample import PredictionBatch, SampleBatch +from getitune.data.entity.tile import TileBatchData from getitune.metrics import MetricCallable, MetricInput from getitune.metrics.fmeasure import MeanAveragePrecisionFMeasureCallable from getitune.types.task import TaskType @@ -202,7 +204,10 @@ def _customize_outputs( dtype=torch.float32, ), ) - scores.append(torch.tensor(output.scores.reshape(-1))) + # Cast scores to float32 to match the float32 bboxes. ModelAPI returns numpy + # arrays whose default float dtype is float64; a float64/float32 mismatch makes + # torchvision NMS fail ("dets should have the same type as scores") during tile merge. + scores.append(torch.tensor(output.scores.reshape(-1), dtype=torch.float32)) labels.append(torch.tensor(output.labels.reshape(-1) - label_shift, dtype=torch.long)) if outputs and outputs[0].saliency_map.size > 1: @@ -268,9 +273,42 @@ def compute_metrics(self, metric: Metric | MetricCollection) -> dict: compute_kwargs = {"best_confidence_threshold": best_confidence_threshold} return super()._compute_metrics(metric, **compute_kwargs) + def forward_tiles(self, inputs: TileBatchData) -> PredictionBatch: + """Run tiled detection inference and merge tile predictions to full-image predictions. + + Unbinds the tile batch into per-tile ``SampleBatch`` inputs, runs inference on + each, and merges the per-tile predictions back to the original image coordinate + space using :class:`DetectionTileMerge`, mirroring the Lightning tiling path. + + Args: + inputs (TileBatchData): Batch of tiles wrapping the original images. + + Returns: + PredictionBatch: Merged full-image detection predictions. + """ + tile_preds: list[PredictionBatch] = [] + tile_infos: list = [] + merger = DetectionTileMerge( + inputs.imgs_info, + self.label_info.num_classes, + self.tile_config, + ) + for batch_tile_infos, batch_tile_input in inputs.unbind(): + tile_preds.append(self(batch_tile_input)) + tile_infos.append(batch_tile_infos) + + pred_entities = merger.merge(tile_preds, tile_infos) + return PredictionBatch( + images=[pred_entity.image for pred_entity in pred_entities], + imgs_info=[pred_entity.img_info for pred_entity in pred_entities], + scores=[pred_entity.scores for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + bboxes=[pred_entity.bboxes for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + labels=[pred_entity.label for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + ) + def predict_step(self, data_batch: SampleBatch) -> PredictionBatch: """Run detection inference and filter by confidence threshold.""" - predictions = self(data_batch) + predictions = self.forward_tiles(data_batch) if isinstance(data_batch, TileBatchData) else self(data_batch) threshold = self.hparams.get("best_confidence_threshold", None) if not threshold: return predictions diff --git a/library/src/getitune/backend/openvino/models/instance_segmentation.py b/library/src/getitune/backend/openvino/models/instance_segmentation.py index 00eca3bd42a..0105f542934 100644 --- a/library/src/getitune/backend/openvino/models/instance_segmentation.py +++ b/library/src/getitune/backend/openvino/models/instance_segmentation.py @@ -11,9 +11,11 @@ from model_api.tilers import InstanceSegmentationTiler from torchvision import tv_tensors +from getitune.backend.lightning.tools.tile_merge import InstanceSegTileMerge from getitune.backend.openvino.models.base import OVModel from getitune.backend.openvino.models.utils import rescale_bboxes_to_original, rescale_masks_to_original from getitune.data.entity.sample import PredictionBatch, SampleBatch +from getitune.data.entity.tile import TileBatchData from getitune.data.utils.structures.mask.mask_util import encode_rle from getitune.metrics import MetricInput from getitune.metrics.fmeasure import MaskRLEMeanAPFMeasureCallable @@ -150,7 +152,10 @@ def _customize_outputs( dtype=torch.float32, ), ) - scores.append(torch.tensor(output.scores.reshape(-1))) + # Cast scores to float32 to match the float32 bboxes. ModelAPI returns numpy + # arrays whose default float dtype is float64; a float64/float32 mismatch makes + # torchvision NMS fail ("dets should have the same type as scores") during tile merge. + scores.append(torch.tensor(output.scores.reshape(-1), dtype=torch.float32)) raw_masks = torch.tensor(output.masks) if raw_masks.shape[-2:] == (ori_h, ori_w): @@ -254,9 +259,44 @@ def compute_metrics(self, metric: Metric | MetricCollection) -> dict: compute_kwargs = {"best_confidence_threshold": best_confidence_threshold} return super()._compute_metrics(metric, **compute_kwargs) + def forward_tiles(self, inputs: TileBatchData) -> PredictionBatch: + """Run tiled instance segmentation inference and merge tile predictions. + + Unbinds the tile batch into per-tile ``SampleBatch`` inputs, runs inference on + each, and merges the per-tile predictions (boxes, labels, scores and masks) back + to the original image coordinate space using :class:`InstanceSegTileMerge`, + mirroring the Lightning tiling path. + + Args: + inputs (TileBatchData): Batch of tiles wrapping the original images. + + Returns: + PredictionBatch: Merged full-image instance segmentation predictions. + """ + tile_preds: list[PredictionBatch] = [] + tile_infos: list = [] + merger = InstanceSegTileMerge( + inputs.imgs_info, + self.label_info.num_classes, + self.tile_config, + ) + for batch_tile_infos, batch_tile_input in inputs.unbind(): + tile_preds.append(self(batch_tile_input)) + tile_infos.append(batch_tile_infos) + + pred_entities = merger.merge(tile_preds, tile_infos) + return PredictionBatch( + images=[pred_entity.image for pred_entity in pred_entities], + imgs_info=[pred_entity.img_info for pred_entity in pred_entities], + scores=[pred_entity.scores for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + bboxes=[pred_entity.bboxes for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + labels=[pred_entity.label for pred_entity in pred_entities], # pyrefly: ignore[bad-argument-type] + masks=[pred_entity.masks for pred_entity in pred_entities], + ) + def predict_step(self, data_batch: SampleBatch) -> PredictionBatch: """Run instance segmentation inference and filter by confidence threshold.""" - predictions = self(data_batch) + predictions = self.forward_tiles(data_batch) if isinstance(data_batch, TileBatchData) else self(data_batch) threshold = self.hparams.get("best_confidence_threshold", None) if not threshold: return predictions diff --git a/library/src/getitune/config/data.py b/library/src/getitune/config/data.py index 02b8e0a0662..e9c8610e9ce 100644 --- a/library/src/getitune/config/data.py +++ b/library/src/getitune/config/data.py @@ -146,6 +146,19 @@ class TileConfig: object_tile_ratio: float = 0.03 sampling_ratio: float = 1.0 + def __repr__(self): + """Return a string representation of the TileConfig instance.""" + return ( + f"TileConfig(enable_tiler={self.enable_tiler}, " + f"enable_adaptive_tiling={self.enable_adaptive_tiling}, " + f"tile_size={self.tile_size}, " + f"overlap={self.overlap}, " + f"iou_threshold={self.iou_threshold}, " + f"max_num_instances={self.max_num_instances}, " + f"object_tile_ratio={self.object_tile_ratio}, " + f"sampling_ratio={self.sampling_ratio})" + ) + def clone(self) -> TileConfig: """Return a deep copied one of this instance.""" return deepcopy(self) diff --git a/library/src/getitune/data/augmentation/pipeline.py b/library/src/getitune/data/augmentation/pipeline.py index 24edde96aaa..0f2bd304d70 100644 --- a/library/src/getitune/data/augmentation/pipeline.py +++ b/library/src/getitune/data/augmentation/pipeline.py @@ -561,6 +561,34 @@ def data_keys(self) -> list[str]: """Get data keys used by the pipeline.""" return self._data_keys + def apply_image_only(self, images: torch.Tensor) -> torch.Tensor: + """Apply only image-level (non-geometric) augmentations to a batch of images. + + Unlike :meth:`forward`, this does not require (or transform) annotations and + therefore does not depend on the Kornia ``AugmentationSequential`` data-keys. + It is used for tile batches during validation/testing, where the tiles carry + no per-tile annotations (predictions are merged back to the original image + afterwards) and only intensity transforms such as ``Normalize`` are relevant. + + Geometric augmentations are skipped because applying them without annotation + tracking would desynchronise boxes/masks; validation/test pipelines only + contain intensity transforms in practice. + + Args: + images: Batched images tensor in BCHW format, values in [0, 1]. + + Returns: + The transformed images tensor. + """ + if not self._augmentations_list: + return images + out = images + for aug in self._augmentations_list: + if isinstance(aug, K.GeometricAugmentationBase2D): + continue + out = aug(out) + return out + @classmethod def list_available_transforms(cls) -> list[type]: """List available Kornia augmentation classes.""" diff --git a/library/src/getitune/data/dataset/_tiling_patches.py b/library/src/getitune/data/dataset/_tiling_patches.py new file mode 100644 index 00000000000..2a754e3445e --- /dev/null +++ b/library/src/getitune/data/dataset/_tiling_patches.py @@ -0,0 +1,131 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime patches for Datumaro's experimental tiling. + +Datumaro's :class:`InstanceMaskTiler` crops every instance mask to each tile but +does **not** implement ``is_filterable``. As a result, instance masks are *never* +filtered during tiling: a tile keeps a mask entry for *every* instance of the +source image, even instances that do not overlap the tile at all. + +Bounding boxes and (list) labels, on the other hand, *are* filterable +(``BboxTiler``/``LabelTiler``) and are pruned to the instances whose box +intersects the tile. Consequently, after tiling an instance-segmentation +sample ends up with ``len(masks) != len(bboxes) == len(labels)``. + +This desynchronisation corrupts training for every instance-segmentation model +(masks no longer correspond to their boxes/labels) and makes models that +concatenate boxes and masks per target — e.g. RF-DETR's Hungarian matcher — +crash outright with a shape-mismatch error. + +The patch below re-registers a fixed ``InstanceMaskTiler`` that self-filters +instances using the **same tile-intersection criterion** as ``BboxTiler`` (the +instance's full-image mask bounding box must intersect the tile rectangle), so +masks stay aligned with boxes and labels. Registration is idempotent and +mirrors the existing Kornia monkey-patch pattern used by the augmentation +pipeline. +""" + +from __future__ import annotations + +import numpy as np +import polars as pl +from datumaro.experimental.fields.masks import InstanceMaskField +from datumaro.experimental.tiling.tiler_registry import TilerRegistry +from datumaro.experimental.tiling.tilers import InstanceMaskTiler + +_INSTANCE_MASK_TILER_PATCHED = False + + +def _instance_intersects_tile(instance: np.ndarray, x: int, y: int, width: int, height: int) -> bool: + """Return whether an instance mask's bounding box intersects the tile. + + The criterion mirrors ``BboxTiler`` exactly: using the instance's full-image + bounding box ``(x1, y1, x2, y2)`` (derived from its non-zero pixels, with + ``x2``/``y2`` exclusive), the instance is kept iff:: + + (x2 > tile_x) & (x1 < tile_x2) & (y2 > tile_y) & (y1 < tile_y2) + + An all-zero mask (no pixels) has no bounding box and is dropped. + + Args: + instance: Full-image instance mask of shape ``(H, W)``. + x: Tile left coordinate. + y: Tile top coordinate. + width: Tile width. + height: Tile height. + + Returns: + ``True`` if the instance overlaps the tile rectangle, else ``False``. + """ + rows = np.nonzero(instance.any(axis=1))[0] + cols = np.nonzero(instance.any(axis=0))[0] + if rows.size == 0 or cols.size == 0: + return False + y1, y2 = int(rows[0]), int(rows[-1]) + 1 + x1, x2 = int(cols[0]), int(cols[-1]) + 1 + return bool((x2 > x) and (x1 < x + width) and (y2 > y) and (y1 < y + height)) + + +class _FilteringInstanceMaskTiler(InstanceMaskTiler): + """``InstanceMaskTiler`` that drops instances not overlapping the tile. + + Unlike the upstream tiler, this implementation prunes each tile's instance + masks to those whose bounding box intersects the tile, keeping the mask + count consistent with the (independently filtered) bounding boxes and + labels. Instances are kept in their original order, so alignment with + boxes/labels is preserved. + """ + + def tile(self, df: pl.DataFrame, tiles_df: pl.DataFrame, slice_offset: int = 0) -> pl.DataFrame: + """Extract and filter instance mask regions for each tile.""" + column_name = self.field_spec.name # type: ignore[attr-defined] + shape_column = f"{column_name}_shape" + results_data = [] + results_shape = [] + + for tile_row in tiles_df["tile"]: + image_id = tile_row["source_sample_idx"] - slice_offset + instances_data = df[column_name][image_id] # Flattened 3D array + instances_shape = df[shape_column][image_id] # (num_instances, height, width) + + x = tile_row["x"] + y = tile_row["y"] + width = tile_row["width"] + height = tile_row["height"] + + instances = instances_data.reshape(instances_shape).to_numpy() # (N, H, W) + + # Keep only instances whose full-image bounding box intersects the + # tile, mirroring BboxTiler so masks stay aligned with boxes/labels. + if instances.shape[0] > 0: + keep = np.array( + [_instance_intersects_tile(inst, x, y, width, height) for inst in instances], + dtype=bool, + ) + instances = instances[keep] + + tile_result = instances[:, y : y + height, x : x + width] # (M, tile_h, tile_w) + + results_data.append(tile_result.reshape(-1)) + results_shape.append(tile_result.shape) + + return pl.DataFrame( + { + column_name: results_data, + shape_column: results_shape, + } + ) + + +def patch_instance_mask_tiler() -> None: + """Register the filtering instance-mask tiler (idempotent). + + Must be called before any tiling transform is created so that the fixed + tiler is picked up by ``create_tilers``. + """ + global _INSTANCE_MASK_TILER_PATCHED # noqa: PLW0603 + if _INSTANCE_MASK_TILER_PATCHED: + return + TilerRegistry.register(InstanceMaskField)(_FilteringInstanceMaskTiler) + _INSTANCE_MASK_TILER_PATCHED = True diff --git a/library/src/getitune/data/dataset/tile.py b/library/src/getitune/data/dataset/tile.py index 1b8d7233380..2abc53a0f44 100644 --- a/library/src/getitune/data/dataset/tile.py +++ b/library/src/getitune/data/dataset/tile.py @@ -23,6 +23,7 @@ ) from getitune.types.task import TaskType +from ._tiling_patches import patch_instance_mask_tiler from .base import VisionDataset, _ensure_chw_format if TYPE_CHECKING: @@ -31,6 +32,12 @@ from getitune.data.dataset.instance_segmentation import InstanceSegDataset from getitune.data.dataset.segmentation import SegmentationDataset +# Fix Datumaro's InstanceMaskTiler so instance masks are filtered in sync with +# bounding boxes/labels during tiling (see ``_tiling_patches`` for details). +# Applied at import time so every tiling transform created via this module uses +# the corrected tiler. +patch_instance_mask_tiler() + # ruff: noqa: SLF001 # NOTE: Disable private-member-access (SLF001). # This is a workaround so we could apply the same transforms to tiles as the original dataset. diff --git a/library/src/getitune/recipe/detection/deim_dfine_l_tile.yaml b/library/src/getitune/recipe/detection/deim_dfine_l_tile.yaml new file mode 100644 index 00000000000..0d8dcdf3483 --- /dev/null +++ b/library/src/getitune/recipe/detection/deim_dfine_l_tile.yaml @@ -0,0 +1,115 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMDFine + init_args: + model_name: deim_dfine_hgnetv2_l + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0005 + betas: [0.9, 0.999] + weight_decay: 0.000125 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + min_delta: 0.001 + warmup_iters: 30 + warmup_epochs: 3 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 100 + warmup_epochs: 7 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] diff --git a/library/src/getitune/recipe/detection/deim_dfine_m_tile.yaml b/library/src/getitune/recipe/detection/deim_dfine_m_tile.yaml new file mode 100644 index 00000000000..74ab549b961 --- /dev/null +++ b/library/src/getitune/recipe/detection/deim_dfine_m_tile.yaml @@ -0,0 +1,115 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMDFine + init_args: + model_name: deim_dfine_hgnetv2_m + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0004 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 15 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 20 + min_delta: 0.001 + warmup_iters: 30 + warmup_epochs: 3 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 100 + warmup_epochs: 7 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] diff --git a/library/src/getitune/recipe/detection/deim_dfine_x_tile.yaml b/library/src/getitune/recipe/detection/deim_dfine_x_tile.yaml new file mode 100644 index 00000000000..704b7fce9c5 --- /dev/null +++ b/library/src/getitune/recipe/detection/deim_dfine_x_tile.yaml @@ -0,0 +1,115 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMDFine + init_args: + model_name: deim_dfine_hgnetv2_x + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0005 + betas: [0.9, 0.999] + weight_decay: 0.000125 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 6 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 10 + min_delta: 0.001 + warmup_iters: 40 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 100 + warmup_epochs: 7 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: [] diff --git a/library/src/getitune/recipe/detection/deimv2_l_tile.yaml b/library/src/getitune/recipe/detection/deimv2_l_tile.yaml new file mode 100644 index 00000000000..1302e408186 --- /dev/null +++ b/library/src/getitune/recipe/detection/deimv2_l_tile.yaml @@ -0,0 +1,128 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMV2 + init_args: + model_name: deimv2_l + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0005 + betas: [0.9, 0.999] + weight_decay: 0.000125 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 10 + min_delta: 0.001 + warmup_iters: 50 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 50 + warmup_epochs: 10 + patience: 10 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] diff --git a/library/src/getitune/recipe/detection/deimv2_m_tile.yaml b/library/src/getitune/recipe/detection/deimv2_m_tile.yaml new file mode 100644 index 00000000000..dfa0a9fa01f --- /dev/null +++ b/library/src/getitune/recipe/detection/deimv2_m_tile.yaml @@ -0,0 +1,127 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMV2 + init_args: + model_name: deimv2_m + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0004 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 10 + min_delta: 0.001 + warmup_iters: 50 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 100 + warmup_epochs: 7 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] diff --git a/library/src/getitune/recipe/detection/deimv2_s_tile.yaml b/library/src/getitune/recipe/detection/deimv2_s_tile.yaml new file mode 100644 index 00000000000..3c7f6bfd355 --- /dev/null +++ b/library/src/getitune/recipe/detection/deimv2_s_tile.yaml @@ -0,0 +1,127 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.DEIMV2 + init_args: + model_name: deimv2_s + label_info: 80 + multi_scale: false + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0004 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 30 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.5 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +# NOTE: Tiling is incompatible with mosaic/mixup based augmentation scheduling, +# so the AugmentationSchedulerCallback used by the non-tiling recipe is omitted here +# and a static tiling-friendly augmentation pipeline is defined in overrides. +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 10 + min_delta: 0.001 + warmup_iters: 50 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" # use engine.work_dir + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + +overrides: + callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 3 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + warmup_iters: 100 + warmup_epochs: 7 + + data: + input_size: + - 640 + - 640 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + augmentations_gpu: + - class_path: kornia.augmentation.RandomHorizontalFlip + init_args: + p: 0.5 + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + test_subset: + batch_size: 8 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + augmentations_gpu: + - class_path: kornia.augmentation.Normalize + init_args: + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] diff --git a/library/src/getitune/recipe/detection/rfdetr_large_tile.yaml b/library/src/getitune/recipe/detection/rfdetr_large_tile.yaml new file mode 100644 index 00000000000..dac1db81f79 --- /dev/null +++ b/library/src/getitune/recipe/detection/rfdetr_large_tile.yaml @@ -0,0 +1,91 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.rfdetr.RFDETR + init_args: + model_name: rfdetr_large + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + gradient_clip_val: 0.1 + data: + input_size: + - 704 + - 704 + task: DETECTION + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + + test_subset: + batch_size: 4 diff --git a/library/src/getitune/recipe/detection/rfdetr_medium_tile.yaml b/library/src/getitune/recipe/detection/rfdetr_medium_tile.yaml new file mode 100644 index 00000000000..7c9a9107862 --- /dev/null +++ b/library/src/getitune/recipe/detection/rfdetr_medium_tile.yaml @@ -0,0 +1,85 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.rfdetr.RFDETR + init_args: + model_name: rfdetr_medium + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.00021 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + gradient_clip_val: 0.1 + data: + input_size: + - 576 + - 576 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler diff --git a/library/src/getitune/recipe/detection/rfdetr_nano_tile.yaml b/library/src/getitune/recipe/detection/rfdetr_nano_tile.yaml new file mode 100644 index 00000000000..56dd699fdcb --- /dev/null +++ b/library/src/getitune/recipe/detection/rfdetr_nano_tile.yaml @@ -0,0 +1,85 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.rfdetr.RFDETR + init_args: + model_name: rfdetr_nano + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.00021 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + gradient_clip_val: 0.1 + data: + input_size: + - 384 + - 384 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler diff --git a/library/src/getitune/recipe/detection/rfdetr_small_tile.yaml b/library/src/getitune/recipe/detection/rfdetr_small_tile.yaml new file mode 100644 index 00000000000..f64281ed562 --- /dev/null +++ b/library/src/getitune/recipe/detection/rfdetr_small_tile.yaml @@ -0,0 +1,85 @@ +task: DETECTION +model: + class_path: getitune.backend.lightning.models.detection.rfdetr.RFDETR + init_args: + model_name: rfdetr_small + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.00021 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + +callback_monitor: val/map_50 + +data: ../_base_/data/detection_tile.yaml + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + gradient_clip_val: 0.1 + data: + input_size: + - 512 + - 512 + task: DETECTION + train_subset: + batch_size: 8 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_2xlarge_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_2xlarge_tile.yaml new file mode 100644 index 00000000000..b9db0283fb2 --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_2xlarge_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_2xl + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 768 + - 768 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_large_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_large_tile.yaml new file mode 100644 index 00000000000..b011588246b --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_large_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_l + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 504 + - 504 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_medium_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_medium_tile.yaml new file mode 100644 index 00000000000..484d7e8c4ae --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_medium_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_m + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 432 + - 432 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_nano_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_nano_tile.yaml new file mode 100644 index 00000000000..eb3c0346dd8 --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_nano_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_n + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 312 + - 312 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_small_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_small_tile.yaml new file mode 100644 index 00000000000..9f89498c07e --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_small_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_s + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 384 + - 384 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_xlarge_tile.yaml b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_xlarge_tile.yaml new file mode 100644 index 00000000000..723b5156c82 --- /dev/null +++ b/library/src/getitune/recipe/instance_segmentation/rfdetr_seg_xlarge_tile.yaml @@ -0,0 +1,109 @@ +task: INSTANCE_SEGMENTATION + +model: + class_path: getitune.backend.lightning.models.instance_segmentation.RFDETRInst + init_args: + model_name: rfdetr_seg_xl + label_info: 91 + multi_scale: true + + optimizer: + class_path: torch.optim.AdamW + init_args: + lr: 0.0001 + betas: [0.9, 0.999] + weight_decay: 0.0001 + + scheduler: + class_path: getitune.backend.lightning.schedulers.LinearWarmupSchedulerCallable + init_args: + num_warmup_steps: 0 + main_scheduler_callable: + class_path: lightning.pytorch.cli.ReduceLROnPlateau + init_args: + mode: max + factor: 0.1 + patience: 10 + monitor: val/map_50 + +engine: + device: auto + accumulate_grad_batches: 4 + +data: ../_base_/data/instance_segmentation.yaml + +callback_monitor: val/map_50 + +callbacks: + - class_path: getitune.backend.lightning.callbacks.adaptive_train_scheduling.AdaptiveTrainScheduling + init_args: + max_interval: 1 + min_lrschedule_patience: 5 + - class_path: getitune.backend.lightning.callbacks.adaptive_early_stopping.EarlyStoppingWithWarmup + init_args: + mode: max + patience: 15 + check_on_train_epoch_end: false + monitor: val/map_50 + min_delta: 0.001 + warmup_iters: 100 + warmup_epochs: 10 + - class_path: lightning.pytorch.callbacks.ModelCheckpoint + init_args: + dirpath: "" + monitor: val/map_50 + mode: max + save_top_k: 1 + save_last: true + auto_insert_metric_name: false + filename: "checkpoints/epoch_{epoch:03d}" + - class_path: lightning.pytorch.callbacks.EMAWeightAveraging + init_args: + decay: 0.993 + +overrides: + max_epochs: 100 + gradient_clip_val: 0.1 + data: + task: INSTANCE_SEGMENTATION + input_size: + - 624 + - 624 + tile_config: + enable_tiler: true + enable_adaptive_tiling: true + train_subset: + batch_size: 4 + augmentations_cpu: + - class_path: torchvision.transforms.v2.RandomZoomOut + init_args: + fill: 0 + p: 0.5 + - class_path: getitune.data.augmentation.transforms.RandomIoUCrop + - class_path: torchvision.transforms.v2.SanitizeBoundingBoxes + init_args: + min_size: 1 + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + sampler: + class_path: getitune.data.samplers.balanced_sampler.BalancedSampler + + val_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false + + test_subset: + batch_size: 4 + augmentations_cpu: + - class_path: getitune.data.augmentation.transforms.Resize + init_args: + size: $(input_size) + keep_aspect_ratio: false + resize_targets: false diff --git a/library/tests/unit/backend/lightning/models/common/test_target_utils.py b/library/tests/unit/backend/lightning/models/common/test_target_utils.py new file mode 100644 index 00000000000..367e8ce8c68 --- /dev/null +++ b/library/tests/unit/backend/lightning/models/common/test_target_utils.py @@ -0,0 +1,85 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the shared annotation-alignment helper.""" + +from __future__ import annotations + +import torch +from torchvision import tv_tensors + +from getitune.backend.lightning.models.common.target_utils import align_sample_batch_annotations +from getitune.data.entity.base import ImageInfo +from getitune.data.entity.sample import SampleBatch + + +def _bbox(n: int) -> tv_tensors.BoundingBoxes: + data = torch.tensor([[0, 0, 10, 10]], dtype=torch.float32).repeat(n, 1) if n else torch.zeros((0, 4)) + return tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + data, format=tv_tensors.BoundingBoxFormat.XYXY, canvas_size=(64, 64) + ) + + +def _mask(n: int) -> tv_tensors.Mask: + return tv_tensors.Mask(torch.zeros((n, 64, 64), dtype=torch.uint8)) + + +def _batch(bbox_counts, label_counts, mask_counts=None) -> SampleBatch: + bs = len(bbox_counts) + return SampleBatch( + images=torch.zeros((bs, 3, 64, 64)), + bboxes=[_bbox(c) for c in bbox_counts], + labels=[torch.zeros(c, dtype=torch.long) for c in label_counts], + masks=None if mask_counts is None else [_mask(c) for c in mask_counts], + imgs_info=[ + ImageInfo(img_idx=i, img_shape=(64, 64), ori_shape=(64, 64)) # pyrefly: ignore[no-matching-overload] + for i in range(bs) + ], + ) + + +class TestAlignSampleBatchAnnotations: + """Tests for align_sample_batch_annotations.""" + + def test_consistent_counts_are_untouched(self) -> None: + """When all counts already match, the annotations are left as-is.""" + batch = _batch(bbox_counts=[2, 1], label_counts=[2, 1], mask_counts=[2, 1]) + align_sample_batch_annotations(batch) + assert batch.bboxes is not None + assert batch.masks is not None + assert batch.labels is not None + assert [b.shape[0] for b in batch.bboxes] == [2, 1] + assert [m.shape[0] for m in batch.masks] == [2, 1] + assert [labels.shape[0] for labels in batch.labels] == [2, 1] + + def test_detection_boxes_labels_mismatch_is_aligned(self) -> None: + """A boxes-vs-labels divergence is trimmed to the per-image minimum.""" + batch = _batch(bbox_counts=[3, 1], label_counts=[1, 1]) + align_sample_batch_annotations(batch) + assert batch.bboxes is not None + assert batch.labels is not None + assert batch.bboxes[0].shape[0] == 1 + assert batch.labels[0].shape[0] == 1 + # second image already consistent + assert batch.bboxes[1].shape[0] == 1 + + def test_instance_seg_masks_mismatch_is_aligned(self) -> None: + """A masks-vs-boxes divergence is aligned across boxes/labels/masks.""" + # Image 0: 1 box / 1 label but 3 masks; Image 1: 2 boxes/labels but 1 mask. + batch = _batch(bbox_counts=[1, 2], label_counts=[1, 2], mask_counts=[3, 1]) + align_sample_batch_annotations(batch) + assert batch.bboxes is not None + assert batch.labels is not None + assert batch.masks is not None + for i in range(2): + n = batch.bboxes[i].shape[0] + assert batch.labels[i].shape[0] == n + assert batch.masks[i].shape[0] == n + assert batch.bboxes[0].shape[0] == 1 + assert batch.bboxes[1].shape[0] == 1 + + def test_no_annotations_is_noop(self) -> None: + """Batches without instance annotations are handled gracefully.""" + batch = SampleBatch(images=torch.zeros((1, 3, 64, 64))) + # Should not raise. + assert align_sample_batch_annotations(batch) is batch diff --git a/library/tests/unit/backend/lightning/models/instance_segmentation/test_rfdetr_inst.py b/library/tests/unit/backend/lightning/models/instance_segmentation/test_rfdetr_inst.py index a5cdd02ae17..0c4e40747aa 100644 --- a/library/tests/unit/backend/lightning/models/instance_segmentation/test_rfdetr_inst.py +++ b/library/tests/unit/backend/lightning/models/instance_segmentation/test_rfdetr_inst.py @@ -233,3 +233,69 @@ def test_customize_inputs(self, fxt_instance_seg_batch) -> None: assert "masks" in target assert "size" in target assert "orig_size" in target + + def test_customize_inputs_aligns_mismatched_annotation_counts(self) -> None: + """Mismatched per-image boxes/labels/masks are aligned to a common count. + + Regression test for a crash inside the RF-DETR Hungarian matcher + (``RuntimeError: The size of tensor a (N) must match the size of tensor + b (M) ...``) that occurred when a geometric/tiling transform dropped one + annotation type (e.g. boxes) without dropping the paired masks. The + criterion concatenates boxes/labels/masks per target and requires their + counts to match, so ``_customize_inputs`` must align them. + """ + from torchvision import tv_tensors + + from getitune.data.entity.base import ImageInfo + from getitune.data.entity.sample import SampleBatch + + model = RFDETRInst(model_name="rfdetr_seg_n", label_info=3) + + # Image 0: 1 box / 1 label but 3 masks (masks not dropped with boxes). + # Image 1: 2 masks / 2 labels but only 1 box. + batch = SampleBatch( + images=torch.stack([torch.randn(3, 320, 320), torch.randn(3, 320, 320)]), + bboxes=[ + tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + torch.tensor([[10, 10, 50, 50]], dtype=torch.float32), + format=tv_tensors.BoundingBoxFormat.XYXY, + canvas_size=(320, 320), + ), + tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + torch.tensor([[20, 20, 80, 80]], dtype=torch.float32), + format=tv_tensors.BoundingBoxFormat.XYXY, + canvas_size=(320, 320), + ), + ], + labels=[ + torch.tensor([0], dtype=torch.long), + torch.tensor([1, 2], dtype=torch.long), + ], + masks=[ + tv_tensors.Mask(torch.zeros((3, 320, 320), dtype=torch.uint8)), + tv_tensors.Mask(torch.zeros((2, 320, 320), dtype=torch.uint8)), + ], + imgs_info=[ + ImageInfo( # pyrefly: ignore[no-matching-overload] + img_idx=0, img_shape=(320, 320), ori_shape=(320, 320) + ), + ImageInfo( # pyrefly: ignore[no-matching-overload] + img_idx=1, img_shape=(320, 320), ori_shape=(320, 320) + ), + ], + ) + + customized = model._customize_inputs(batch) + targets = customized["targets"] + assert len(targets) == 2 + + # Every target must have matching boxes/labels/masks counts so the + # criterion's per-target concatenation does not raise. + for target in targets: + n_boxes = target["boxes"].shape[0] + assert target["labels"].shape[0] == n_boxes + assert target["masks"].shape[0] == n_boxes + + # Counts are aligned to the per-image minimum (1 for both images here). + assert targets[0]["boxes"].shape[0] == 1 + assert targets[1]["boxes"].shape[0] == 1 diff --git a/library/tests/unit/backend/openvino/models/test_detection.py b/library/tests/unit/backend/openvino/models/test_detection.py index cf6f0c0f195..c4407d3efcb 100644 --- a/library/tests/unit/backend/openvino/models/test_detection.py +++ b/library/tests/unit/backend/openvino/models/test_detection.py @@ -12,8 +12,10 @@ from torchvision import tv_tensors from getitune.backend.openvino.models.detection import OVDetectionModel +from getitune.config.data import TileConfig from getitune.data.entity.base import ImageInfo from getitune.data.entity.sample import PredictionBatch, SampleBatch +from getitune.data.entity.tile import TileBatchData class _FakeDetectionResult: @@ -99,6 +101,32 @@ def test_rescales_bboxes_when_shapes_differ(self, detection_model): torch.testing.assert_close(result_bboxes.data, expected_bboxes) assert result_bboxes.canvas_size == ori_shape + def test_scores_are_float32_when_modelapi_returns_float64(self, detection_model): + """Scores must be float32 to match float32 bboxes. + + ModelAPI returns numpy arrays whose default float dtype is float64. If scores were + left as float64, torchvision NMS during tile merge fails with + "dets should have the same type as scores". This regression test locks the dtype. + """ + img_shape = (640, 640) + ori_shape = (640, 640) + + bboxes = np.array([[10.0, 20.0, 30.0, 40.0]], dtype=np.float32) + # ModelAPI default float dtype is float64. + scores = np.array([0.9], dtype=np.float64) + labels = np.array([0], dtype=np.int64) + + outputs = [_FakeDetectionResult(bboxes, scores, labels)] + inputs = self._make_sample_batch(batch_size=1, img_shape=img_shape, ori_shape=ori_shape) + + result = detection_model._customize_outputs(outputs, inputs) + + assert result.scores is not None + assert result.bboxes is not None + assert result.scores[0].dtype == torch.float32 + # Scores and bboxes share a dtype so torchvision NMS won't raise during tile merge. + assert result.scores[0].dtype == result.bboxes[0].dtype + def test_no_rescaling_when_shapes_equal(self, detection_model): """When img_shape == ori_shape, bboxes should remain unchanged.""" shape = (640, 640) @@ -277,3 +305,127 @@ def test_letterbox_rescaling(self, detection_model): expected = torch.tensor([[expected_x1, expected_y1, expected_x2, expected_y2]], dtype=torch.float32) torch.testing.assert_close(result_bboxes.data, expected, atol=1e-4, rtol=1e-4) assert result_bboxes.canvas_size == ori_shape + + +class _FakeTileBatchDet(TileBatchData): + """Minimal TileBatchData subclass exposing detection ground-truth fields for tests.""" + + def __init__(self, imgs_info, unbind_result): + # Bypass the dataclass __init__: we only need imgs_info and unbind() for these tests. + self.imgs_info = imgs_info + self._unbind_result = unbind_result + self.bboxes = [ + tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + torch.empty((0, 4)), + format="XYXY", + canvas_size=info.ori_shape, + ) + for info in imgs_info + ] + self.labels = [torch.empty((0,), dtype=torch.long) for _ in imgs_info] + + def unbind(self) -> list: + return self._unbind_result + + +class TestOVDetectionModelTiling: + """Tests for tile-aware inference in OVDetectionModel (forward_tiles / test_step routing).""" + + @pytest.fixture + def detection_model(self): + with patch.object(OVDetectionModel, "__init__", lambda *_args, **_kwargs: None): + model = OVDetectionModel.__new__(OVDetectionModel) + model.model = MagicMock() + model.tile_config = TileConfig(enable_tiler=True, tile_size=(200, 200)) + model._label_info = MagicMock() + model._label_info.num_classes = 3 + model.hparams = {} + return model + + def _make_tile_batch(self) -> _FakeTileBatchDet: + ori_info = ImageInfo( # pyrefly: ignore[no-matching-overload] + img_idx=0, + img_shape=(256, 256), + ori_shape=(256, 256), + ) + tile_info = ImageInfo( # pyrefly: ignore[no-matching-overload] + img_idx=0, + img_shape=(200, 200), + ori_shape=(200, 200), + ) + # A single unbound tile batch (one SampleBatch of tile images) with matching tile infos. + tile_sample_batch = SampleBatch( + images=[torch.rand(3, 200, 200)], + imgs_info=[tile_info], + ) + tile_infos = [MagicMock()] # opaque to the (mocked) merger + return _FakeTileBatchDet(imgs_info=[ori_info], unbind_result=[(tile_infos, tile_sample_batch)]) + + def test_forward_tiles_unbinds_and_merges(self, detection_model): + """forward_tiles should run per-tile inference and merge results back to the original image.""" + tile_batch = self._make_tile_batch() + + # Per-tile forward returns an (arbitrary) prediction batch. + per_tile_pred = PredictionBatch(images=[], imgs_info=[]) + detection_model.forward = MagicMock(return_value=per_tile_pred) + + # Merged full-image prediction returned by the tile merger. + merged = MagicMock() + merged.image = torch.empty(3, 256, 256) + merged.img_info = tile_batch.imgs_info[0] + merged.scores = torch.tensor([0.9]) + merged.bboxes = tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + torch.tensor([[10.0, 10.0, 50.0, 50.0]]), + format="XYXY", + canvas_size=(256, 256), + ) + merged.label = torch.tensor([1], dtype=torch.long) + + with patch( + "getitune.backend.openvino.models.detection.DetectionTileMerge", + ) as mock_merge_cls: + mock_merger = mock_merge_cls.return_value + mock_merger.merge.return_value = [merged] + result = detection_model.forward_tiles(tile_batch) + + # The merger was constructed with original image infos, num_classes and tile_config. + mock_merge_cls.assert_called_once_with( + tile_batch.imgs_info, + detection_model._label_info.num_classes, + detection_model.tile_config, + ) + # Per-tile inference ran once for the single unbound tile batch. + detection_model.forward.assert_called_once() + # merge() received the collected per-tile predictions and infos. + merge_args = mock_merger.merge.call_args.args + assert merge_args[0] == [per_tile_pred] + + # The returned batch contains the merged full-image prediction. + assert isinstance(result, PredictionBatch) + torch.testing.assert_close(result.bboxes[0].data, merged.bboxes.data) # pyrefly: ignore[unsupported-operation] + torch.testing.assert_close(result.labels[0], merged.label) # pyrefly: ignore[unsupported-operation] + torch.testing.assert_close(result.scores[0], merged.scores) # pyrefly: ignore[unsupported-operation] + + def test_test_step_routes_tile_batch_to_forward_tiles(self, detection_model): + """test_step must dispatch TileBatchData inputs to forward_tiles, not the plain forward path.""" + tile_batch = self._make_tile_batch() + merged_preds = PredictionBatch( + images=[torch.empty(3, 256, 256)], + imgs_info=[tile_batch.imgs_info[0]], + scores=[torch.tensor([0.9])], + bboxes=[ + tv_tensors.BoundingBoxes( # pyrefly: ignore[no-matching-overload] + torch.tensor([[1.0, 2.0, 3.0, 4.0]]), format="XYXY", canvas_size=(256, 256) + ) + ], + labels=[torch.tensor([0], dtype=torch.long)], + ) + detection_model.forward_tiles = MagicMock(return_value=merged_preds) + detection_model.forward = MagicMock() + metric = MagicMock() + + detection_model.test_step(tile_batch, metric) + + detection_model.forward_tiles.assert_called_once_with(tile_batch) + detection_model.forward.assert_not_called() + metric.update.assert_called_once() diff --git a/library/tests/unit/backend/openvino/test_engine.py b/library/tests/unit/backend/openvino/test_engine.py index 21045057667..6f494341411 100644 --- a/library/tests/unit/backend/openvino/test_engine.py +++ b/library/tests/unit/backend/openvino/test_engine.py @@ -179,6 +179,39 @@ def test_predict(self, fxt_engine, tmp_path, explain, mocker: MockerFixture) -> ): fxt_engine.predict(checkpoint=checkpoint) + def test_predict_propagates_tile_config(self, fxt_engine, mocker: MockerFixture) -> None: + """predict() must propagate datamodule.tile_config to the model, like test() does. + + Without this, tiled predictions route through OVModel.forward_tiles using the model's + default tile_config, diverging from evaluation (test) for tile-merge parameters. + """ + from getitune.config.data import TileConfig + + mocker.patch( + "getitune.backend.openvino.engine.AutoConfigurator.update_ov_subset_pipeline", + return_value=fxt_engine.datamodule, + ) + mock_model = MagicMock() + mock_model.label_info = fxt_engine.datamodule.label_info + mocker.patch("getitune.backend.openvino.engine.AutoConfigurator.get_ov_model", return_value=mock_model) + fxt_engine._model = mock_model + + # Attach a tiling configuration to the datamodule. + tile_config = TileConfig(enable_tiler=True, tile_size=(200, 200)) + fxt_engine.datamodule.tile_config = tile_config + + # Mock the dataloader to avoid actual data processing. + mock_dataloader = MagicMock() + mock_batch = MagicMock() + mock_dataloader.__iter__ = MagicMock(return_value=iter([mock_batch])) + mock_dataloader.__len__ = MagicMock(return_value=1) + mocker.patch.object(fxt_engine.datamodule, "test_dataloader", return_value=mock_dataloader) + + fxt_engine.predict() + + # The model's tile_config should now match the datamodule's tile_config. + assert mock_model.tile_config == tile_config + def test_predict_with_onnx_checkpoint(self, fxt_engine, mocker, fxt_onnx_model_path) -> None: """Test that predict accepts ONNX checkpoints.""" mocker.patch( diff --git a/library/tests/unit/data/augmentation/test_pipeline.py b/library/tests/unit/data/augmentation/test_pipeline.py index 0cfab8857f6..33266060c9d 100644 --- a/library/tests/unit/data/augmentation/test_pipeline.py +++ b/library/tests/unit/data/augmentation/test_pipeline.py @@ -1213,3 +1213,82 @@ def test_data_keys_per_task(self): for task_type, expected in expected_keys.items(): data_keys = ["input", *GPUAugmentationCallback._DATA_KEYS_BY_TASK.get(task_type, [])] assert data_keys == expected, f"Mismatch for {task_type}: {data_keys} != {expected}" + + def _make_detection_tile_batch(self): # noqa: ANN202 + """Build a minimal TileBatchDetDataEntity with two samples of tiles.""" + from getitune.data.entity.tile import TileBatchDetDataEntity + + def _tile() -> tv_tensors.Image: + return tv_tensors.Image(torch.ones(3, 8, 8)) + + # Two original images, each split into two tiles. + batch_tiles = [[_tile(), _tile()], [_tile(), _tile()]] + return TileBatchDetDataEntity( + batch_size=2, + batch_tiles=batch_tiles, + batch_tile_img_infos=[[], []], + batch_tile_tile_infos=[[], []], + imgs_info=[], + bboxes=[], + labels=[], + ) + + def test_apply_pipeline_normalizes_tile_batch(self): + """A TileBatchData validation batch is normalized per-tile without error.""" + from getitune.backend.lightning.callbacks.gpu_augmentation import GPUAugmentationCallback + from getitune.types.task import TaskType + + val_config = SubsetConfig( + augmentations_gpu=[ + { + "class_path": "kornia.augmentation.Normalize", + "init_args": {"mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5]}, + }, + ], + ) + callback = GPUAugmentationCallback(val_config=val_config) + + pl_module = MagicMock() + pl_module.task = TaskType.DETECTION + pl_module.data_input_params = MagicMock() + pl_module.data_input_params.mean = None + pl_module.data_input_params.std = None + callback.setup(MagicMock(), pl_module, stage="fit") + + tile_batch = self._make_detection_tile_batch() + + # Should not raise AttributeError on ``batch.images`` and should normalize + # every tile in place: (1.0 - 0.5) / 0.5 == 1.0 stays 1.0, so use a value + # that changes to prove the transform ran. + for tiles in tile_batch.batch_tiles: + for i in range(len(tiles)): + tiles[i] = tv_tensors.Image(torch.full((3, 8, 8), 0.75)) + + callback.on_validation_batch_start(MagicMock(), pl_module, tile_batch, batch_idx=0) + + # Normalizing 0.75 with mean 0.5 and std 0.5 yields 0.5 for every tile. + for tiles in tile_batch.batch_tiles: + for tile in tiles: + assert isinstance(tile, tv_tensors.Image) + torch.testing.assert_close(tile, tv_tensors.Image(torch.full((3, 8, 8), 0.5))) + + def test_apply_pipeline_tile_batch_no_pipeline_augs(self): + """A TileBatchData routed through an empty pipeline is left unchanged (no crash).""" + from getitune.backend.lightning.callbacks.gpu_augmentation import GPUAugmentationCallback + from getitune.types.task import TaskType + + callback = GPUAugmentationCallback(val_config=SubsetConfig(augmentations_gpu=[])) + pl_module = MagicMock() + pl_module.task = TaskType.DETECTION + pl_module.data_input_params = MagicMock() + pl_module.data_input_params.mean = None + pl_module.data_input_params.std = None + callback.setup(MagicMock(), pl_module, stage="fit") + + tile_batch = self._make_detection_tile_batch() + # Must not raise 'TileBatchDetDataEntity object has no attribute images' + callback.on_validation_batch_start(MagicMock(), pl_module, tile_batch, batch_idx=0) + + for tiles in tile_batch.batch_tiles: + for tile in tiles: + torch.testing.assert_close(tile, tv_tensors.Image(torch.ones(3, 8, 8)))