Add new datasets to benchmark. Add Kaggle support

.github/workflows/lib-lint-and-test.yaml

@@ -202,6 +202,13 @@ jobs:
       - check-paths
     if: needs.check-paths.outputs.run_benchmark_dataset_scripts == 'true'
     timeout-minutes: 30
+    # Kaggle API token for the (optional, credential-gated) real-download
+    # test in test_prepare_brain_tumor.py. Only set for same-repo runs —
+    # GitHub never exposes secrets to `pull_request` workflows triggered from
+    # forks, so that one test skips itself gracefully (via a `skipif` check)
+    # rather than failing on external contributors' PRs.
+    env:
+      KAGGLE_API_TOKEN: ${{ secrets.KAGGLE_API_TOKEN }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -224,7 +231,7 @@ jobs:
         working-directory: library
         env:
           GIT_LFS_SKIP_SMUDGE: "1"
-        run: just venv
+        run: just venv-benchmark
 
       - name: Run benchmark dataset script tests
         working-directory: library

library/Justfile

@@ -20,6 +20,13 @@ venv device="cpu": _ensure_uv
     uv lock --check
     uv sync --frozen --extra {{ device }} --extra ultralytics
 
+# Create a virtual environment including benchmark-dataset dependencies (e.g. kagglehub).
+# Use this when running or testing the benchmark dataset preparation scripts.
+[arg("device", short="d", long="device")]
+venv-benchmark device="cpu": _ensure_uv
+    uv lock --check
+    uv sync --frozen --extra {{ device }} --extra ultralytics --extra benchmark
+
 # Generate or update the uv lock file
 venv-lock: _ensure_uv
     uv lock

library/benchmark_catalog.yaml

@@ -10,36 +10,107 @@
 #
 # Fields:
 #   name        – unique identifier (referenced from benchmark_manifest.yaml)
-#   script      – path to preparation script (relative to library/ root)
+#   script      – path to preparation script (relative to library/ root).
+#                 Mutually exclusive with `local_path` — exactly one must be set.
+#   local_path  – path to an already-prepared dataset directory, used as-is
+#                 with no script execution. Supports ${VAR}/~ expansion so the
+#                 same entry resolves differently per machine/CI runner (e.g.
+#                 "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/my_dataset"). Use this
+#                 for datasets that were prepared out-of-band (manually, or on
+#                 a shared/network mount) and should never be re-downloaded.
+#   raw_dir     – optional, only valid together with `script`. Path (also
+#                 supports ${VAR}/~ expansion) to a pre-fetched raw archive or
+#                 directory, forwarded to the script as `--raw-dir` so it can
+#                 skip its own network download step (e.g. for credentialed
+#                 sources like Kaggle) while still running the same
+#                 transform/export logic. If the path doesn't exist, the
+#                 script falls back to its normal download behavior.
 #   size_tier   – tiny | small | medium | large
+#                 rough dataset size.
+#   data_group  – weekly | extended | all (default: all, may be omitted)
+#                 `weekly` datasets are only included in weekly-benchmark runs;
+#                 `extended` datasets are only included in extended/full runs;
+#                 `all` (the default) is included in both.
 #   url         – upstream source the preparation script downloads from
+#
+# See src/getitune/benchmark/README.md ("Datasets requiring credentials or
+# manual placement") for a walkthrough of `local_path` / `raw_dir` usage.
 
 version: 1
 
 datasets:
   - name: wgisd
     script: "scripts/benchmark_datasets/prepare_wgisd.py"
     size_tier: small
+    data_group: weekly
     url: "https://github.com/thsant/wgisd"
     description: "Wine Grape Instance Segmentation Dataset — 137 images of grape clusters with COCO-style polygon annotations (5 varieties)."
     compatible_tasks:
       - instance_segmentation
       - detection # polygons → bounding boxes
       - rotated_detection # polygons → oriented bounding boxes
       - semantic_segmentation # polygons → pixel masks
-      - classification/multi_class_cls # crop-level or image-level class from category
+      - classification/multi_label_cls # crop-level or image-level class from category
 
   - name: flowers102
     script: "scripts/benchmark_datasets/prepare_flowers102.py"
     size_tier: large
+    data_group: extended
     url: "https://www.robots.ox.ac.uk/~vgg/data/flowers/102/"
     description: "Oxford Flowers 102 — 8,189 images across 102 flower categories with official train/val/test splits."
     compatible_tasks:
       - classification/multi_class_cls
 
+  - name: paddy_disease
+    script: "scripts/benchmark_datasets/prepare_paddy_disease.py"
+    size_tier: small
+    data_group: weekly
+    url: "https://huggingface.co/datasets/anthony2261/paddy-disease-classification"
+    description: "Paddy Disease Classification — public HF mirror of a rice disease dataset with image-level class labels."
+    compatible_tasks:
+      - classification/multi_class_cls
+
+  - name: chest_ct
+    script: "scripts/benchmark_datasets/prepare_chest_ct.py"
+    size_tier: large
+    data_group: extended
+    url: "https://github.com/shreyabandyopadhyay/CTScanImageClassification/blob/main/DS3.zip"
+    description: "Chest CT Scan Images — public 3-class (abdomen/chest/head) body-part CT classification dataset."
+    compatible_tasks:
+      - classification/multi_class_cls
+
+  - name: aircraft
+    script: "scripts/benchmark_datasets/prepare_aircraft.py"
+    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/aircraft_raw"
+    size_tier: large
+    data_group: weekly
+    url: "https://www.kaggle.com/datasets/seryouxblaster764/fgvc-aircraft"
+    description: >-
+      FGVC-Aircraft — 10,200 aircraft images with official train/val/test splits and
+      hierarchical labels (variant / family / manufacturer). This benchmark uses the
+      variant level and requires a Kaggle account for automatic download, or a
+      pre-extracted raw directory at $GETITUNE_BENCHMARK_EXTERNAL_DATA/aircraft_raw.
+    compatible_tasks:
+      - classification/multi_class_cls
+
+  - name: food41
+    script: "scripts/benchmark_datasets/prepare_food41.py"
+    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/food41_raw"
+    size_tier: large
+    data_group: extended
+    url: "https://www.kaggle.com/datasets/kmader/food41"
+    description: >-
+      Food-101 — 101 food categories with official train/test splits, sourced from
+      a Kaggle mirror. This benchmark uses a stratified validation split from the
+      training set and supports Kaggle auto-download or a pre-extracted raw
+      directory at $GETITUNE_BENCHMARK_EXTERNAL_DATA/food41_raw.
+    compatible_tasks:
+      - classification/multi_class_cls
+
   - name: oxford_pets_hlabel
     script: "scripts/benchmark_datasets/prepare_oxford_pets_hlabel.py"
     size_tier: large
+    data_group: extended
     url: "https://www.robots.ox.ac.uk/~vgg/data/pets/"
     description: "Oxford-IIIT Pet with hierarchical labels — 7,349 images with natural 2-level hierarchy: species (cat/dog) → 37 breeds."
     compatible_tasks:
@@ -48,6 +119,7 @@ datasets:
   - name: oxford_pets
     script: "scripts/benchmark_datasets/prepare_oxford_pets.py"
     size_tier: large
+    data_group: extended
     url: "https://www.robots.ox.ac.uk/~vgg/data/pets/"
     description: "Oxford-IIIT Pet — 7,349 images of 37 pet breeds with pixel-level trimap segmentation masks."
     compatible_tasks:
@@ -58,6 +130,7 @@ datasets:
   - name: bccd
     script: "scripts/benchmark_datasets/prepare_bccd.py"
     size_tier: small
+    data_group: weekly
     url: "https://github.com/Shenggan/BCCD_Dataset"
     description: "Blood Cell Count and Detection — 364 microscopy blood-smear images with Pascal VOC bounding-box annotations for 3 cell types (RBC, WBC, Platelets)."
     compatible_tasks:
@@ -66,14 +139,16 @@ datasets:
   - name: aid_multilabel
     script: "scripts/benchmark_datasets/prepare_aid_multilabel.py"
     size_tier: medium
+    data_group: weekly
     url: "https://huggingface.co/datasets/jonathan-roberts1/AID_MultiLabel"
     description: "AID Multi-Label — 3,000 overhead aerial-scene images (600×600) with multi-label annotations across 17 co-occurring land-cover classes."
     compatible_tasks:
       - classification/multi_label_cls
 
   - name: axial_mri
     script: "scripts/benchmark_datasets/prepare_axial_mri.py"
-    size_tier: tiny
+    size_tier: small
+    data_group: extended
     url: "https://huggingface.co/datasets/Francesco/axial-mri"
     description: "Axial MRI (Roboflow-100) — 371 brain axial-MRI slices with tumour bounding-box annotations (negative/positive). Tiny, privacy-safe medical detection benchmark."
     compatible_tasks:
@@ -82,6 +157,7 @@ datasets:
   - name: cable_damage
     script: "scripts/benchmark_datasets/prepare_cable_damage.py"
     size_tier: medium
+    data_group: extended
     url: "https://huggingface.co/datasets/Francesco/cable-damage"
     description: "Cable Damage (Roboflow-100) — 1,318 close-up cable images with damage bounding-box annotations (break/thunderbolt). Industrial damage-inspection detection benchmark."
     compatible_tasks:
@@ -90,7 +166,95 @@ datasets:
   - name: vehicles
     script: "scripts/benchmark_datasets/prepare_vehicles.py"
     size_tier: large
+    data_group: extended
     url: "https://huggingface.co/datasets/Francesco/vehicles-q0x2v"
     description: "Vehicles (Roboflow-100) — 4,058 road-scene images with bounding-box annotations across 12 vehicle types (cars, buses, trucks). Large multi-class on-road detection benchmark."
     compatible_tasks:
       - detection
+
+  - name: pothole
+    script: "scripts/benchmark_datasets/prepare_pothole.py"
+    size_tier: tiny
+    data_group: weekly
+    url: "https://huggingface.co/datasets/keremberke/pothole-segmentation"
+    description: "Pothole Segmentation — 90 road-surface images with polygon annotations for potholes. Tiny real-world segmentation benchmark."
+    compatible_tasks:
+      - instance_segmentation
+      - semantic_segmentation
+      - detection # polygons -> bounding boxes
+
+  - name: pcb_defect
+    script: "scripts/benchmark_datasets/prepare_pcb_defect.py"
+    size_tier: small
+    data_group: weekly
+    url: "https://huggingface.co/datasets/keremberke/pcb-defect-segmentation"
+    description: "PCB Defect Segmentation — 189 close-up PCB images with polygon annotations across 4 defect classes. Small industrial segmentation benchmark."
+    compatible_tasks:
+      - instance_segmentation
+      - semantic_segmentation
+      - detection # polygons -> bounding boxes
+
+  - name: satellite_building
+    script: "scripts/benchmark_datasets/prepare_satellite_building.py"
+    size_tier: large
+    data_group: extended
+    url: "https://huggingface.co/datasets/keremberke/satellite-building-segmentation"
+    description: "Satellite Building Segmentation — 9,665 aerial images with polygon annotations for building footprints. Medium-scale segmentation benchmark."
+    compatible_tasks:
+      - instance_segmentation
+      - semantic_segmentation
+      - detection # polygons -> bounding boxes
+
+  - name: corrosion
+    script: "scripts/benchmark_datasets/prepare_corrosion.py"
+    size_tier: tiny
+    data_group: weekly
+    url: "https://huggingface.co/datasets/rkumari/corrosion_segment"
+    description: "Corrosion segmentation — 44 images with per-pixel severity masks across 4 classes. Tiny semantic-segmentation benchmark."
+    compatible_tasks:
+      - semantic_segmentation
+
+  - name: crack_semantic
+    script: "scripts/benchmark_datasets/prepare_crack_semantic.py"
+    size_tier: large
+    data_group: weekly
+    url: "https://huggingface.co/datasets/varcoder/crack-segmentation-dataset"
+    description: "Crack segmentation — 11,298 images with binary crack masks. Large semantic-segmentation benchmark."
+    compatible_tasks:
+      - semantic_segmentation
+
+  - name: brain_tumor
+    script: "scripts/benchmark_datasets/prepare_brain_tumor.py"
+    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/brain_tumor_raw"
+    size_tier: large
+    data_group: extended
+    url: "https://www.kaggle.com/datasets/pkdarabi/medical-image-dataset-brain-tumor-segmentation"
+    description: >-
+      Brain Tumor instance segmentation — ~1,744 MRI images with polygon annotations
+      (3 tumor-likelihood classes) from a Kaggle-gated Roboflow YOLO export. Requires a
+      Kaggle account: either configure Kaggle API credentials (see the preparation
+      script's docstring) for automatic download, or place a pre-fetched copy at
+      $GETITUNE_BENCHMARK_EXTERNAL_DATA/brain_tumor_raw (raw_dir is forwarded to
+      the script and used instead of downloading when present).
+    compatible_tasks:
+      - instance_segmentation
+      - detection # polygons → bounding boxes
+      - semantic_segmentation # polygons → pixel masks
+
+  - name: cardd
+    script: "scripts/benchmark_datasets/prepare_cardd.py"
+    raw_dir: "${GETITUNE_BENCHMARK_EXTERNAL_DATA}/cardd_instseg_raw"
+    size_tier: large
+    data_group: extended
+    url: "https://www.kaggle.com/datasets/issamjebnouni/cardd"
+    description: >-
+      CarDD instance segmentation — 4,000 images with COCO polygon annotations across 6
+      damage classes (dent, scratch, crack, glass shatter, lamp broken, tire flat), from
+      a Kaggle-gated COCO-format mirror. Requires a Kaggle account: either configure
+      Kaggle API credentials (see the preparation script's docstring) for automatic
+      download, or place a pre-fetched copy at $GETITUNE_BENCHMARK_EXTERNAL_DATA/cardd_instseg_raw
+      (raw_dir is forwarded to the script and used instead of downloading when present).
+    compatible_tasks:
+      - instance_segmentation
+      - detection # polygons → bounding boxes
+      - semantic_segmentation # polygons → pixel masks