From af1c32c995ccb3e03b9b119089180803c82a4ad6 Mon Sep 17 00:00:00 2001 From: kprokofi Date: Fri, 3 Jul 2026 21:25:45 +0900 Subject: [PATCH 1/3] update docs --- README.md | 8 +- library/README.md | 2 +- library/docs/Makefile | 31 - library/docs/README.md | 21 - .../design/automated-model-benchmarking.md | 1780 ----------------- library/docs/make.bat | 44 - library/docs/source/_static/css/custom.css | 211 -- .../docs/source/_static/logos/geti-header.png | Bin 210812 -> 0 bytes .../docs/source/_static/logos/github_icon.png | Bin 2161 -> 0 bytes .../redirects/guide-homepage-redirect.html | 4 - library/docs/source/conf.py | 150 -- .../additional_features/adaptive_training.rst | 121 -- .../class_incremental_sampler.rst | 58 - .../configurable_input_size.rst | 118 -- .../additional_features/fast_data_loading.rst | 38 - .../explanation/additional_features/index.rst | 16 - .../models_optimization.rst | 26 - .../additional_features/tiling.rst | 217 -- .../explanation/additional_features/xai.rst | 130 -- .../hierarhical_classification.rst | 58 - .../algorithms/classification/index.rst | 10 - .../multi_class_classification.rst | 88 - .../multi_label_classification.rst | 39 - .../guide/explanation/algorithms/index.rst | 27 - .../algorithms/object_detection/index.rst | 8 - .../object_detection/object_detection.rst | 157 -- .../algorithms/segmentation/index.rst | 9 - .../segmentation/instance_segmentation.rst | 81 - .../segmentation/semantic_segmentation.rst | 95 - .../source/guide/get_started/api_tutorial.rst | 650 ------ .../source/guide/get_started/cli_commands.rst | 503 ----- .../source/guide/get_started/installation.rst | 190 -- .../source/guide/get_started/introduction.rst | 146 -- library/docs/source/guide/index.rst | 43 - library/docs/source/guide/reference/index.rst | 11 - .../docs/source/guide/release_notes/index.rst | 639 ------ .../tutorials/advanced/configuration.rst | 228 --- .../advanced/hier_metric_collection.rst | 104 - .../source/guide/tutorials/advanced/index.rst | 15 - .../guide/tutorials/advanced/multi_gpu.rst | 51 - .../source/guide/tutorials/advanced/peft.rst | 107 - .../tutorials/advanced/torch_compile.rst | 39 - .../advanced/tree_path_kl_loss_hcls.rst | 170 -- .../source/guide/tutorials/base/explain.rst | 140 -- .../source/guide/tutorials/base/export.rst | 284 --- .../base/how_to_train/classification.rst | 388 ---- .../tutorials/base/how_to_train/detection.rst | 415 ---- .../tutorials/base/how_to_train/index.rst | 44 - .../how_to_train/instance_segmentation.rst | 419 ---- .../how_to_train/semantic_segmentation.rst | 340 ---- .../source/guide/tutorials/base/index.rst | 34 - library/docs/source/index.rst | 15 - library/docs/utils/auto_sphinx_build.py | 59 - library/docs/utils/images/anomaly_tasks.png | Bin 633006 -> 0 bytes library/docs/utils/images/auto_config.png | Bin 1458705 -> 0 bytes library/docs/utils/images/cli.png | Bin 1353775 -> 0 bytes .../docs/utils/images/diagram_getitune.png | Bin 1445278 -> 0 bytes .../docs/utils/images/dota_tiling_example.jpg | Bin 1233988 -> 0 bytes library/docs/utils/images/explain_wgisd.png | Bin 3516345 -> 0 bytes library/docs/utils/images/flowers_example.jpg | Bin 151293 -> 0 bytes .../utils/images/instance_seg_example.png | Bin 1327347 -> 0 bytes library/docs/utils/images/label_tree.png | Bin 331910 -> 0 bytes library/docs/utils/images/padim.png | Bin 76034 -> 0 bytes .../images/product_design/engine_diagram.png | Bin 231650 -> 0 bytes .../product_design/getitune_workflow.png | Bin 111527 -> 0 bytes .../images/product_design/reuse_model.png | Bin 270631 -> 0 bytes .../support_various_data_format.png | Bin 427365 -> 0 bytes .../images/product_design/task_data_model.png | Bin 393940 -> 0 bytes .../utils/images/semantic_seg_example.png | Bin 175011 -> 0 bytes library/docs/utils/images/stfpm.png | Bin 1046950 -> 0 bytes library/docs/utils/images/voc_example.png | Bin 447527 -> 0 bytes .../utils/images/wgisd_dataset_sample.jpg | Bin 156070 -> 0 bytes library/docs/utils/images/wgisd_gt_sample.jpg | Bin 3654161 -> 0 bytes library/docs/utils/images/wgisd_pr_sample.jpg | Bin 297193 -> 0 bytes library/docs/utils/images/xai_cls.jpg | Bin 251865 -> 0 bytes library/docs/utils/images/xai_det.jpg | Bin 171342 -> 0 bytes library/docs/utils/images/xai_example.jpg | Bin 80216 -> 0 bytes 77 files changed, 5 insertions(+), 8576 deletions(-) delete mode 100644 library/docs/Makefile delete mode 100644 library/docs/README.md delete mode 100644 library/docs/design/automated-model-benchmarking.md delete mode 100644 library/docs/make.bat delete mode 100644 library/docs/source/_static/css/custom.css delete mode 100644 library/docs/source/_static/logos/geti-header.png delete mode 100644 library/docs/source/_static/logos/github_icon.png delete mode 100644 library/docs/source/_static/redirects/guide-homepage-redirect.html delete mode 100644 library/docs/source/conf.py delete mode 100644 library/docs/source/guide/explanation/additional_features/adaptive_training.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/class_incremental_sampler.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/configurable_input_size.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/fast_data_loading.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/index.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/models_optimization.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/tiling.rst delete mode 100644 library/docs/source/guide/explanation/additional_features/xai.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/classification/hierarhical_classification.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/classification/index.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/classification/multi_class_classification.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/classification/multi_label_classification.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/index.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/object_detection/index.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/object_detection/object_detection.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/segmentation/index.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/segmentation/instance_segmentation.rst delete mode 100644 library/docs/source/guide/explanation/algorithms/segmentation/semantic_segmentation.rst delete mode 100644 library/docs/source/guide/get_started/api_tutorial.rst delete mode 100644 library/docs/source/guide/get_started/cli_commands.rst delete mode 100644 library/docs/source/guide/get_started/installation.rst delete mode 100644 library/docs/source/guide/get_started/introduction.rst delete mode 100644 library/docs/source/guide/index.rst delete mode 100644 library/docs/source/guide/reference/index.rst delete mode 100644 library/docs/source/guide/release_notes/index.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/configuration.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/hier_metric_collection.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/index.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/multi_gpu.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/peft.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/torch_compile.rst delete mode 100644 library/docs/source/guide/tutorials/advanced/tree_path_kl_loss_hcls.rst delete mode 100644 library/docs/source/guide/tutorials/base/explain.rst delete mode 100644 library/docs/source/guide/tutorials/base/export.rst delete mode 100644 library/docs/source/guide/tutorials/base/how_to_train/classification.rst delete mode 100644 library/docs/source/guide/tutorials/base/how_to_train/detection.rst delete mode 100644 library/docs/source/guide/tutorials/base/how_to_train/index.rst delete mode 100644 library/docs/source/guide/tutorials/base/how_to_train/instance_segmentation.rst delete mode 100644 library/docs/source/guide/tutorials/base/how_to_train/semantic_segmentation.rst delete mode 100644 library/docs/source/guide/tutorials/base/index.rst delete mode 100644 library/docs/source/index.rst delete mode 100644 library/docs/utils/auto_sphinx_build.py delete mode 100644 library/docs/utils/images/anomaly_tasks.png delete mode 100644 library/docs/utils/images/auto_config.png delete mode 100644 library/docs/utils/images/cli.png delete mode 100644 library/docs/utils/images/diagram_getitune.png delete mode 100644 library/docs/utils/images/dota_tiling_example.jpg delete mode 100644 library/docs/utils/images/explain_wgisd.png delete mode 100644 library/docs/utils/images/flowers_example.jpg delete mode 100644 library/docs/utils/images/instance_seg_example.png delete mode 100644 library/docs/utils/images/label_tree.png delete mode 100644 library/docs/utils/images/padim.png delete mode 100644 library/docs/utils/images/product_design/engine_diagram.png delete mode 100644 library/docs/utils/images/product_design/getitune_workflow.png delete mode 100644 library/docs/utils/images/product_design/reuse_model.png delete mode 100644 library/docs/utils/images/product_design/support_various_data_format.png delete mode 100644 library/docs/utils/images/product_design/task_data_model.png delete mode 100644 library/docs/utils/images/semantic_seg_example.png delete mode 100644 library/docs/utils/images/stfpm.png delete mode 100644 library/docs/utils/images/voc_example.png delete mode 100644 library/docs/utils/images/wgisd_dataset_sample.jpg delete mode 100644 library/docs/utils/images/wgisd_gt_sample.jpg delete mode 100644 library/docs/utils/images/wgisd_pr_sample.jpg delete mode 100644 library/docs/utils/images/xai_cls.jpg delete mode 100644 library/docs/utils/images/xai_det.jpg delete mode 100644 library/docs/utils/images/xai_example.jpg diff --git a/README.md b/README.md index 94f06486bb9..ae549e20195 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [Quick Start](#quick-start) • [Geti™ documentation](https://docs.geti.intel.com/) • -[`getitune` documentation](library/README.md) +[`getitune` documentation](https://docs.geti.intel.com/docs/user-guide/library/) [![Container build](https://github.com/open-edge-platform/geti/actions/workflows/build.yaml/badge.svg)](https://github.com/open-edge-platform/geti/actions/workflows/build.yaml) [![Codecov](https://codecov.io/gh/open-edge-platform/geti/branch/develop/graph/badge.svg?token=9HVFNMPFGD)](https://codecov.io/gh/open-edge-platform/geti) @@ -180,7 +180,7 @@ uv pip install getitune # CPU-only by default > uv sync --extra cpu --extra ultralytics # CPU + YOLO > ``` > -> See the [library README](library/README.md#installation) for more details. +> See the [getitune documentation](https://docs.geti.intel.com/docs/user-guide/library/) for more details. > > ℹ️ Ultralytics YOLO models are distributed under the [AGPL-3.0 license](https://www.ultralytics.com/license). @@ -223,8 +223,8 @@ metrics = ov_engine.test() predictions = ov_engine.predict() ``` -See the [library README](library/README.md) for the full list of recipes, advanced configuration, dataset support, -backend-specific options, and deployment/optimization examples. +See the [getitune documentation](https://docs.geti.intel.com/docs/user-guide/library/) for the full list of recipes, +advanced configuration, dataset support, backend-specific options, and deployment/optimization examples. ## Key Features diff --git a/library/README.md b/library/README.md index 6e9b9fa8bf1..0d09dcd5c69 100644 --- a/library/README.md +++ b/library/README.md @@ -10,7 +10,7 @@ [Supported Tasks & Models](#supported-tasks--models) • [Installation](#installation) • [Quick Start](#quick-start) • -[Docs](https://open-edge-platform.github.io/geti/latest/index.html) • +[Docs](https://docs.geti.intel.com/docs/user-guide/library/) • [License](#license) [![PyPI](https://img.shields.io/pypi/v/getitune)](https://pypi.org/project/getitune) diff --git a/library/docs/Makefile b/library/docs/Makefile deleted file mode 100644 index 2750a53f965..00000000000 --- a/library/docs/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -html: - @$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)"/html $(SPHINXOPTS) $(O) - - cp source/_static/redirects/guide-homepage-redirect.html "$(BUILDDIR)"/html/index.html - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -# Custom clean target that also removes autosummary generated files. Can -# be removed when https://github.com/sphinx-doc/sphinx/issues/1999 is fixed. -clean: - rm -rf "$(SOURCEDIR)/guide/reference/_autosummary" - $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/library/docs/README.md b/library/docs/README.md deleted file mode 100644 index af6a9a4e7ab..00000000000 --- a/library/docs/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# getitune Documentation - -## Introduction - -This is the source code for the getitune documentation. It is built using sphinx-design and myst parser. - -## Installation - -To install the dependencies, run the following command: - -```bash -pip install getitune[docs] -``` - -## Build - -To build the documentation, run the following command: - -```bash -sphinx-build -b html source build -``` diff --git a/library/docs/design/automated-model-benchmarking.md b/library/docs/design/automated-model-benchmarking.md deleted file mode 100644 index d1488e4fe5a..00000000000 --- a/library/docs/design/automated-model-benchmarking.md +++ /dev/null @@ -1,1780 +0,0 @@ -# Design Document: Automated Model Benchmarking Workflow - -| Field | Value | -| ----------- | ---------- | -| **Status** | Draft | -| **Date** | 2026-04-08 | -| **Authors** | Albert | - ---- - -## Table of Contents - -1. [Objective](#1-objective) -2. [Motivation & Background](#2-motivation--background) -3. [Current State Analysis](#3-current-state-analysis) -4. [Proposed Architecture](#4-proposed-architecture) -5. [Dataset Catalog](#5-dataset-catalog) -6. [Benchmark Manifest](#6-benchmark-manifest) -7. [Benchmark Runner](#7-benchmark-runner) -8. [Experiment Tracking with MLflow](#8-experiment-tracking-with-mlflow) -9. [Reporting & Regression Detection](#9-reporting--regression-detection) -10. [PR Change Detection & Targeted Benchmarks](#10-pr-change-detection--targeted-benchmarks) -11. [CI/CD Integration (GitHub Actions)](#11-cicd-integration-github-actions) -12. [Hardware & Accelerator Support](#12-hardware--accelerator-support) -13. [Failure Handling](#13-failure-handling) -14. [Directory & File Layout](#14-directory--file-layout) -15. [Migration from Existing Code](#15-migration-from-existing-code) -16. [Rollout Plan](#16-rollout-plan) -17. [Open Questions](#17-open-questions) - ---- - -## 1. Objective - -Set up a fully automated, CI-integrated workflow that evaluates the performance -of every supported OTX model on a curated set of testing datasets. The workflow -must produce comparable, versioned reports so that engineers can detect -regressions in accuracy, training time, and inference speed across releases, -branches, and hardware targets. - ---- - -## 2. Motivation & Background - -OTX is a framework for training vision models. Engineers continuously add new -architectures, loss functions, data augmentations, and training tricks. Every -change risks introducing regressions that are hard to detect during code -review alone — models must actually be trained and evaluated. - -Today this evaluation is **manual**: an engineer picks a model, picks a -dataset, trains locally, and eyeballs the numbers. This approach is: - -- **Time-consuming** — a full matrix of models × datasets × seeds takes days of - human effort. -- **Error-prone** — inconsistent hyperparameters, forgotten datasets, typos in - metric comparisons. -- **Non-reproducible** — no single source of truth for "what was the accuracy of - `yolox_s` on `pothole_tiny` last week?" -- **Opaque** — results live on individual engineers' machines and are not - accessible to the wider team. - -An automated benchmark workflow eliminates all four problems and lets -engineers focus on _building_ while the CI system takes care of _validating_. - ---- - -## 3. Current State Analysis - -The repository already contains two partially overlapping systems for model -evaluation. Both have significant limitations that motivate a clean redesign. - -### 3.1 `tests/perf_v2/` — Performance Benchmark Suite - -| Aspect | Description | -| -------------------- | ---------------------------------------------------------------------------------------------- | -| Entry point | `python -m tests.perf_v2.run --task DETECTION ...` or `python -m tests.perf_v2.benchmark ...` | -| Runner | Spawns one subprocess per `(model, dataset, seed)` triple via `subprocess.run` | -| Engine | Uses `OTXEngine` / `OVEngine` Python API directly | -| Reporting | Writes `benchmark.raw.csv` per experiment; `summary.py` aggregates into Excel/CSV pivot tables | -| Tracking | None — no MLflow, no DB, no dashboard | -| CI integration | None — no workflow file exists | -| Dataset provisioning | Assumes datasets are pre-downloaded into a `data/` root; no automated download | -| Failure handling | Retries up to 2 times; collects failures into `failed_jobs.json` | -| Accelerator support | `--device gpu\|xpu\|cpu` flag; single device assumption | - -**Key limitations:** - -- No CI workflow — the suite can only be run manually. -- Datasets must already exist on the machine; there is no download/provisioning step. -- Results are flat CSV files with no experiment-tracking UI, making cross-run comparison tedious. -- The `Criterion` checking is all-or-nothing assertions that raise on first violation; not suitable for a continuous monitoring system that should _report_ regressions, not just crash. -- The model/dataset/criteria registrations are scattered across `tasks/*.py` files with no single declarative manifest. -- No support for scenario variants (e.g. tiling, different hyperparameter profiles). - -### 3.2 `tests/regression/` — MLflow Regression Tests - -| Aspect | Description | -| -------------------- | -------------------------------------------------------------------------- | -| Entry point | `pytest tests/regression/test_regression.py` | -| Runner | pytest parametrize over model × dataset × seed | -| Engine | Shells out to `otx train` / `otx test` CLI via subprocess | -| Tracking | MLflow — creates experiment, logs metrics step-by-step, logs CSV artifacts | -| Reporting | Relies entirely on MLflow UI for visualization | -| Dataset provisioning | Same assumption — pre-existing `--data-root` | - -**Key limitations:** - -- Tightly coupled to pytest — parametrization, fixtures, and assertions are deeply intertwined. This makes it hard to run a single model ad-hoc, resume a partial run, or integrate with non-pytest CI systems. -- Test class hierarchy is copy-pasted per task; adding a new task means duplicating ~60 lines of boilerplate. -- MLflow integration is promising but incomplete (see `TODO` comments about `metrics.csv` not being produced correctly for the `test` subcommand). -- No reference comparison logic — metrics are logged but never compared against a baseline. -- No support for export/optimize evaluation or tiling variants. - -### 3.3 Assessment - -Neither system is complete enough to serve as the automated benchmark. Rather -than patching the gaps, we propose a unified redesign that takes the best ideas -from both (MLflow tracking from `regression/`, the `Benchmark` orchestration -from `perf_v2/`) while fixing the structural issues. - ---- - -## 4. Proposed Architecture - -``` -┌──────────────────────────────────────────────────────────────────────────┐ -│ GitHub Actions Workflow │ -│ │ -│ trigger: schedule (weekly) | workflow_dispatch | PR label │ -│ │ -│ ┌────────────────┐ ┌────────────────┐ ┌───────────────────────┐ │ -│ │ Provision │───▶│ Run │───▶│ Report & │ │ -│ │ Datasets │ │ Benchmarks │ │ Regression Check │ │ -│ └───────┬────────┘ └───────┬────────┘ └───────────┬───────────┘ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌────────────────┐ ┌────────────────┐ ┌───────────────────────┐ │ -│ │ Preparation │ │ Benchmark │ │ MLflow │ │ -│ │ Scripts │ │ Runner │ │ Tracking Server │ │ -│ │ (scripts/ │ │ (Python) │ │ (local or remote) │ │ -│ │ benchmark_ │ │ │ │ │ │ -│ │ datasets/) │ │ │ │ │ │ -│ └────────────────┘ └───────┬────────┘ └───────────┬───────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌────────────────┐ ┌───────────────────────┐ │ -│ │ OTXEngine │ │ Markdown / HTML │ │ -│ │ (train/test/ │ │ Summary Report │ │ -│ │ export/ │ │ (PR comment / │ │ -│ │ optimize) │ │ artifact / GH Pages) │ │ -│ └────────────────┘ └───────────────────────┘ │ -└──────────────────────────────────────────────────────────────────────────┘ -``` - -The system is composed of four independent layers: - -| Layer | Responsibility | Artifact | -| -------------------- | ----------------------------------------------------------- | -------------------------------------------------------- | -| **Dataset Catalog** | Declare datasets, their preparation scripts, and size class | `benchmark_catalog.yaml` + `scripts/benchmark_datasets/` | -| **Benchmark Runner** | Provision data via scripts, run experiments, log to MLflow | Python package `getitune.benchmark` | -| **Report Generator** | Compare against baselines, produce human-readable report | Markdown + CSV | -| **CI Workflow** | Orchestrate the above on GitHub Actions runners | `.github/workflows/benchmark.yml` | - -Each layer is independently testable and usable outside CI (e.g., an engineer -can run the benchmark runner locally with `python -m getitune.benchmark run ...`). - ---- - -## 5. Dataset Catalog - -### 5.1 Principles - -- Every dataset used for benchmarking is **declared in a single YAML manifest** - (`benchmark_catalog.yaml`) rather than scattered across Python files. -- Datasets are classified by **size tier**: `tiny`, `small`, `medium`, `large`. -- Each entry references a **preparation script** — a Python script that lives in - the repository under `scripts/benchmark_datasets/`. The script is responsible - for downloading the raw dataset from its source (e.g. a public URL, a - cloud bucket, or any other origin), applying any necessary transformations - (format conversion, annotation adjustments, subset sampling), and placing - the final dataset in the designated data directory. -- Datasets must have **verified, meaningful annotations**. Because scripts are - version-controlled, the exact transformation logic is reviewable and - reproducible. - -#### Size Tier Definitions - -| Tier | Item Count (images / samples) | Typical Use | -| ---------- | ----------------------------- | ---------------------------------------------------- | -| **tiny** | **< 50** | Unit-level smoke tests, PR checks, fast iteration | -| **small** | **50 – 200** | Nightly quick benchmarks, scenario sweeps | -| **medium** | **200 – 1 000** | Weekly full benchmarks, release validation | -| **large** | **> 1 000** | On-demand deep evaluation, publication-grade results | - -> **Guideline:** Every task should have at least one `tiny` and one `small` -> dataset so that PR-level and nightly benchmarks can run quickly. `medium` and -> `large` datasets are reserved for weekly and on-demand runs where statistical -> significance and real-world representativeness matter more than speed. - -### 5.2 Schema - -Datasets are declared as a **flat list** — each dataset is listed once and -referenced by name from the benchmark manifest. Since the same dataset can be -used by multiple tasks (e.g. a multi-annotation dataset used for detection, -segmentation, and classification), the catalog is **task-independent**. The -manifest (§6) declares which datasets each task uses. - -Each entry specifies a `script` field — the path (relative to the repository -root) to a Python preparation script. The script must accept two arguments: - -1. `--output-dir` — the directory where the final dataset should be placed. -2. `--name` — the dataset name (used by the script to determine its output - sub-directory within the output dir). - -```yaml -# benchmark_catalog.yaml -version: 1 - -datasets: - - name: pothole_tiny - script: "scripts/benchmark_datasets/prepare_pothole.py" - size_tier: tiny - - - name: wgisd_small - script: "scripts/benchmark_datasets/prepare_wgisd.py" - size_tier: small - - - name: diopsis_medium - script: "scripts/benchmark_datasets/prepare_diopsis.py" - size_tier: medium - - - name: visdrone_large - script: "scripts/benchmark_datasets/prepare_visdrone.py" - size_tier: large - - - name: pneumonia_tiny - script: "scripts/benchmark_datasets/prepare_pneumonia.py" - size_tier: tiny - # ... -``` - -### 5.3 Script-Based Provisioning & Cache - -A utility (`getitune.benchmark.catalog`) will: - -1. Read the catalog. -2. For each required dataset, check if the directory `/` - already exists. If it does, the dataset is considered ready and the - script is skipped. -3. If the directory does not exist, run the preparation script via - `python