Add multimodal benchmark evaluation with lmms-eval

docs/source/how-to/configure-workflows/metrics-configuration.md

@@ -281,3 +281,93 @@ Olive provides three built-in accuracy sub-types for evaluating vision/multimoda
 - Multiple valid answers (lists) are joined with `|` and the metric matches against any valid answer.
 - For ONNX models, provide a custom pre-process that applies the processor/tokenizer to produce numeric tensors.
 ```
+
+## Standard Multimodal Benchmarks with lmms-eval
+
+Use `LMMSEvaluator` when a public image or audio generation benchmark is available
+in [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval). It delegates task
+loading, prompting, and scoring to lmms-eval so results use the benchmark's
+declared protocol and metric, such as DocVQA ANLS, TextVQA accuracy, WER, or
+BLEU.
+
+Olive's existing `OnnxEvaluator` and custom evaluator support remain the
+appropriate choices for proprietary tasks, lightweight smoke tests, and
+workflows that should not depend on lmms-eval. `LMMSEvaluator` does not
+deprecate those paths.
+
+Install Olive and the pinned upstream lmms-eval dependency before using this
+evaluator:
+
+```bash
+pip install olive-ai
+pip install \
+  "lmms-eval[audio,metrics] @ git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git@3e675904f8cba6793de12b91979b04d91754bdf3"
+```
+
+Olive cannot publish a package extra containing a direct Git dependency. The
+installation command therefore pins the upstream lmms-eval commit containing
+[the wheel package-data fix](https://github.com/EvolvingLMMs-Lab/lmms-eval/pull/1390).
+The published `0.7.2` wheel omits extensionless task templates and cannot load
+its default task registry. A normal `olive-ai[lmms-eval]` extra can replace this
+command after upstream publishes a release containing that fix.
+
+For an ONNX input, also install the ORT-GenAI package for the target provider,
+such as `onnxruntime-genai` or `onnxruntime-genai-cuda`.
+
+`LMMSEvaluator` accepts:
+
+- an `HfModelHandler`, dispatched to an upstream lmms-eval model wrapper; or
+- an `ONNXModelHandler` that represents a complete ORT-GenAI package containing
+  `genai_config.json` and all referenced model components.
+
+`MobiusBuilder` returns a `CompositeModelHandler`. Add
+`CompositeToOnnxPackage` after it to preserve the package layout while exposing
+an `ONNXModelHandler` to the evaluator:
+
+```json
+{
+    "passes": {
+        "build": {
+            "type": "MobiusBuilder"
+        },
+        "as_onnx_package": {
+            "type": "CompositeToOnnxPackage"
+        }
+    },
+    "evaluators": {
+        "multimodal_benchmarks": {
+            "type": "LMMSEvaluator",
+            "tasks": ["ai2d"],
+            "batch_size": 1,
+            "log_samples": true,
+            "output_path": "results/lmms_eval.json"
+        }
+    },
+    "evaluator": "multimodal_benchmarks"
+}
+```
+
+Use `include_path` with one directory or a list of directories to load custom
+lmms-eval tasks. Custom task names must not collide with built-in task names.
+Public benchmark fixes should be contributed to lmms-eval rather than hidden
+behind a colliding local task.
+
+The following limitations apply:
+
+- Raw single-file ONNX models are not supported because they do not contain the
+  ORT-GenAI multimodal preprocessing pipeline.
+- The ORT-GenAI adapter processes requests individually. It does not implement
+  batched generation, video, multi-round or interleaved generation, beam
+  search, multiple return sequences, or loglikelihood tasks.
+- The adapter is registered through lmms-eval's legacy model registry for
+  in-process use by Olive. It is not an lmms-eval command-line entry point.
+- Automatic Hugging Face dispatch is limited to model wrappers in the pinned
+  upstream lmms-eval release. Set `model_class` explicitly for another wrapper
+  that is registered in the installed lmms-eval environment.
+- `HfModelHandler.adapter_path` and handler `load_kwargs` are rejected rather
+  than silently ignored. Merge an adapter into the checkpoint first, and pass
+  wrapper-supported constructor options through `hf_model_kwargs`.
+
+`image_serialization_profile` accepts `lossless` (PNG, the default) or
+`jpeg85`. ONNX audio uses the sample rates declared by the ORT-GenAI package;
+`audio_target_sample_rate` is an explicit host-resampling override.

olive/cache.py

@@ -22,6 +22,7 @@
 from olive.common.container_client_factory import AzureContainerClientFactory
 from olive.common.utils import hardlink_copy_file, hash_dict, hf_repo_exists, set_nested_dict_value
 from olive.model.config.model_config import ModelConfig
+from olive.model.utils.onnx_utils import get_onnx_file_path
 from olive.resource_path import ResourcePath, create_resource_path, find_all_resources
 
 if TYPE_CHECKING:
@@ -612,6 +613,54 @@ def _save_model(
             model_path_resource = model_json["config"]["model_path"]
             source_path = Path(model_path_resource.get_path())
             onnx_file_name = model_json["config"].get("onnx_file_name")
+            model_attributes = model_json["config"].get("model_attributes") or {}
+
+            package_root = source_path if source_path.is_dir() else source_path.parent
+            for _ in range(3):
+                if (package_root / "genai_config.json").is_file():
+                    break
+                if package_root.parent == package_root:
+                    package_root = None
+                    break
+                package_root = package_root.parent
+            else:
+                package_root = None
+
+            if package_root is not None or model_attributes.get("ort_genai_package"):
+                if package_root is None:
+                    raise ValueError(f"ORT-GenAI package has no discoverable genai_config.json: {source_path}")
+                if output_dir.suffix == ".onnx":
+                    raise ValueError("ORT-GenAI packages must be saved to a directory, not an ONNX file path.")
+                package_root = package_root.resolve()
+                resolved_output_dir = output_dir.resolve()
+                if (
+                    package_root == resolved_output_dir
+                    or package_root.is_relative_to(resolved_output_dir)
+                    or resolved_output_dir.is_relative_to(package_root)
+                ):
+                    raise ValueError(
+                        "ORT-GenAI package source and output directories must not overlap: "
+                        f"source={package_root}, output={resolved_output_dir}."
+                    )
+                entry_path = Path(get_onnx_file_path(str(source_path), onnx_file_name)).resolve()
+                if not entry_path.is_relative_to(package_root) or not entry_path.is_file():
+                    raise ValueError(f"ORT-GenAI package entry point is invalid: {entry_path}")
+                if resolved_output_dir.exists() and any(resolved_output_dir.iterdir()) and not overwrite:
+                    raise FileExistsError(f"Output directory is not empty: {resolved_output_dir}")
+                resolved_output_dir.mkdir(parents=True, exist_ok=True)
+                if overwrite:
+                    # Replace package-owned entries without deleting sibling run artifacts such as
+                    # the reference model saved by the CLI's --test workflow.
+                    for source_item in package_root.iterdir():
+                        output_item = resolved_output_dir / source_item.name
+                        if output_item.is_dir() and not output_item.is_symlink():
+                            shutil.rmtree(output_item)
+                        elif output_item.exists() or output_item.is_symlink():
+                            output_item.unlink()
+                shutil.copytree(package_root, resolved_output_dir, dirs_exist_ok=True)
+                model_json["config"]["model_path"] = str(resolved_output_dir)
+                model_json["config"]["onnx_file_name"] = entry_path.relative_to(package_root).as_posix()
+                return self._save_additional_files(model_json, resolved_output_dir)
 
             # Determine if source has external data or additional files
             has_additional_files = bool(onnx_file_name)