[Model Manager]: Litellm enhancements

ai-services/Makefile

@@ -1,6 +1,6 @@
 REGISTRY ?= icr.io/ai-services-private
 IMAGE = ai-services
-TAG ?= v0.0.443
+TAG ?= v0.0.444
 CONTAINER_BUILDER ?= podman
 CREDS_ARG := $(if $(and $(REGISTRY_USER),$(REGISTRY_PASSWORD)),--creds="$(REGISTRY_USER):$(REGISTRY_PASSWORD)")
 

ai-services/assets/applications/rag-cpu/podman/values.yaml

@@ -104,7 +104,7 @@ litellm:
   # @description Enable LiteLLM proxy for remote LLM service. When true, uses LiteLLM with WatsonX; when false, uses local vLLM server. Default: false
   enable: false
   # @hidden
-  image: icr.io/ai-services-cicd/litellm:v1.101.2-0
+  image: icr.io/ai-services-cicd/litellm:v1.106.4-0
   # @description WatsonX API Key for LiteLLM integration. Required when litellm.enable is true.
   watsonxApiKey: ""
   # @description WatsonX Project ID for LiteLLM integration. Required when litellm.enable is true.

ai-services/assets/catalog/openshift/values.yaml

@@ -9,7 +9,7 @@ ui:
       cpu: "500m"
 
 backend:
-  image: icr.io/ai-services-cicd/ai-services:v0.0.443
+  image: icr.io/ai-services-cicd/ai-services:v0.0.444
   runtime: "openshift"
   adminPasswordHash: ""
   # @generate:password length=32, special=false

ai-services/assets/catalog/podman/values.yaml

@@ -4,7 +4,7 @@ ui:
 
 backend:
   port: ""
-  image: icr.io/ai-services-cicd/ai-services:v0.0.443
+  image: icr.io/ai-services-cicd/ai-services:v0.0.444
   runtime: ""
   adminPasswordHash: ""
   # @generate:password length=32, special=false

ai-services/assets/components/llm/watsonx/openshift/values.yaml

@@ -1,4 +1,4 @@
-image: "icr.io/ai-services-cicd/litellm:v1.101.2-0"
+image: "icr.io/ai-services-cicd/litellm:v1.106.4-0"
 model: "ibm/granite-4-h-small"
 watsonxApiKey: ""
 watsonxProjectId: ""

ai-services/assets/components/llm/watsonx/podman/values.yaml

@@ -1,4 +1,4 @@
-image: "icr.io/ai-services-cicd/litellm:v1.101.2-0"
+image: "icr.io/ai-services-cicd/litellm:v1.106.4-0"
 model: "ibm/granite-4-h-small"
 watsonxApiKey: ""
 watsonxProjectId: ""

ai-services/assets/worker/openshift/values.yaml

@@ -1,5 +1,5 @@
 worker:
-  image: icr.io/ai-services-cicd/ai-services:v0.0.443
+  image: icr.io/ai-services-cicd/ai-services:v0.0.444
   runtime: "openshift"
   token: ""
   gatewayAddr: ""

ai-services/assets/worker/podman/values.yaml

@@ -1,5 +1,5 @@
 worker:
-  image: icr.io/ai-services-cicd/ai-services:v0.0.443
+  image: icr.io/ai-services-cicd/ai-services:v0.0.444
   runtime: "podman"
   token: ""
   gatewayAddr: ""

images/litellm/Containerfile

@@ -1,87 +1,143 @@
-# Builder stage
-FROM registry.access.redhat.com/ubi9/ubi:9.8 as builder
+#### Base stage: shared build dependencies for the parallel stages below
+FROM registry.access.redhat.com/ubi9/ubi:9.8 as base
 
-ARG VERSION=${VERSION}
+ARG DNF_FLAGS="--disablerepo=* --enablerepo=ubi-9-baseos-rpms --enablerepo=ubi-9-appstream-rpms"
+ARG TARGETARCH
+ARG RUST_VERSION
+ARG PYTHON_VERSION
 
 # 1. Install heavy build dependencies (gcc, rust, etc.)
-RUN dnf -y update && \
-    dnf -y install \
-        wget \
+# Derive RPM_ARCH, NODE_ARCH and SKIP_CHECKSUM from the builder-provided TARGETARCH.
+# SKIP_CHECKSUM=1 for non-ppc64le arches since NODE_SHA256 is only set for ppc64le.
+RUN case "${TARGETARCH}" in \
+        x86_64|amd64)  RPM_ARCH=x86_64;  NODE_ARCH=x64;    SKIP_CHECKSUM=1; PROTOC_ARCH=x86_64    ;; \
+        aarch64|arm64) RPM_ARCH=aarch64; NODE_ARCH=arm64;   SKIP_CHECKSUM=1; PROTOC_ARCH=aarch_64  ;; \
+        ppc64le)       RPM_ARCH=ppc64le; NODE_ARCH=ppc64le; SKIP_CHECKSUM=0; PROTOC_ARCH=ppcle_64  ;; \
+        *)             RPM_ARCH="${TARGETARCH}"; NODE_ARCH="${TARGETARCH}"; SKIP_CHECKSUM=1; PROTOC_ARCH="${TARGETARCH}" ;; \
+    esac && \
+    echo "NODE_ARCH=${NODE_ARCH}" > /tmp/arch_env && \
+    echo "PROTOC_ARCH=${PROTOC_ARCH}" >> /tmp/arch_env && \
+    echo "SKIP_CHECKSUM=${SKIP_CHECKSUM}" >> /tmp/arch_env && \
+    dnf update ${DNF_FLAGS} -y && \
+    dnf install ${DNF_FLAGS} -y \
         patch \
-        python3.12 \
-        python3.12-pip \
-        python3.12-devel \
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-pip \
+        python${PYTHON_VERSION}-devel \
         gcc \
         gcc-c++ \
         openssl-devel \
-        zlib-devel && \
+        zlib-devel \
+        git \
+        unzip && \
     dnf clean all
 
+# 2. Install rust toolchain (shared by the wheels and prisma-engines stages)
+# RUSTC_BOOTSTRAP is needed by both the litellm wheel builds and the prisma engines
+ENV RUSTC_BOOTSTRAP=1 \
+    PATH="/root/.cargo/bin:${PATH}"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    rustup install ${RUST_VERSION} && \
+    rustup default ${RUST_VERSION}
+
 WORKDIR /build
 
-COPY pre-requirements.txt patches ./
+#### Litellm wheels stage (runs in parallel with the prisma-engines stage)
+FROM base as wheels
 
-# 2. Install Pre-requistes
-RUN python3.12 -m pip install -r pre-requirements.txt \
-	--extra-index-url=https://wheels.developerfirst.ibm.com/ppc64le/linux/ \
-	--prefer-binary
+ARG VERSION
+ARG LITELLM_SHA256
+ARG PYTHON_VERSION
+ENV PYTHON_VERSION=${PYTHON_VERSION} \
+    UV_PYTHON=${PYTHON_VERSION}
 
-# 3. Set compiler variables for ppc64le
-ENV RUSTC_BOOTSTRAP=1 \
-    GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
-    GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1
-
-# 4. Download litellm & patch it
-RUN wget https://github.com/BerriAI/litellm/archive/refs/tags/v${VERSION}.tar.gz >/dev/null
-RUN tar -xzf v${VERSION}.tar.gz && \
-    cd litellm-${VERSION} && \
-    patch -p1 pyproject.toml < ../pyproject-toml.patch && \
-    patch -p1 < ../watsonx-passthrough.patch && \
-    uv lock && \
-# 5. Export dependencies to a requirements.txt file \
-    uv export --no-hashes --no-dev \
-	--extra proxy \
-	--extra semantic-router \
-	-o requirements.txt && \
-    cat requirements.txt
-
-WORKDIR /build/litellm-${VERSION}
-
-# 6. Compile all dependencies into .whl files
+COPY pre-requirements.txt \
+     patches/litellm/build-litellm.sh \
+     patches/litellm/*.patch \
+     ./
+
+# 3. Download, patch and compile litellm and its dependencies into wheels
 RUN --mount=type=cache,target=/root/.cache/pip \
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | /usr/bin/sh -s -- -y && \
-    source "$HOME/.cargo/env" && rustup install 1.94.1 && rustup default 1.94.1 && \
-    python3.12 -m pip wheel -r requirements.txt \
-	-w /wheels \
-	--extra-index-url=https://wheels.developerfirst.ibm.com/ppc64le/linux/ \
-	--prefer-binary && \
-    uv build --wheel --out-dir /wheels
-
-# Runtime Stage
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8
+    bash build-litellm.sh
 
-ARG VERSION=${VERSION}
+#### Prisma engines stage (runs in parallel with the wheels stage)
+FROM base as prisma-engines
 
-ENV DISABLE_ADMIN_UI=true \
-    LITELLM_NON_ROOT=true
+ARG PROTOC_VERSION
+ARG PROTOC_SHA256
 
-RUN --mount=type=bind,from=builder,source=/wheels,target=/wheels,ro \
-    microdnf -y update && \
-    microdnf -y install \
-        python3.12 \
-        python3.12-pip && \
-    python3.12 -m pip install --no-cache-dir --no-index \
-	--find-links=/wheels  \
-	litellm[proxy,semantic-router]==${VERSION} && \
-    python3.12 -m pip cache purge && \
-    microdnf remove -y python3.12-pip && \
-    microdnf clean all 
+COPY patches/prisma ./prisma
 
-USER 1001
+# 4. Patch & build Prisma engines
+RUN PROTOC_VERSION=${PROTOC_VERSION} PROTOC_SHA256=${PROTOC_SHA256} \
+    bash prisma/build-prisma.sh
+
+#### Builder stage: needs both the wheels and the compiled engines
+FROM wheels as builder
+
+ARG NODE_VERSION
+ARG NODE_SHA256
+
+WORKDIR /build
+COPY --from=prisma-engines /build/prisma-engines/target/release/query-engine prisma-engines/target/release/query-engine
+COPY --from=prisma-engines /build/prisma-engines/target/release/schema-engine prisma-engines/target/release/schema-engine
+COPY --from=prisma-engines /build/prisma-engines/target/release/libquery_engine.so prisma-engines/target/release/libquery_engine.so
+COPY install-prisma-cli.sh ./
+
+# 5. Install Node.js and bake Prisma CLI
+RUN NODE_VERSION=${NODE_VERSION} NODE_SHA256=${NODE_SHA256} \
+    bash install-prisma-cli.sh
+
+#### Runtime Stage
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8
+
+ARG VERSION=${VERSION}
+ARG PYTHON_VERSION
+ARG DNF_FLAGS="--disablerepo=* --enablerepo=ubi-9-baseos-rpms --enablerepo=ubi-9-appstream-rpms"
+
+ENV LITELLM_NON_ROOT=true \
+    #pyroscope isn't supported on ppc64le
+    LITELLM_ENABLE_PYROSCOPE=false \
+    LITELLM_UI_PATH=/app/ui \
+    LITELLM_ASSETS_PATH=/app/assets \
+    PRISMA_QUERY_ENGINE_LIBRARY=/app/prisma/libquery_engine-rhel-openssl-3.0.x.so.node \
+    PRISMA_QUERY_ENGINE_BINARY=/app/prisma/query-engine \
+    PRISMA_SCHEMA_ENGINE_BINARY=/app/prisma/schema-engine \
+    PRISMA_ENGINES_CHECKSUM_IGNORE_WARNING="true" \
+    PRISMA_BINARY_CACHE_DIR=/app/prisma/binaries \
+    PRISMA_CLI_PATH=/app/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true \
+    HOME=/app \
+    PATH="/app/node/bin:/app/.venv/bin:${PATH}"
 
 WORKDIR /app
 
 COPY config ./
+COPY --from=builder /build/prisma-engines/target/release/query-engine prisma/query-engine
+COPY --from=builder /build/prisma-engines/target/release/schema-engine prisma/schema-engine
+COPY --from=builder /build/prisma-engines/target/release/libquery_engine.so prisma/libquery_engine-rhel-openssl-3.0.x.so.node
+COPY --from=builder /build/litellm-${VERSION}/schema.prisma ./
+COPY --from=builder /build/prisma/binaries prisma/binaries
+COPY --from=builder /usr/local/bin/node /app/node/bin/node
+
+RUN --mount=type=bind,from=builder,source=/wheels,target=/wheels,ro \
+    microdnf -y update ${DNF_FLAGS} && \
+    microdnf -y install ${DNF_FLAGS} \
+        python${PYTHON_VERSION} \
+        libatomic \
+        libsndfile \
+        openssl && \
+    microdnf clean all && \
+    useradd -u 1001 -M -d /app -s /sbin/nologin litellm && \
+    python${PYTHON_VERSION} -m venv /app/.venv && \
+    /app/.venv/bin/pip install --no-cache-dir --no-index --find-links=/wheels \
+        litellm[proxy,semantic-router]==${VERSION} && \
+    prisma generate --schema=/app/schema.prisma && \
+    rm -rf /app/.npm /tmp/node-compile-cache && \
+    chown -R 1001:1001 /app
+
+USER 1001
 
 EXPOSE 4000
 

images/litellm/Makefile

@@ -1,15 +1,74 @@
 REGISTRY?=icr.io/ai-services-private
 IMAGE=litellm
-# 1. Must run update-patch after updating the Litellm version
-# 2. Manually regenerate the patches/watsonx-passthrough.patch
-LITELLM_VERSION?=1.101.2
+# When updating LITELLM_VERSION:
+# 1. Run update-patch to regenerate patches/litellm/pyproject-toml.patch
+# 2. Manually update patches/litellm/watsonx-passthrough.patch if the patched file changed upstream
+# 3. Run verify-patch to confirm all patches apply cleanly
+# 4. Run update-checksums to regenerate the checksums
+LITELLM_VERSION?=1.106.4
+LITELLM_SHA256?=dc58da933f25e8efec12348a4821b71e15e358d97093821576073fee37c49589
 IMAGE_REVISION?=0
+PYTHON_VERSION?=3.14
+# Node.js version for baking Prisma CLI — Active LTS version
+NODE_VERSION?=24.21.0
+NODE_SHA256?=51c5d53066ee92920b61783b03b06ecb99f3661160f7b15ca77e8474115a67bc
+# Rust toolchain version used to build the litellm wheels and the prisma engines
+RUST_VERSION?=1.94.1
+# protoc version from https://github.com/protocolbuffers/protobuf/releases
+# PROTOC_SHA256 is the sha256 of protoc-$(PROTOC_VERSION)-linux-ppcle_64.zip
+PROTOC_VERSION?=34.1
+PROTOC_SHA256?=9160f1336e266dd0b389a29c4ffabded6cc9b80ab29f2da423106d6fe75b2ce2
 TAG?=v$(LITELLM_VERSION)-$(IMAGE_REVISION)
 CREDS_ARG := $(if $(and $(REGISTRY_USER),$(REGISTRY_PASSWORD)),--creds="$(REGISTRY_USER):$(REGISTRY_PASSWORD)")
 
 CONTAINER_BUILDER?=podman
+
+# Dev env vars — override on the command line or export before running
+INSTRUCT_MODEL_NAME?=my-instruct-model
+INSTRUCT_MODEL?=watsonx/ibm/granite-4-h-small
+WATSONX_APIKEY?=
+WATSONX_PROJECT_ID?=
+WATSONX_URL?=https://us-south.ml.cloud.ibm.com
+LITELLM_MASTER_KEY?=sk-0987612345
+DB_USER?=postgres
+DB_PASSWORD?=devpassword
+DB_HOST?=localhost
+DB_PORT?=5432
+DB_NAME?=litellm
+DATABASE_URL?=postgresql://$(DB_USER):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)
+
+run:
+	$(CONTAINER_BUILDER) run -d \
+		--name litellm-dev \
+		--net host \
+		-e INSTRUCT_MODEL_NAME="$(INSTRUCT_MODEL_NAME)" \
+		-e INSTRUCT_MODEL="$(INSTRUCT_MODEL)" \
+		-e WATSONX_APIKEY="$(WATSONX_APIKEY)" \
+		-e WATSONX_PROJECT_ID="$(WATSONX_PROJECT_ID)" \
+		-e WATSONX_URL="$(WATSONX_URL)" \
+		-e LITELLM_MASTER_KEY="$(LITELLM_MASTER_KEY)" \
+		-e DATABASE_URL="$(DATABASE_URL)" \
+		$(REGISTRY)/$(IMAGE):$(TAG) \
+		litellm -c /app/config.yaml --detailed_debug
+.PHONY: run
+
+clean:
+	-$(CONTAINER_BUILDER) stop litellm-dev
+	-$(CONTAINER_BUILDER) rm litellm-dev
+.PHONY: clean
+
 build:
-	$(CONTAINER_BUILDER) build -t $(REGISTRY)/$(IMAGE):$(TAG) --build-arg VERSION=$(LITELLM_VERSION) .
+	$(CONTAINER_BUILDER) build -t $(REGISTRY)/$(IMAGE):$(TAG) \
+		--jobs 2 \
+		--build-arg VERSION=$(LITELLM_VERSION) \
+		--build-arg LITELLM_SHA256=$(LITELLM_SHA256) \
+		--build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+		--build-arg NODE_VERSION=$(NODE_VERSION) \
+		--build-arg NODE_SHA256=$(NODE_SHA256) \
+		--build-arg RUST_VERSION=$(RUST_VERSION) \
+		--build-arg PROTOC_VERSION=$(PROTOC_VERSION) \
+		--build-arg PROTOC_SHA256=$(PROTOC_SHA256) \
+		.
 .PHONY: build
 
 push:
@@ -21,5 +80,23 @@ image-tag:
 .PHONY: image-tag
 
 update-patch:
-	cd ./patches && ./pyproject-toml.sh $(LITELLM_VERSION)
+	cd ./patches/litellm && ./pyproject-toml.sh $(LITELLM_VERSION)
 .PHONY: update-patch
+
+# Fetch the upstream source files for LITELLM_VERSION, apply each patch, and
+# show the resulting diff so you can confirm the patches apply cleanly.
+verify-patch:
+	./patches/litellm/verify-patch.sh $(LITELLM_VERSION)
+.PHONY: verify-patch
+
+update-checksums:
+	$(eval NEW_LITELLM_SHA256 := $(shell curl -fsSL https://github.com/BerriAI/litellm/archive/refs/tags/v$(LITELLM_VERSION).tar.gz | sha256sum | cut -d' ' -f1))
+	$(eval NEW_NODE_SHA256 := $(shell curl -fsSL https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-linux-ppc64le.tar.gz | sha256sum | cut -d' ' -f1))
+	$(eval NEW_PROTOC_SHA256 := $(shell curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-ppcle_64.zip | sha256sum | cut -d' ' -f1))
+	sed -i "s|^LITELLM_SHA256?=.*|LITELLM_SHA256?=$(NEW_LITELLM_SHA256)|" $(MAKEFILE_LIST)
+	sed -i "s|^NODE_SHA256?=.*|NODE_SHA256?=$(NEW_NODE_SHA256)|" $(MAKEFILE_LIST)
+	sed -i "s|^PROTOC_SHA256?=.*|PROTOC_SHA256?=$(NEW_PROTOC_SHA256)|" $(MAKEFILE_LIST)
+	@echo "Updated LITELLM_SHA256 to $(NEW_LITELLM_SHA256)"
+	@echo "Updated NODE_SHA256    to $(NEW_NODE_SHA256)"
+	@echo "Updated PROTOC_SHA256  to $(NEW_PROTOC_SHA256)"
+.PHONY: update-checksums

images/litellm/config/config.yaml

@@ -1,13 +1,6 @@
-model_list:
-  - model_name: "os.environ/INSTRUCT_MODEL_NAME"
-    litellm_params:
-      model: "os.environ/INSTRUCT_MODEL"
-      api_key: "os.environ/WATSONX_APIKEY"
-      project_id: "os.environ/WATSONX_PROJECT_ID"
-      base_url: "os.environ/WATSONX_URL"
 litellm_settings:
   callbacks: custom_callbacks.translator_instance
 general_settings:
-  pass_through_endpoints:
-    - path: "/tokenize"
-      target: http://localhost:8000/watsonx/ml/v1/text/tokenization
+  reject_clientside_metadata_tags: true
+router_settings:
+  enable_tag_filtering: True