Skip to content

Add MiniMax provider support to Mobile-Agent-E #290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
octo-patch wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add MiniMax provider support to Mobile-Agent-E #290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 135 additions & 17 deletions Mobile-Agent-E/MobileAgentE/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,90 @@
from time import sleep
import json


MINIMAX_PRICING = {
"USD": {
"MiniMax-M3": {
"standard": {
"short": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": None},
"long": {"input": 0.6, "output": 2.4, "cache_read": 0.12, "cache_write": None},
},
"priority": {
"short": {"input": 0.45, "output": 1.8, "cache_read": 0.09, "cache_write": None},
"long": {"input": 0.9, "output": 3.6, "cache_read": 0.18, "cache_write": None},
},
},
"MiniMax-M2.7": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.375},
},
"CNY": {
"MiniMax-M3": {
"standard": {
"short": {"input": 2.1, "output": 8.4, "cache_read": 0.42, "cache_write": None},
"long": {"input": 4.2, "output": 16.8, "cache_read": 0.84, "cache_write": None},
},
"priority": {
"short": {"input": 3.15, "output": 12.6, "cache_read": 0.63, "cache_write": None},
"long": {"input": 6.3, "output": 25.2, "cache_read": 1.26, "cache_write": None},
},
},
"MiniMax-M2.7": {"input": 2.1, "output": 8.4, "cache_read": 0.42, "cache_write": 2.625},
},
}


def encode_image(image_path):
with open(image_path, "rb") as image_file:
return base64.b64encode(image_file.read()).decode('utf-8')


def track_usage(res_json, api_key):
def build_api_url(base_url, api_protocol):
api_protocol = api_protocol.lower()
if api_protocol not in ("openai", "anthropic"):
raise ValueError(f"Unsupported API protocol: {api_protocol}")
path = "/chat/completions" if api_protocol == "openai" else "/v1/messages"
return f"{base_url.rstrip('/')}{path}"


def _minimax_rates(model, input_tokens, service_tier, price_currency):
currency_pricing = MINIMAX_PRICING.get(price_currency)
if currency_pricing is None or model not in currency_pricing:
return None
model_pricing = currency_pricing[model]
if model == "MiniMax-M3":
length_tier = "long" if input_tokens > 512000 else "short"
return model_pricing[service_tier][length_tier]
return model_pricing


def track_usage(res_json, api_key, service_tier="standard", price_currency="USD"):
"""
{'id': 'chatcmpl-AbJIS3o0HMEW9CWtRjU43bu2Ccrdu', 'object': 'chat.completion', 'created': 1733455676, 'model': 'gpt-4o-2024-11-20', 'choices': [...], 'usage': {'prompt_tokens': 2731, 'completion_tokens': 235, 'total_tokens': 2966, 'prompt_tokens_details': {'cached_tokens': 0, 'audio_tokens': 0}, 'completion_tokens_details': {'reasoning_tokens': 0, 'audio_tokens': 0, 'accepted_prediction_tokens': 0, 'rejected_prediction_tokens': 0}}, 'system_fingerprint': 'fp_28935134ad'}
"""
model = res_json['model']
usage = res_json['usage']
if "prompt_tokens" in usage and "completion_tokens" in usage:
openai_usage = "prompt_tokens" in usage and "completion_tokens" in usage
if openai_usage:
prompt_tokens, completion_tokens = usage['prompt_tokens'], usage['completion_tokens']
elif "promptTokens" in usage and "completionTokens" in usage:
prompt_tokens, completion_tokens = usage['promptTokens'], usage['completionTokens']
elif "input_tokens" in usage and "output_tokens" in usage:
prompt_tokens, completion_tokens = usage['input_tokens'], usage['output_tokens']
else:
prompt_tokens, completion_tokens = None, None


cache_read_tokens = usage.get("cache_read_input_tokens", 0)
cache_write_tokens = usage.get("cache_creation_input_tokens", 0)
if openai_usage:
cache_read_tokens = usage.get("prompt_tokens_details", {}).get("cached_tokens", 0)

prompt_token_price = None
completion_token_price = None
cache_read_token_price = None
cache_write_token_price = None
selected_service_tier = res_json.get("service_tier") or service_tier or "standard"
if selected_service_tier == "default":
selected_service_tier = "standard"
price_currency = price_currency.upper()
if prompt_tokens is not None and completion_tokens is not None:
if "gpt-4o" in model:
prompt_token_price = (2.5 / 1000000) * prompt_tokens
Expand All @@ -35,20 +97,54 @@ def track_usage(res_json, api_key):
elif "claude" in model:
prompt_token_price = (3 / 1000000) * prompt_tokens
completion_token_price = (15 / 1000000) * completion_tokens
elif model in ("MiniMax-M3", "MiniMax-M2.7"):
pricing_input_tokens = prompt_tokens
billable_input_tokens = prompt_tokens
if openai_usage:
billable_input_tokens = max(prompt_tokens - cache_read_tokens, 0)
else:
pricing_input_tokens += cache_read_tokens + cache_write_tokens
rates = _minimax_rates(model, pricing_input_tokens, selected_service_tier, price_currency)
if rates is not None:
prompt_token_price = (rates["input"] / 1000000) * billable_input_tokens
completion_token_price = (rates["output"] / 1000000) * completion_tokens
if rates["cache_read"] is not None:
cache_read_token_price = (rates["cache_read"] / 1000000) * cache_read_tokens
if rates["cache_write"] is not None:
cache_write_token_price = (rates["cache_write"] / 1000000) * cache_write_tokens
return {
# "api_key": api_key, # remove for better safety
"id": res_json['id'] if "id" in res_json else None,
"model": model,
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"cache_read_tokens": cache_read_tokens,
"cache_write_tokens": cache_write_tokens,
"prompt_token_price": prompt_token_price,
"completion_token_price": completion_token_price
"completion_token_price": completion_token_price,
"cache_read_token_price": cache_read_token_price,
"cache_write_token_price": cache_write_token_price,
"price_currency": price_currency,
"service_tier": selected_service_tier,
}

def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max_tokens = 2048, temperature = 0.0):

def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max_tokens = 2048,
temperature = 0.0, api_protocol = None, service_tier = None,
thinking = None, price_currency = "USD"):
if token is None:
raise ValueError("API key is required")


if api_protocol is None:
api_protocol = "anthropic" if "claude" in model.lower() else "openai"
api_protocol = api_protocol.lower()
if api_protocol not in ("openai", "anthropic"):
raise ValueError(f"Unsupported API protocol: {api_protocol}")
if service_tier is not None and service_tier not in ("standard", "priority"):
raise ValueError(f"Unsupported service tier: {service_tier}")
if thinking is not None and thinking not in ("adaptive", "disabled"):
raise ValueError(f"Unsupported thinking mode: {thinking}")

headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {token}"
Expand All @@ -61,7 +157,12 @@ def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max
'temperature': temperature
}

if "claude" in model:
if service_tier is not None:
data["service_tier"] = service_tier
if thinking is not None:
data["thinking"] = {"type": thinking}

if api_protocol == "anthropic":
if "47.88.8.18:8088" not in api_url:
# using official api url
headers = {
Expand All @@ -78,14 +179,25 @@ def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max
for item in content:
if item['type'] == "text":
converted_content.append({"type": "text", "text": item['text']})
elif item['type'] == "image_url":
converted_content.append({
"type": "image",
"source": {
elif item['type'] in ("image_url", "video_url"):
media_type = "image" if item['type'] == "image_url" else "video"
media = item[item['type']]
media_url = media['url']
if media_url.startswith("data:"):
metadata, encoded_data = media_url.split(",", 1)
source = {
"type": "base64",
"media_type": "image/jpeg",
"data": item['image_url']['url'].replace("data:image/jpeg;base64,", "")
"media_type": metadata.removeprefix("data:").split(";", 1)[0],
"data": encoded_data,
}
else:
source = {"type": "url", "url": media_url}
for option in ("detail", "fps", "max_long_side_pixel"):
if option in media:
source[option] = media[option]
converted_content.append({
"type": media_type,
"source": source,
})
else:
raise ValueError(f"Invalid content type: {item['type']}")
Expand All @@ -99,18 +211,24 @@ def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max

while True:
try:
if "claude" in model:
if api_protocol == "anthropic":
res = requests.post(api_url, headers=headers, data=json.dumps(data))
res_json = res.json()
# print(res_json)
res_content = res_json['content'][0]['text']
text_blocks = [block['text'] for block in res_json['content'] if block.get('type') == 'text']
res_content = "\n".join(text_blocks)
else:
res = requests.post(api_url, headers=headers, json=data)
res_json = res.json()
# print(res_json)
res_content = res_json['choices'][0]['message']['content']
if usage_tracking_jsonl:
usage = track_usage(res_json, api_key=token)
usage = track_usage(
res_json,
api_key=token,
service_tier=service_tier or "standard",
price_currency=price_currency,
)
with open(usage_tracking_jsonl, "a") as f:
f.write(json.dumps(usage) + "\n")
except:
Expand All @@ -128,4 +246,4 @@ def inference_chat(chat, model, api_url, token, usage_tracking_jsonl = None, max
print(f"Failed after {max_retry} retries...")
return None

return res_content
return res_content
22 changes: 21 additions & 1 deletion Mobile-Agent-E/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,26 @@ Please refer to the `# Edit your Setting #` section in `inference_agent_E.py` fo
export BACKBONE_TYPE="Claude"
export CLAUDE_API_KEY="your-claude-key"
```
To use MiniMax, select a supported model and API protocol. MiniMax-M3 supports text, image, and video input with a 1,000,000-token context window and optional `adaptive` or `disabled` thinking. MiniMax-M2.7 has a 204,800-token context window, accepts text input, and always uses thinking.
```
export BACKBONE_TYPE="MiniMax"
export MINIMAX_API_KEY="your-minimax-key"
export MINIMAX_MODEL="MiniMax-M3" # MiniMax-M3 or MiniMax-M2.7
export MINIMAX_API_PROTOCOL="anthropic" # anthropic or openai
export MINIMAX_BASE_URL="https://api.minimax.io/anthropic"
export MINIMAX_SERVICE_TIER="standard" # standard or priority for MiniMax-M3
export MINIMAX_THINKING="disabled" # disabled or adaptive for MiniMax-M3
```
Use the matching base URL for your region and protocol. The application appends the protocol-specific request path internally.

| Region | Protocol | Base URL |
| --- | --- | --- |
| Global | Anthropic | `https://api.minimax.io/anthropic` |
| Global | OpenAI | `https://api.minimax.io/v1` |
| China | Anthropic | `https://api.minimaxi.com/anthropic` |
| China | OpenAI | `https://api.minimaxi.com/v1` |

Usage tracking applies the official regional rates, including MiniMax-M3's context-sensitive and service-tier pricing. The currency defaults to `USD` for the global endpoint and `CNY` for the China endpoint; set `MINIMAX_PRICE_CURRENCY` explicitly when routing through a custom proxy. See the [MiniMax API documentation](https://platform.minimax.io/docs/api-reference/api-overview) for current model and protocol details.
3. Perceptor: By default, the icon captioning model (`CAPTION_MODEL`) in Perceptor uses "qwen-vl-plus" from Qwen API:
- Follow this to get an [Qwen API Key](https://help.aliyun.com/document_detail/2712195.html?spm=a2c4g.2712569.0.0.5d9e730aymB3jH)
- Set the Qwen API key:
Expand Down Expand Up @@ -137,4 +157,4 @@ The proposed Mobile-Eval-E benchmark can be found in `data/Mobile-Eval-E` and al
journal={arXiv preprint arXiv:2501.11733},
year={2025}
}
```
```
53 changes: 49 additions & 4 deletions Mobile-Agent-E/inference_agent_E.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from PIL import Image, ImageDraw
from time import sleep

from MobileAgentE.api import inference_chat
from MobileAgentE.api import build_api_url, inference_chat
from MobileAgentE.text_localization import ocr
from MobileAgentE.icon_localization import det
from MobileAgentE.controller import get_screenshot, start_recording, end_recording
Expand Down Expand Up @@ -35,8 +35,8 @@
ADB_PATH = os.environ.get("ADB_PATH", default="adb")

## Reasoning model configs
BACKBONE_TYPE = os.environ.get("BACKBONE_TYPE", default="OpenAI") # "OpenAI" or "Gemini" or "Claude"
assert BACKBONE_TYPE in ["OpenAI", "Gemini", "Claude"], "Unknown BACKBONE_TYPE"
BACKBONE_TYPE = os.environ.get("BACKBONE_TYPE", default="OpenAI") # "OpenAI" or "Gemini" or "Claude" or "MiniMax"
assert BACKBONE_TYPE in ["OpenAI", "Gemini", "Claude", "MiniMax"], "Unknown BACKBONE_TYPE"
print("### Using BACKBONE_TYPE:", BACKBONE_TYPE)

OPENAI_API_URL = "https://api.openai.com/v1/chat/completions"
Expand All @@ -48,6 +48,35 @@
CLAUDE_API_URL = "https://api.anthropic.com/v1/messages"
CLAUDE_API_KEY = os.environ.get("CLAUDE_API_KEY", default=None)

MINIMAX_MODELS = ("MiniMax-M3", "MiniMax-M2.7")
MINIMAX_MODEL = os.environ.get("MINIMAX_MODEL", default="MiniMax-M3")
assert MINIMAX_MODEL in MINIMAX_MODELS, "Unknown MINIMAX_MODEL"
MINIMAX_API_PROTOCOL = os.environ.get("MINIMAX_API_PROTOCOL", default="anthropic").lower()
assert MINIMAX_API_PROTOCOL in ("openai", "anthropic"), "Unknown MINIMAX_API_PROTOCOL"
MINIMAX_DEFAULT_BASE_URLS = {
"openai": "https://api.minimax.io/v1",
"anthropic": "https://api.minimax.io/anthropic",
}
MINIMAX_BASE_URL = os.environ.get("MINIMAX_BASE_URL", default=MINIMAX_DEFAULT_BASE_URLS[MINIMAX_API_PROTOCOL])
MINIMAX_API_URL = build_api_url(MINIMAX_BASE_URL, MINIMAX_API_PROTOCOL)
MINIMAX_API_KEY = os.environ.get("MINIMAX_API_KEY", default=None)
MINIMAX_SERVICE_TIER = os.environ.get("MINIMAX_SERVICE_TIER", default="standard")
assert MINIMAX_SERVICE_TIER in ("standard", "priority"), "Unknown MINIMAX_SERVICE_TIER"
assert MINIMAX_MODEL == "MiniMax-M3" or MINIMAX_SERVICE_TIER == "standard", "Priority requires MiniMax-M3"
MINIMAX_THINKING = os.environ.get(
"MINIMAX_THINKING",
default="disabled" if MINIMAX_MODEL == "MiniMax-M3" else "always_on",
)
if MINIMAX_MODEL == "MiniMax-M3":
assert MINIMAX_THINKING in ("adaptive", "disabled"), "Unknown MINIMAX_THINKING"
else:
assert MINIMAX_THINKING == "always_on", "MiniMax-M2.7 thinking is always on"
MINIMAX_PRICE_CURRENCY = os.environ.get(
"MINIMAX_PRICE_CURRENCY",
default="CNY" if "api.minimaxi.com" in MINIMAX_BASE_URL else "USD",
).upper()
assert MINIMAX_PRICE_CURRENCY in ("USD", "CNY"), "Unknown MINIMAX_PRICE_CURRENCY"

if BACKBONE_TYPE == "OpenAI":
REASONING_MODEL = "gpt-4o-2024-11-20"
KNOWLEDGE_REFLECTION_MODEL = "gpt-4o-2024-11-20"
Expand All @@ -57,6 +86,9 @@
elif BACKBONE_TYPE == "Claude":
REASONING_MODEL = "claude-3-5-sonnet-20241022"
KNOWLEDGE_REFLECTION_MODEL = "claude-3-5-sonnet-20241022"
elif BACKBONE_TYPE == "MiniMax":
REASONING_MODEL = MINIMAX_MODEL
KNOWLEDGE_REFLECTION_MODEL = MINIMAX_MODEL

## you can specify a jsonl file path for tracking API usage
USAGE_TRACKING_JSONL = None # e.g., usage_tracking.jsonl
Expand Down Expand Up @@ -385,6 +417,19 @@ def get_reasoning_model_api_response(chat, model_type=BACKBONE_TYPE, model=None,
return inference_chat(chat, model, GEMINI_API_URL, GEMINI_API_KEY, usage_tracking_jsonl=USAGE_TRACKING_JSONL, temperature=temperature)
elif model_type == "Claude":
return inference_chat(chat, model, CLAUDE_API_URL, CLAUDE_API_KEY, usage_tracking_jsonl=USAGE_TRACKING_JSONL, temperature=temperature)
elif model_type == "MiniMax":
return inference_chat(
chat,
model,
MINIMAX_API_URL,
MINIMAX_API_KEY,
usage_tracking_jsonl=USAGE_TRACKING_JSONL,
temperature=temperature,
api_protocol=MINIMAX_API_PROTOCOL,
service_tier=MINIMAX_SERVICE_TIER if model == "MiniMax-M3" else None,
thinking=MINIMAX_THINKING if model == "MiniMax-M3" else None,
price_currency=MINIMAX_PRICE_CURRENCY,
)
else:
raise ValueError(f"Unknown model type: {model_type}")

Expand Down Expand Up @@ -1032,4 +1077,4 @@ def run_single_task(
end_recording(ADB_PATH, output_recording_path=cur_output_recording_path)
print("\n=========================================================")
print(f"sleeping for {SLEEP_BETWEEN_STEPS} before next iteration ...\n\n")
sleep(SLEEP_BETWEEN_STEPS)
sleep(SLEEP_BETWEEN_STEPS)
Loading