diff --git a/reme/config/jinli_lme.yaml b/reme/config/jinli_lme.yaml index 6b6f76d6..aaf3c637 100644 --- a/reme/config/jinli_lme.yaml +++ b/reme/config/jinli_lme.yaml @@ -92,6 +92,41 @@ jobs: steps: - backend: python_execute_step + lme_one_question: + backend: base + description: "Run context_answer_step + answer_judge_step for a single LME question." + parameters: + type: object + properties: + query: + type: string + description: "the LME question being answered" + session_context: + type: string + description: "concatenated chat history for the relevant session" + current_date: + type: string + description: "today's date in YYYY-MM-DD, used to resolve relative phrases" + agent_answer: + type: string + description: "answer from the agent-under-test; can be the value of `context_answer` from a prior call" + golden_answer: + type: string + description: "ground-truth answer from the LME dataset" + question_type: + type: string + description: "LME category; one of temporal_reasoning / knowledge_update / single_session_preference / other" + required: + - query + - session_context + - current_date + - agent_answer + - golden_answer + - question_type + steps: + - backend: context_answer_step + - backend: answer_judge_step + components: tokenizer: default: diff --git a/tests/unit/test_config_parser.py b/tests/unit/test_config_parser.py index effe99b3..f1cdc0ff 100644 --- a/tests/unit/test_config_parser.py +++ b/tests/unit/test_config_parser.py @@ -86,3 +86,27 @@ def test_expand_env_vars_converts_expanded_scalar_types(monkeypatch): "url": "http://localhost:18080", "string_bool": "false", } + + +def test_jinli_lme_config_registers_lme_one_question_job(): + """``jinli_lme.yaml`` exposes a ``lme_one_question`` job that bundles the + LME ``context_answer`` and ``answer_judge`` steps and requires every + input the two steps consume.""" + cfg = _load_config("jinli_lme.yaml") + + job = cfg["jobs"]["lme_one_question"] + assert job["backend"] == "base" + assert [step["backend"] for step in job["steps"]] == [ + "context_answer_step", + "answer_judge_step", + ] + assert job["parameters"]["required"] == [ + "query", + "session_context", + "current_date", + "agent_answer", + "golden_answer", + "question_type", + ] + props = set(job["parameters"]["properties"]) + assert props == set(job["parameters"]["required"])