Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ pub(crate) fn lifecycle_action_eligibility_with_roots(
roots: Option<&homeboy_core::paths::PathRoots>,
) -> ControlPlaneActionEligibilityReport {
let state = CookRunState::of(record);
if record.metadata.get("cook_id").is_some() && super::legal_actions(&state).is_empty() {
return ControlPlaneActionEligibilityReport {
schema: CONTROL_PLANE_ACTION_ELIGIBILITY_SCHEMA.to_string(),
run: RunId::new(&record.run_id).expect("validated durable run ID"),
actions: Vec::new(),
};
}
let cancellation = if !state.allows(CookRunAction::Cancel) {
unavailable(format!(
"agent-task run '{}' is already terminal with state {} and no live owner",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ mod handoff_and_proxy;
mod lifecycle_store;
mod non_terminal_transitions;
mod private_attachment;
mod read_projection_guard;
mod runner_exec;
mod state_graph;
mod status_and_recovery;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
//! Read surfaces must not grow a second retained-candidate classifier.

#[test]
fn read_surfaces_do_not_classify_candidate_labels() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let files = [
"homeboy-cli/src/commands/agent_task/status.rs",
"homeboy-cli/src/commands/cook_verdict.rs",
"homeboy-cli/src/commands/agent_task_summary/mod.rs",
"homeboy-agents/src/agent_task_notify.rs",
"homeboy-agents/src/agent_task_notify_work.rs",
"homeboy-agents/src/agent_task_notify_tracker.rs",
"homeboy-agents/src/agent_task_notify_tracker_note.rs",
"homeboy-agents/src/agent_task_lifecycle/durable_state_probe.rs",
"homeboy-agents/src/agent_task_scheduler/engine.rs",
"homeboy-agents/src/agent_task_scheduler/scheduling.rs",
"homeboy-agents/src/agent_task_schedule.rs",
];
for file in files {
let source = std::fs::read_to_string(root.join("crates").join(file)).unwrap();
// Remove inline test items, not the remainder of the file: status has
// production functions following several inline test modules.
let mut test_item = false;
let mut opened = false;
let mut depth = 0isize;
for (line, text) in source.lines().enumerate() {
let text = text.trim();
if text == "#[cfg(test)]" || text == "#[test]" {
test_item = true;
opened = false;
depth = 0;
continue;
}
if test_item {
opened |= text.contains('{');
depth += text.matches('{').count() as isize - text.matches('}').count() as isize;
if (opened && depth <= 0) || (!opened && text.ends_with(';')) {
test_item = false;
}
continue;
}
if text.starts_with("//") {
continue;
}
assert!(
!text.contains("let candidate_recoverable")
&& !(text.contains("candidate_recoverable")
&& (text.contains("==") || text.contains("!=") || text.starts_with("if "))),
"{file}:{} reclassifies a Cook candidate: {text}",
line + 1
);
}
}
}
15 changes: 13 additions & 2 deletions crates/homeboy-agents/src/agent_task_notify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,12 @@ fn terminal_payload(
let cancelled = metadata.cancelled;
let kind = if cancelled {
NotifyEventKind::Cancelled
} else if exit_code == 0 && report.completed_successfully() {
} else if exit_code == 0
&& metadata.state.as_ref().map_or_else(
|| report.completed_successfully(),
|state| state.phase == crate::agent_task_lifecycle::CookRunPhase::Succeeded,
)
{
NotifyEventKind::Completed
} else {
NotifyEventKind::NeedsAttention
Expand Down Expand Up @@ -472,7 +477,13 @@ fn terminal_payload(
.collect::<Vec<_>>();
payload = payload.with_fact("Gates", gate_summary(&gates));
}
let status = homeboy_core::cook_status::CookStatus::from_status(&report.status);
let status = homeboy_core::cook_status::CookStatus::from_status(
metadata
.state
.as_ref()
.map(|state| state.status.as_str())
.unwrap_or(&report.status),
);
if status.is_gate_failure() && !payload.facts.iter().any(|fact| fact.label == "Gates") {
payload = payload.with_fact("Gates", "failed");
} else if status == homeboy_core::cook_status::CookStatus::GateEnvironmentUnavailable {
Expand Down
4 changes: 3 additions & 1 deletion crates/homeboy-agents/src/agent_task_notify_tracker_note.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ fn durable_note(
.run_id
.clone();
let record = store.read_record(&run_id).ok()?;
if record.state() == homeboy_lifecycle_contract::RunExecutionState::Cancelled {
if crate::agent_task_lifecycle::CookRunState::of(&record).phase
== crate::agent_task_lifecycle::CookRunPhase::Cancelled
{
return None;
}
let aggregate = store.read_aggregate(&run_id).ok()?;
Expand Down
13 changes: 10 additions & 3 deletions crates/homeboy-agents/src/agent_task_notify_work.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use super::*;
pub(super) struct Work {
pub component: Option<String>,
pub cancelled: bool,
pub state: Option<crate::agent_task_lifecycle::CookRunState>,
task: String,
pub pr: Option<String>,
pub links: Vec<homeboy_core::notification_payload::NotifyLink>,
Expand Down Expand Up @@ -60,9 +61,14 @@ pub(super) fn metadata(report: &AgentTaskCookReport, component: Option<&str>) ->
Some((_, store)) => crate::agent_task_lifecycle::status_in_store(store, record_id).ok(),
None => crate::agent_task_lifecycle::status(record_id).ok(),
};
let cancelled = report.status == "cancelled"
|| record.as_ref().is_some_and(|record| {
record.state() == homeboy_lifecycle_contract::RunExecutionState::Cancelled
let state = record
.as_ref()
.map(crate::agent_task_lifecycle::CookRunState::of);
let cancelled = record
.as_ref()
.map_or(report.status == "cancelled", |record| {
crate::agent_task_lifecycle::CookRunState::of(record).phase
== crate::agent_task_lifecycle::CookRunPhase::Cancelled
});

let mut facts = Vec::new();
Expand Down Expand Up @@ -125,6 +131,7 @@ pub(super) fn metadata(report: &AgentTaskCookReport, component: Option<&str>) ->
Work {
component,
cancelled,
state,
task: task_title,
pr,
links,
Expand Down
3 changes: 2 additions & 1 deletion crates/homeboy-agents/src/agent_task_scheduler/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,8 @@ impl AgentTaskScheduler {
stop,
thread::spawn(move || loop {
if store.read_record(&run_id).is_ok_and(|record| {
record.state() == homeboy_lifecycle_contract::RunExecutionState::Cancelled
crate::agent_task_lifecycle::CookRunState::of(&record).phase
== crate::agent_task_lifecycle::CookRunPhase::Cancelled
}) {
watcher_cancellation.cancel();
return;
Expand Down
6 changes: 5 additions & 1 deletion crates/homeboy-agents/src/agent_task_service/cook_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9261,9 +9261,13 @@ fn cook_observation_agrees_across_real_stores_without_mutation() {
"{name}"
);
if live_owner {
// A leftover runner PID does not override a terminal
// Cook phase. Only the classifier's continuation owner
// can project a terminal attempt as still running.
assert_eq!(canonical.state, crate::orchestration::run_state(&observed));
assert_eq!(
canonical.state,
homeboy_control_plane_contract::ControlPlaneState::Running
homeboy_control_plane_contract::ControlPlaneState::CandidateRecoverable
);
}
}
Expand Down
126 changes: 110 additions & 16 deletions crates/homeboy-agents/src/orchestration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4032,18 +4032,13 @@ pub fn project_record(
// interruption; the reconcile action remains the mutating path.
let record = &project_dead_candidate_adoption(record);
let mut resource = project_summary_fields(record, plan)?;
// Both the recovery step and the eligibility report project the one
// `CookRunState` and its `legal_actions()`: the step is the
// `CookRecoveryPlan` that `recover` executes, gated on `Recover` being
// legal, and the report is empty exactly when no action is legal. So a
// projected step other than `none` always comes with a non-empty report.
resource.recovery = projected_cook_recovery(record);
resource.action_eligibility =
if crate::agent_task_service::live_cook_continuation(record).is_some() {
None
} else {
Some(lifecycle_action_eligibility(record, plan))
};
if record.metadata.get("cancellation_fence").is_some()
|| record.metadata.get("candidate_retirement").is_some()
{
resource.action_eligibility = None;
}
resource.action_eligibility = Some(lifecycle_action_eligibility(record, plan));
Ok(resource)
}

Expand Down Expand Up @@ -4154,7 +4149,10 @@ fn project_summary_fields(
resource.gates = gates(record, delivery.as_ref());
resource.publication = delivery.as_ref().and_then(publication);
project_live_gate_verification(&mut resource, record);
if let Some(continuation) = crate::agent_task_service::live_cook_continuation(record) {
if crate::agent_task_lifecycle::CookRunState::of(record).phase
== crate::agent_task_lifecycle::CookRunPhase::Continuing
{
let continuation = &record.metadata["cook_continuation"];
resource.state = ControlPlaneState::Running;
resource.phase = Some("publishing".to_string());
resource.owner = Some(ControlPlaneOwner {
Expand All @@ -4168,6 +4166,8 @@ fn project_summary_fields(
.get("cancellation_fence")
.or_else(|| record.metadata.get("candidate_retirement"))
{
// The classifier's `fenced` fact; the public projection keeps its
// historical `cancelled` label for a fenced mission.
resource.state = ControlPlaneState::Cancelled;
resource.phase = Some("cancelled".into());
resource.blocker = Some(ControlPlaneBlocker {
Expand Down Expand Up @@ -4206,6 +4206,9 @@ pub const LIVE_GATE_VERIFYING_PHASE: &str = "verifying";
/// as running in the `verifying` phase, with no recovery step. The durable
/// run state is unchanged; this is a projection only.
fn project_live_gate_verification(resource: &mut ControlPlaneRun, record: &AgentTaskRunRecord) {
if record.metadata.get("cook_id").is_some() {
return;
}
if !awaits_gate_verification(record) {
return;
}
Expand Down Expand Up @@ -4279,8 +4282,19 @@ fn projected_cook_recovery(record: &AgentTaskRunRecord) -> Option<ControlPlaneRe
cook_id,
});
};
let step = plan.step.action_name().unwrap_or("none").to_string();
let allowed = crate::agent_task_lifecycle::CookRunState::of(record)
.allows(crate::agent_task_lifecycle::CookRunAction::Recover);
let step = if allowed {
plan.step.action_name().unwrap_or("none")
} else {
"none"
}
.to_string();
let (candidate_ref, command) = match &plan.step {
_ if !allowed => (
None,
format!("homeboy agent-task diagnose {}", quote(&plan.cook_id)),
),
crate::agent_task_service::CookRecoveryStep::Adopt { candidate_ref } => (
Some(candidate_ref.clone()),
format!(
Expand Down Expand Up @@ -4515,6 +4529,19 @@ fn fanout_mission(record: &AgentTaskRunRecord) -> Result<Option<MissionId>, Cont
}

pub(crate) fn run_state(record: &AgentTaskRunRecord) -> ControlPlaneState {
if record.metadata.get("cook_id").is_some() {
use crate::agent_task_lifecycle::{CookRunPhase, CookRunState};
return match CookRunState::of(record).phase {
CookRunPhase::Queued => ControlPlaneState::Queued,
CookRunPhase::Running | CookRunPhase::Continuing => ControlPlaneState::Running,
CookRunPhase::CandidateRecoverable => ControlPlaneState::CandidateRecoverable,
CookRunPhase::Succeeded => ControlPlaneState::Succeeded,
CookRunPhase::PartialFailure => ControlPlaneState::PartialFailure,
CookRunPhase::Failed => ControlPlaneState::Failed,
CookRunPhase::Cancelled => ControlPlaneState::Cancelled,
CookRunPhase::Unknown => ControlPlaneState::Failed,
};
}
if CookDelivery::from_run_metadata(&record.metadata).is_some_and(|d| d.published()) {
return ControlPlaneState::Succeeded;
}
Expand Down Expand Up @@ -10355,6 +10382,72 @@ mod tests {
);
}

/// The case we hit: status reported `candidate_recoverable` with
/// `next_action: recover` while `action_eligibility.actions` was empty.
/// The recovery step and the report now project one `CookRunState`, so a
/// step other than `none` never coexists with an empty report.
#[test]
fn candidate_recoverable_status_never_names_recover_with_empty_actions() {
let live_continuation = json!({
"state": "claimed",
"owner_pid": std::process::id(),
});
for (label, extra) in [
("plain", json!({})),
(
"fenced",
json!({"cancellation_fence": {"reason": "cancelled"}}),
),
(
"retired",
json!({"candidate_retirement": {"reason": "retired"}}),
),
(
"continuing",
json!({"cook_continuation": live_continuation}),
),
] {
let mut record = record(AGENT_TASK_RUN);
record.set_state_for_test(RunExecutionState::CandidateRecoverable);
record.metadata["cook_id"] = json!("candidate-cook");
for (key, value) in extra.as_object().unwrap() {
record.metadata[key] = value.clone();
}
let state = crate::agent_task_lifecycle::CookRunState::of(&record);
let legal = crate::agent_task_lifecycle::legal_actions(&state);
let projected = project_record(&record, None).expect("projection");
let actions = projected.action_eligibility.expect("report").actions;
let recovery = projected.recovery.expect("Cook recovery projection");
assert_eq!(actions.is_empty(), legal.is_empty(), "{label}");
if actions.is_empty()
|| !state.allows(crate::agent_task_lifecycle::CookRunAction::Recover)
{
assert_eq!(recovery.step, "none", "{label}: {recovery:?}");
assert!(
!recovery.command.contains("agent-task recover "),
"{label}: {recovery:?}"
);
}
if recovery.step != "none" {
assert!(!actions.is_empty(), "{label}: {recovery:?}");
}
}
}

#[test]
fn fenced_candidate_status_never_advertises_recovery_without_actions() {
let mut record = record(AGENT_TASK_RUN);
record.set_state_for_test(RunExecutionState::CandidateRecoverable);
record.metadata["cook_id"] = json!("fenced-cook");
record.metadata["cancellation_fence"] = json!({"reason": "cancelled"});
let projected = project_record(&record, None).expect("projection");
assert_eq!(projected.state, ControlPlaneState::Cancelled);
assert!(projected.action_eligibility.unwrap().actions.is_empty());
let recovery = projected.recovery.expect("Cook recovery projection");
assert_eq!(recovery.step, "none");
assert!(!recovery.command.contains("agent-task recover "));
}

#[test]
fn project_record_matches_service_run() {
let seeded = record(AGENT_TASK_RUN);
Expand All @@ -10365,17 +10458,18 @@ mod tests {
}

#[test]
fn active_execution_step_owner_projects_running_promotion_gate_state() {
fn execution_step_owner_does_not_override_the_cook_classifier() {
let mut record = record(AGENT_TASK_RUN);
record.set_state_for_test(RunExecutionState::Succeeded);
record.metadata.as_object_mut().unwrap().extend(
crate::agent_task_lifecycle::execution_step_owner_fixture("cook"),
);

let resource = project_record(&record, None).expect("project active supervisor");
assert_eq!(resource.state, ControlPlaneState::Running);
assert_eq!(resource.state, run_state(&record));
assert_eq!(resource.state, ControlPlaneState::Succeeded);
assert_eq!(resource.phase.as_deref(), Some("promotion_gates"));
assert!(resource.finished_at.is_none());
assert_eq!(resource.finished_at, record.updated_at);
}

#[test]
Expand Down
6 changes: 5 additions & 1 deletion crates/homeboy-agents/src/orchestration_status_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,11 @@ fn status_current_delivery_blocker_outranks_provider_diagnostic_and_success_has_
);
let published = fixture("contradictory-publication");
let resource = project_snapshot(&published, false).unwrap();
assert_eq!(resource.state, ControlPlaneState::Succeeded);
// A publication receipt does not rewrite the durable Cook execution
// state. This contradictory fixture must project the classifier rather
// than deriving success locally from publication artifacts.
assert_eq!(resource.state, run_state(&published.record));
assert_eq!(resource.state, ControlPlaneState::Failed);
assert!(resource.blocker.is_none());
assert_eq!(resource.publication.unwrap().state, "review_ready");
}
Expand Down
Loading
Loading