diff --git a/dialects/hir/src/transforms/spill.rs b/dialects/hir/src/transforms/spill.rs index b65c9cac1..fad49cff1 100644 --- a/dialects/hir/src/transforms/spill.rs +++ b/dialects/hir/src/transforms/spill.rs @@ -1,4 +1,4 @@ -use alloc::rc::Rc; +use alloc::{format, rc::Rc}; use midenc_hir::{ BlockRef, BuilderExt, EntityMut, Op, OpBuilder, OperationName, OperationRef, Report, Rewriter, @@ -8,11 +8,16 @@ use midenc_hir::{ pass::{Pass, PassExecutionState, PostPassStatus}, }; use midenc_hir_analysis::analyses::SpillAnalysis; -use midenc_hir_transform::{self as transforms, ReloadLike, SpillLike, TransformSpillsInterface}; +use midenc_hir_transform::{self as transforms, SpillLike, TransformSpillsInterface}; #[derive(Default)] pub struct TransformSpills; +midenc_hir::inventory::submit!(::midenc_hir::pass::registry::PassInfo::new::( + "transform-spills", + "materialize operand stack spills as stores/loads of procedure locals" +)); + impl Pass for TransformSpills { type Target = Function; @@ -129,14 +134,18 @@ impl TransformSpillsInterface for TransformSpillsImpl { &mut self, rewriter: &mut dyn Rewriter, spill: OperationRef, + value: ValueRef, ) -> Result<(), Report> { use crate::HirOpBuilder; + // The local is keyed by the analysis's identity of the spilled value; the stored operand + // is the op's current one, which SSA reconstruction may have rewritten (e.g. to a + // preceding reload's result), but always carries the same runtime value. let spilled = spill.borrow().as_trait::().unwrap().spilled_value(); let mut function = self.function; - let local = *self.locals.entry(spilled).or_insert_with(|| { + let local = *self.locals.entry(value).or_insert_with(|| { let mut function = function.borrow_mut(); - function.alloc_local(spilled.borrow().ty().clone()) + function.alloc_local(value.borrow().ty().clone()) }); let store = rewriter.store_local(local, spilled, spill.span())?; @@ -150,11 +159,18 @@ impl TransformSpillsInterface for TransformSpillsImpl { &mut self, rewriter: &mut dyn Rewriter, reload: OperationRef, + value: ValueRef, ) -> Result<(), Report> { use crate::HirOpBuilder; - let spilled = reload.borrow().as_trait::().unwrap().spilled_value(); - let local = self.locals[&spilled]; + let Some(local) = self.locals.get(&value).copied() else { + let function = self.function.borrow(); + let function = function.get_name(); + return Err(Report::msg(format!( + "internal error: live reload of {value} in {function} has no corresponding spill: \ + every kept reload must be covered by at least one spill of the same value" + ))); + }; let reloaded = rewriter.load_local(local, reload.span())?; rewriter.replace_op_with_values(reload, &[Some(reloaded)]); diff --git a/dialects/hir/src/transforms/spill/tests.rs b/dialects/hir/src/transforms/spill/tests.rs index a775fd564..d9109721f 100644 --- a/dialects/hir/src/transforms/spill/tests.rs +++ b/dialects/hir/src/transforms/spill/tests.rs @@ -1,4 +1,4 @@ -use alloc::{format, string::ToString, sync::Arc}; +use alloc::{format, rc::Rc, string::ToString, sync::Arc}; use litcheck_filecheck::{filecheck, litcheck}; use midenc_dialect_arith::ArithOpBuilder; @@ -6,14 +6,54 @@ use midenc_dialect_cf::ControlFlowOpBuilder as Cf; use midenc_dialect_scf::StructuredControlFlowOpBuilder; use midenc_expect_test::expect_file; use midenc_hir::{ - AddressSpace, Builder, Op, PointerType, ProgramPoint, Report, SourceSpan, Type, ValueRef, - dialects::builtin::BuiltinOpBuilder, testing::Test, + AddressSpace, BlockRef, Builder, Context, Op, Operation, OperationRef, PointerType, + ProgramPoint, Reachability, Report, SourceSpan, Type, ValueRef, + diagnostics::Uri, + dialects::builtin::{BuiltinOpBuilder, FunctionRef}, + parse::{self, ParserConfig}, + testing::{Test, parse_function_fixpoint}, }; use crate::{HirOpBuilder, transforms::TransformSpills}; type TestResult = Result; +/// Shorthand for [Operation::reachability] in assertions. +fn reach(from: OperationRef, to: OperationRef) -> Reachability { + Operation::reachability(from, to) +} + +/// Returns the `index`-th block in the body of `function`. +fn block_at(function: FunctionRef, index: usize) -> BlockRef { + let function = function.borrow(); + function + .body() + .body() + .iter() + .nth(index) + .map(|block| block.as_block_ref()) + .expect("block index out of bounds") +} + +/// Returns the `index`-th operation in `block`. +fn op_at(block: BlockRef, index: usize) -> OperationRef { + let block = block.borrow(); + block + .body() + .iter() + .nth(index) + .map(|op| op.as_operation_ref()) + .expect("operation index out of bounds") +} + +/// Returns the `op_index`-th operation in the entry block of `op`'s `region_index`-th region. +fn op_in_region(op: OperationRef, region_index: usize, op_index: usize) -> OperationRef { + let op = op.borrow(); + let region = op.regions().iter().nth(region_index).expect("region index out of bounds"); + let entry = region.entry_block_ref().expect("expected region to have an entry block"); + op_at(entry, op_index) +} + /// Build a simple single-block function which triggers spills and reloads, /// then run the `TransformSpills` pass and check that spills/reloads are /// materialized as `hir.store_local`/`hir.load_local`. @@ -573,3 +613,541 @@ fn materializes_spills_nested_scf_while_after_region() -> TestResult<()> { Ok(()) } + +/// All spills of one value must share one procedure local, even when SSA reconstruction has +/// rewritten a spill's operand. +/// +/// We synthesize a `spill; reload; spill; reload` chain over a single value: the rewrite phase +/// redirects the second spill's operand to the first reload's result (spill operands are not +/// exempt from use rewriting), so pairing the local slot by the op operand would allocate a +/// second local that no reload ever reads. Materialization must key locals by the value tracked +/// in the spills analysis, storing the (possibly rewritten) operand into the shared slot. +#[test] +fn materializes_spills_shared_local_for_rewritten_spill() -> TestResult<()> { + let mut test = + Test::named("materializes_spills_shared_local_for_rewritten_spill").in_module("test"); + let span = SourceSpan::UNKNOWN; + + test.with_function(test.name(), &[Type::U32], &[Type::U32]); + let func = test.function(); + + let (spilled_value, spill_points, reload_points) = { + let mut b = test.function_builder(); + let entry = b.current_block(); + let v0 = entry.borrow().arguments()[0] as ValueRef; + let k1 = b.u32(1, span); + let v = b.add_unchecked(v0, k1, span)?; + // Anchor operations to place the spill/reload pseudo-ops in front of + let a1 = b.add_unchecked(v0, v0, span)?; + let a2 = b.add_unchecked(a1, k1, span)?; + let a3 = b.add_unchecked(a2, k1, span)?; + let a4 = b.add_unchecked(a3, k1, span)?; + // The post-reload use of the spilled value + let out = b.add_unchecked(v, a4, span)?; + b.ret(Some(out), span)?; + + let anchor = |value: ValueRef| { + ProgramPoint::before( + value.borrow().get_defining_op().expect("expected anchor to have a defining op"), + ) + }; + (v, [anchor(a1), anchor(a3)], [anchor(a2), anchor(a4)]) + }; + + let mut analysis = midenc_hir_analysis::analyses::SpillAnalysis::default(); + analysis.spilled.insert(spilled_value); + for (index, place) in spill_points.into_iter().enumerate() { + analysis.spills.push(midenc_hir_analysis::analyses::spills::SpillInfo { + id: midenc_hir_analysis::analyses::spills::Spill::new(index), + place: midenc_hir_analysis::analyses::spills::Placement::At(place), + value: spilled_value, + span, + inst: None, + }); + } + for (index, place) in reload_points.into_iter().enumerate() { + analysis.reloads.push(midenc_hir_analysis::analyses::spills::ReloadInfo { + id: midenc_hir_analysis::analyses::spills::Reload::new(index), + place: midenc_hir_analysis::analyses::spills::Placement::At(place), + value: spilled_value, + span, + inst: None, + }); + } + + let mut interface = super::TransformSpillsImpl { + function: func, + locals: Default::default(), + }; + let analysis_manager = midenc_hir::pass::AnalysisManager::new(func.as_operation_ref(), None); + midenc_hir_transform::transform_spills( + func.as_operation_ref(), + &mut analysis, + &mut interface, + analysis_manager, + )?; + + let after = func.as_operation_ref().borrow().to_string(); + std::println!("{after}"); + + assert_eq!( + func.borrow().num_locals(), + 1, + "all spills of one value must share one procedure local\n{after}" + ); + // The second store must write the first reload's result into the same local, proving the + // rewritten operand landed in the shared slot + filecheck!( + &after, + r#" +; CHECK: hir.store_local %{{\d+}} <{ local = #builtin.local_variable<[[L:\d+]], u32> }> +; CHECK: %[[R0:\d+]] = hir.load_local <{ local = #builtin.local_variable<[[L]], u32> }> +; CHECK: hir.store_local %[[R0]] <{ local = #builtin.local_variable<[[L]], u32> }> +; CHECK: hir.load_local <{ local = #builtin.local_variable<[[L]], u32> }> +"# + ); + + Ok(()) +} + +/// Positional reachability over a diamond CFG. +/// +/// This pins the join-covered shape that dominance-based spill pruning got wrong: a reload after +/// the join is covered by the set of per-arm spills, none of which dominates it, so each arm must +/// count as (maybe) reaching the join. Sibling arms must stay provably unreachable from each +/// other so the dead-edge-spill elimination keeps working. +#[test] +fn reachability_in_branching_cfg() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_in_branching_cfg(%a: u32) -> u32 { + %one = arith.constant 1 : u32; + %first = arith.add %a, %one <{ overflow = #builtin.overflow }>; + %cond = arith.eq %first, %one; + cf.cond_br %cond ^left, ^right : (i1); +^left: + %lv = arith.add %first, %one <{ overflow = #builtin.overflow }>; + cf.br ^join(%lv : u32); +^right: + %rv = arith.add %first, %first <{ overflow = #builtin.overflow }>; + cf.br ^join(%rv : u32); +^join(%j: u32): + %jv = arith.add %j, %one <{ overflow = #builtin.overflow }>; + builtin.ret %jv : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = + parse_function_fixpoint(&context, "reachability_in_branching_cfg.hir", source)?; + + let entry = block_at(function, 0); + let entry_first = op_at(entry, 1); + let entry_second = op_at(entry, 2); + let left_op = op_at(block_at(function, 1), 0); + let right_op = op_at(block_at(function, 2), 0); + let join_op = op_at(block_at(function, 3), 0); + + assert_eq!(reach(left_op, join_op), Reachability::Maybe, "arm must reach the join"); + assert_eq!(reach(right_op, join_op), Reachability::Maybe, "arm must reach the join"); + assert_eq!( + reach(left_op, right_op), + Reachability::Impossible, + "sibling arms must not reach each other" + ); + assert_eq!( + reach(entry_first, entry_second), + Reachability::Maybe, + "adjacency alone does not prove that the source operation falls through" + ); + assert_eq!( + reach(op_at(entry, 0), entry_second), + Reachability::Maybe, + "an intervening op may diverge, so non-adjacent positions are only Maybe" + ); + assert_eq!( + reach(entry_second, entry_first), + Reachability::Impossible, + "later op must not reach an earlier op without a cycle" + ); + assert_eq!(reach(join_op, left_op), Reachability::Impossible, "join must not reach an arm"); + Ok(()) +} + +/// Positional reachability through a loop back-edge. +#[test] +fn reachability_through_loop_back_edge() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_through_loop_back_edge(%n: u32) -> u32 { + cf.br ^header(%n : u32); +^header(%i: u32): + %one = arith.constant 1 : u32; + %next = arith.add %i, %one <{ overflow = #builtin.overflow }>; + %cond = arith.eq %next, %one; + cf.cond_br %cond ^body, ^exit : (i1); +^body: + %step = arith.add %next, %one <{ overflow = #builtin.overflow }>; + cf.br ^header(%step : u32); +^exit: + %out = arith.add %next, %next <{ overflow = #builtin.overflow }>; + builtin.ret %out : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = + parse_function_fixpoint(&context, "reachability_through_loop_back_edge.hir", source)?; + + let header = block_at(function, 1); + let header_op = op_at(header, 1); + let header_second = op_at(header, 2); + let body_op = op_at(block_at(function, 2), 0); + let exit_op = op_at(block_at(function, 3), 0); + + assert_eq!( + reach(body_op, header_op), + Reachability::Maybe, + "back edge must reach the header" + ); + assert_eq!( + reach(header_second, header_op), + Reachability::Maybe, + "later op must reach an earlier op in the same block through the loop" + ); + assert_eq!( + reach(exit_op, body_op), + Reachability::Impossible, + "exit must not reach the loop body" + ); + Ok(()) +} + +/// Positional reachability with operations nested in `scf.if` regions. +/// +/// Nested operations are normalized to their ancestor in the common region; positions involving +/// a normalized side are at most [Reachability::Maybe], since entering a sub-region depends on +/// the region op's semantics, and operations nested under the same region-branch op +/// conservatively reach each other (e.g. across loop iterations). +#[test] +fn reachability_across_nested_regions() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_across_nested_regions(%a: u32) -> u32 { + %one = arith.constant 1 : u32; + %pre = arith.add %a, %one <{ overflow = #builtin.overflow }>; + %cond = arith.eq %pre, %one; + %r = scf.if %cond then { + %t = arith.add %pre, %one <{ overflow = #builtin.overflow }>; + scf.yield %t : (u32); + } else { + %e = arith.add %pre, %pre <{ overflow = #builtin.overflow }>; + scf.yield %e : (u32); + } : (i1) -> (u32); + %post = arith.add %r, %one <{ overflow = #builtin.overflow }>; + builtin.ret %post : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = + parse_function_fixpoint(&context, "reachability_across_nested_regions.hir", source)?; + + let entry = block_at(function, 0); + let pre_op = op_at(entry, 1); + let if_op = op_at(entry, 3); + let post_op = op_at(entry, 4); + let then_op = op_in_region(if_op, 0, 0); + let else_op = op_in_region(if_op, 1, 0); + + assert_eq!( + reach(pre_op, then_op), + Reachability::Maybe, + "op before the region op must reach into it" + ); + assert_eq!( + reach(then_op, post_op), + Reachability::Maybe, + "nested op must reach past the region op" + ); + assert_eq!( + reach(post_op, then_op), + Reachability::Impossible, + "op after the region op must not reach back into it" + ); + assert_eq!( + reach(then_op, else_op), + Reachability::Impossible, + "mutually exclusive arms of an if outside any loop cannot reach each other" + ); + assert_eq!( + reach(if_op, then_op), + Reachability::Maybe, + "a region op may reach the operations nested within it" + ); + assert_eq!( + reach(then_op, if_op), + Reachability::Maybe, + "a nested operation flows back through its region op" + ); + Ok(()) +} + +/// Positional reachability through re-entry of a repetitive region. +/// +/// A repetitive region (e.g. the `before` region of an `scf.while`) has no CFG back edge: its +/// block ends in a region terminator with no block successors, and the loop is expressed in the +/// region graph of the parent op. An op positioned after another op in such a region still +/// reaches it, through the next iteration. +#[test] +fn reachability_through_region_re_entry() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_through_region_re_entry(%n: u32) -> u32 { + %zero = arith.constant 0 : u32; + %r = scf.while %zero before { + ^head(%i: u32): + %one = arith.constant 1 : u32; + %early = arith.add %i, %one <{ overflow = #builtin.overflow }>; + %late = arith.add %early, %early <{ overflow = #builtin.overflow }>; + %continue = arith.lt %late, %n; + scf.condition %continue, %late : (i1, u32); + } after { + ^body(%j: u32): + %next = arith.incr %j; + scf.yield %next : (u32); + } : (u32) -> u32; + builtin.ret %r : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = + parse_function_fixpoint(&context, "reachability_through_region_re_entry.hir", source)?; + + let entry = block_at(function, 0); + let while_op = op_at(entry, 1); + let ret_op = op_at(entry, 2); + let early_op = op_in_region(while_op, 0, 1); + let late_op = op_in_region(while_op, 0, 2); + + assert_eq!( + reach(late_op, early_op), + Reachability::Maybe, + "a later op must reach an earlier op in the same repetitive region through re-entry" + ); + assert_eq!( + reach(early_op, late_op), + Reachability::Maybe, + "same-block order does not prove that every intervening operation falls through" + ); + assert_eq!( + reach(while_op, ret_op), + Reachability::Maybe, + "an adjacent while may never exit, so reaching its successor is not guaranteed" + ); + assert_eq!( + reach(ret_op, late_op), + Reachability::Impossible, + "an op after the loop must not reach into it" + ); + let after_op = op_in_region(while_op, 1, 0); + assert_eq!( + reach(late_op, after_op), + Reachability::Maybe, + "the after region is reachable from the before region in the while region graph" + ); + assert_eq!( + reach(after_op, early_op), + Reachability::Maybe, + "the before region is reachable from the after region through the loop" + ); + assert_eq!( + reach(after_op, while_op), + Reachability::Maybe, + "leaving the after region can traverse after -> before -> parent" + ); + Ok(()) +} + +/// Positional reachability through re-execution of a region whose owner sits on a CFG cycle. +/// +/// The `then` region of the `scf.if` is not repetitive in the op's own region graph, but the +/// block hosting the `scf.if` lies on a CFG cycle, so every iteration re-enters the region: an +/// op positioned after another op inside it still reaches it, through the next iteration. +#[test] +fn reachability_through_cfg_cycle_re_entry() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_through_cfg_cycle_re_entry(%n: u32) -> u32 { + %zero = arith.constant 0 : u32; + cf.br ^head(%zero : u32); +^head(%i: u32): + %one = arith.constant 1 : u32; + %c = arith.eq %i, %one; + %r = scf.if %c then { + %early = arith.add %i, %one <{ overflow = #builtin.overflow }>; + %late = arith.add %early, %early <{ overflow = #builtin.overflow }>; + scf.yield %late : (u32); + } else { + scf.yield %i : (u32); + } : (i1) -> (u32); + %done = arith.eq %r, %n; + cf.cond_br %done ^exit, ^head(%r : u32) : (i1); +^exit: + builtin.ret %r : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = + parse_function_fixpoint(&context, "reachability_through_cfg_cycle_re_entry.hir", source)?; + + let head = block_at(function, 1); + let if_op = op_at(head, 2); + let early_op = op_in_region(if_op, 0, 0); + let late_op = op_in_region(if_op, 0, 1); + let ret_op = op_at(block_at(function, 2), 0); + + assert_eq!( + reach(late_op, early_op), + Reachability::Maybe, + "re-entry of a region through an outer CFG cycle must count as reachable" + ); + assert_eq!( + reach(ret_op, early_op), + Reachability::Impossible, + "an op after the loop must not reach into it" + ); + let else_op = op_in_region(if_op, 1, 0); + assert_eq!( + reach(early_op, else_op), + Reachability::Maybe, + "sibling if arms stay reachable when an outer CFG cycle re-executes the if" + ); + Ok(()) +} + +/// Enclosure reachability is directional and must respect the function body's CFG. +/// +/// A reachable sink can be entered from the function but cannot return to it. Conversely, a +/// disconnected block cannot be entered from the function, although an operation already in that +/// block can follow its own return. +#[test] +fn reachability_across_directional_function_enclosure() -> TestResult<()> { + let source = r#"builtin.function public extern("C") @reachability_across_directional_function_enclosure(%a: u32, %cond: i1) -> u32 { + cf.cond_br %cond ^sink, ^exit : (i1); +^sink: + %sink_value = arith.add %a, %a <{ overflow = #builtin.overflow }>; + cf.br ^sink; +^exit: + %exit_value = arith.add %a, %a <{ overflow = #builtin.overflow }>; + builtin.ret %exit_value : (u32); +^dead: + %dead_value = arith.add %a, %a <{ overflow = #builtin.overflow }>; + builtin.ret %dead_value : (u32); +};"#; + + let context = Rc::new(Context::default()); + let (function, _) = parse_function_fixpoint( + &context, + "reachability_across_directional_function_enclosure.hir", + source, + )?; + + let function_op = function.as_operation_ref(); + let sink_op = op_at(block_at(function, 1), 0); + let exit_op = op_at(block_at(function, 2), 0); + let dead_op = op_at(block_at(function, 3), 0); + + assert_eq!( + reach(function_op, sink_op), + Reachability::Maybe, + "the function entry can flow into the reachable sink" + ); + assert_eq!( + reach(sink_op, function_op), + Reachability::Impossible, + "the sink SCC has no path back to the enclosing function" + ); + assert_eq!( + reach(function_op, dead_op), + Reachability::Impossible, + "the function entry cannot reach a disconnected block" + ); + assert_eq!( + reach(dead_op, function_op), + Reachability::Maybe, + "a position in the disconnected block can still reach its own return" + ); + assert_eq!( + reach(function_op, exit_op), + Reachability::Maybe, + "the ordinary exit block remains reachable from function entry" + ); + assert_eq!( + reach(exit_op, function_op), + Reachability::Maybe, + "the ordinary exit block reaches the enclosing function return" + ); + + Ok(()) +} + +/// Queries that leave a single function's control flow have no positional answer. +/// +/// Operations in two different functions can only be related interprocedurally, and for the +/// function operations themselves the module body is a graph-like region where operation order +/// does not define control flow. Both classifications are direction-independent. +#[test] +fn reachability_across_functions_and_graph_regions() -> TestResult<()> { + let source = r#"builtin.module public @test { + builtin.function public extern("C") @first(%a: u32) -> u32 { + %r = arith.add %a, %a <{ overflow = #builtin.overflow }>; + builtin.ret %r : (u32); + }; + builtin.function public extern("C") @second(%b: u32) -> u32 { + %r = arith.add %b, %b <{ overflow = #builtin.overflow }>; + builtin.ret %r : (u32); + }; +};"#; + + let context = Rc::new(Context::default()); + let module_op = parse::parse_any( + ParserConfig::new(context.clone()), + Uri::new("reachability_across_functions_and_graph_regions.hir"), + source, + )?; + + let first_fn = op_in_region(module_op, 0, 0); + let second_fn = op_in_region(module_op, 0, 1); + let in_first = op_in_region(first_fn, 0, 0); + let in_second = op_in_region(second_fn, 0, 0); + + assert_eq!( + reach(in_second, in_first), + Reachability::MaybeInterprocedurally, + "ops in different functions relate only interprocedurally, regardless of module order" + ); + assert_eq!( + reach(in_first, in_second), + Reachability::MaybeInterprocedurally, + "ops in different functions relate only interprocedurally" + ); + assert_eq!( + reach(second_fn, first_fn), + Reachability::Indeterminate, + "graph-region op order does not define control flow, regardless of module order" + ); + assert_eq!( + reach(first_fn, second_fn), + Reachability::Indeterminate, + "graph-region op order does not define control flow" + ); + assert_eq!( + reach(first_fn, in_first), + Reachability::Maybe, + "enclosure is intra-procedural even when the encloser resides in a graph region" + ); + assert_eq!( + reach(in_first, first_fn), + Reachability::Maybe, + "enclosure is intra-procedural in both directions" + ); + assert_eq!( + reach(module_op, first_fn), + Reachability::Indeterminate, + "enclosure crossing a graph-like region defines no control-flow order" + ); + assert_eq!( + reach(in_first, module_op), + Reachability::Indeterminate, + "enclosure crossing a graph-like region is indeterminate in both directions" + ); + Ok(()) +} diff --git a/hir-transform/src/spill.rs b/hir-transform/src/spill.rs index 74cad1ab0..9453291da 100644 --- a/hir-transform/src/spill.rs +++ b/hir-transform/src/spill.rs @@ -1,9 +1,10 @@ -use alloc::{collections::VecDeque, rc::Rc}; +use alloc::{collections::VecDeque, format, rc::Rc}; use midenc_hir::{ BlockRef, Builder, Context, FxHashMap, OpBuilder, OpOperand, Operation, OperationRef, - ProgramPoint, Region, RegionBranchOpInterface, RegionBranchPoint, RegionRef, Report, Rewriter, - SmallVec, SourceSpan, Spanned, StorableEntity, TraceTarget, Usable, ValueRange, ValueRef, + ProgramPoint, Reachability, ReachabilityCache, Region, RegionBranchOpInterface, + RegionBranchPoint, RegionRef, Report, Rewriter, SmallVec, SourceSpan, Spanned, StorableEntity, + TraceTarget, Usable, ValueRange, ValueRef, adt::{SmallDenseMap, SmallSet}, cfg::Graph, dominance::{DomTreeNode, DominanceFrontier, DominanceInfo}, @@ -46,18 +47,27 @@ pub trait TransformSpillsInterface { /// Convert `spill`, a [SpillLike] operation, into a primitive memory store of the spilled /// value. + /// + /// `value` is the spilled value as tracked by the spills analysis; implementations must key + /// any per-value storage by it, while storing the operation's current operand (the live SSA + /// name at that point, which SSA reconstruction may have rewritten). fn convert_spill_to_store( &mut self, rewriter: &mut dyn Rewriter, spill: OperationRef, + value: ValueRef, ) -> Result<(), Report>; /// Convert `reload`, a [ReloadLike] operation, into a primitive memory load of the spilled /// value. + /// + /// `value` is the spilled value as tracked by the spills analysis; the lookup must use the + /// same key as [Self::convert_spill_to_store]. fn convert_reload_to_load( &mut self, rewriter: &mut dyn Rewriter, reload: OperationRef, + value: ValueRef, ) -> Result<(), Report>; } @@ -79,11 +89,13 @@ pub trait SpillLike { /// An operation trait for operations that implement reload-like behavior for purposes of the /// spills transformation/rewrite. /// -/// A reload-like operation is expected to take a single value, for which a dominating [SpillLike] -/// op exists, and produce a new, unique SSA value corresponding to the reloaded spill value. The -/// spills transformation will handle rewriting any uses of the [SpillLike] and [ReloadLike] ops -/// such that they are not present after the transformation, in conjunction with an implementation -/// of the [TransformSpillsInterface]. +/// A reload-like operation is expected to take a single value, for which at least one [SpillLike] +/// op of that value executes on every control-flow path reaching the reload (a reload after a +/// join may be covered by a set of per-path spills, none of which individually dominates it), and +/// produce a new, unique SSA value corresponding to the reloaded spill value. The spills +/// transformation will handle rewriting any uses of the [SpillLike] and [ReloadLike] ops such +/// that they are not present after the transformation, in conjunction with an implementation of +/// the [TransformSpillsInterface]. pub trait ReloadLike { /// Returns the operand corresponding to the spilled value fn spilled(&self) -> OpOperand; @@ -107,7 +119,9 @@ pub trait ReloadLike { /// * Rewrites `op` such that all uses of a spilled value dominated by a reload, are rewritten to /// use that reload, or in the case of crossing a dominance frontier, a materialized block /// argument/phi representing the closest definition of that value from each predecessor. -/// * Rewrites all spill and reload instructions to their primitive memory store/load ops +/// * Rewrites all spill and reload instructions to their primitive memory store/load ops. +/// Dominance governs only the SSA use rewrite above; whether a spill is materialized or elided +/// is decided by reachability to live reloads (see `rewrite_spill_pseudo_instructions`). pub fn transform_spills( op: OperationRef, analysis: &mut SpillAnalysis, @@ -454,7 +468,7 @@ fn rewrite_single_block_spills( } let context = { op.borrow().context_rc() }; - rewrite_spill_pseudo_instructions(context, analysis, interface, None, trace_target) + rewrite_spill_pseudo_instructions(context, analysis, interface, trace_target) } fn rewrite_cfg_spills( @@ -570,7 +584,7 @@ fn rewrite_cfg_spills( used_sets.insert(block_ref, used); } - rewrite_spill_pseudo_instructions(context, analysis, interface, Some(dominfo), trace_target) + rewrite_spill_pseudo_instructions(context, analysis, interface, trace_target) } /// Rewrite uses of spilled values in `op` and any nested regions of `op`. @@ -838,64 +852,89 @@ fn rewrite_inserted_phi_uses( /// /// However, this produces dead spills on some paths through the function, which are not /// needed once rewrites have been performed. So we eliminate dead spills by identifying -/// those spills which do not dominate any reloads - if a store to a spill slot can never -/// be read, then the store can be elided. +/// those spills which cannot reach any live reload of their value - if a store to a spill +/// slot can never be read, then the store can be elided. +/// +/// Reachability, not dominance, is the criterion: the spills analysis places spills per path +/// (e.g. along each edge of a join), so a reload after the join is covered by a set of spills, +/// none of which individually dominates it. Eliding a spill is only sound when it provably +/// cannot reach any live reload of its value ([Reachability::Impossible]), otherwise some path +/// would reload from a local that was never written. A spill/reload pair related across +/// functions, or through a graph-like region, is invalid IR and reported as an error. fn rewrite_spill_pseudo_instructions( context: Rc, analysis: &mut SpillAnalysis, interface: &mut dyn TransformSpillsInterface, - dominfo: Option<&DominanceInfo>, trace_target: &TraceTarget, ) -> Result<(), Report> { - use midenc_hir::{ - dominance::Dominates, - patterns::{RewriterImpl, TracingRewriterListener}, - }; + use midenc_hir::patterns::{RewriterImpl, TracingRewriterListener}; let mut builder = RewriterImpl::::new(context) .with_listener(TracingRewriterListener); + + // Index the live reloads by their spilled value once, so each spill only considers the + // reloads it can possibly cover. Spills and reloads are paired through the analysis's value + // bookkeeping rather than the spill op's current operand, which SSA reconstruction may + // rewrite (only reload operands are exempt). Liveness is snapshotted before any spills are + // erased, which errs toward keeping a spill whose reload only dies as part of the erasure + // cascade below. + let mut live_reloads = SmallDenseMap::, 8>::default(); + for rinfo in analysis.reloads() { + let Some(reload_op) = rinfo.inst else { + continue; + }; + let reload_used = { + let rop = reload_op.borrow(); + let rl = rop + .as_trait::() + .expect("expected materialized reload op to implement ReloadLike"); + rl.reloaded().borrow().is_used() + }; + if reload_used { + live_reloads.entry(rinfo.value).or_default().push(reload_op); + } + } + + // One reachability cache is shared across all pairings: pruning only erases operations, + // never blocks, so the cached block-reachability stays valid throughout. + let mut reachability = ReachabilityCache::default(); for spill in analysis.spills() { let operation = spill.inst.expect("expected spill to have been materialized"); - let spilled = { - let op = operation.borrow(); - let spill_like = op - .as_trait::() - .expect("expected materialized spill operation to implement SpillLike"); - spill_like.spilled_value() - }; - // Only keep spills that feed a live reload dominated by this spill + // Only keep spills that can reach a live reload of their value let mut is_used = false; - for rinfo in analysis.reloads() { - if rinfo.value != spilled { - continue; - } - let Some(reload_op) = rinfo.inst else { - continue; - }; - let (reload_used, dom_ok) = { - let rop = reload_op.borrow(); - let rl = rop - .as_trait::() - .expect("expected materialized reload op to implement ReloadLike"); - let used = rl.reloaded().borrow().is_used(); - let dom_ok = match dominfo { - None => true, - Some(dominfo) => { - let sop = operation.borrow(); - sop.dominates(&rop, dominfo) - } - }; - (used, dom_ok) - }; - if reload_used && dom_ok { - is_used = true; - break; + for reload_op in live_reloads + .get(&spill.value) + .map(|reloads| reloads.as_slice()) + .unwrap_or_default() + { + let reload_op = *reload_op; + match Operation::reachability_cached(operation, reload_op, &mut reachability) { + Reachability::Guaranteed | Reachability::Maybe => { + is_used = true; + break; + } + // This spill cannot cover the reload; other spills of the value may. + Reachability::Impossible => {} + Reachability::MaybeInterprocedurally => { + return Err(Report::msg(format!( + "internal error: a spill of {} and a reload of it are in different \ + functions", + spill.value + ))); + } + Reachability::Indeterminate => { + return Err(Report::msg(format!( + "internal error: control flow between a spill of {} and a reload of it is \ + not well-defined", + spill.value + ))); + } } } if is_used { builder.set_insertion_point_after(operation); - interface.convert_spill_to_store(&mut builder, operation)?; + interface.convert_spill_to_store(&mut builder, operation, spill.value)?; } else { builder.erase_op(operation); } @@ -921,7 +960,7 @@ fn rewrite_spill_pseudo_instructions( reload.place ); builder.set_insertion_point_after(operation); - interface.convert_reload_to_load(&mut builder, operation)?; + interface.convert_reload_to_load(&mut builder, operation, reload.value)?; } else { log::trace!( target: trace_target, diff --git a/hir/src/ir.rs b/hir/src/ir.rs index 6fc32c497..31ad1931f 100644 --- a/hir/src/ir.rs +++ b/hir/src/ir.rs @@ -15,6 +15,7 @@ mod operands; mod operation; pub mod parse; pub mod print; +mod reachability; mod region; mod successor; pub(crate) mod symbols; @@ -64,6 +65,7 @@ pub use self::{ }, parse::{OpAsmParser, OpParser, ParseResult}, print::{AttrPrinter, OpPrinter, OpPrintingFlags}, + reachability::{Reachability, ReachabilityCache}, region::{ InvocationBounds, LoopLikeOpInterface, Region, RegionBranchOpInterface, RegionBranchPoint, RegionBranchTerminatorOpInterface, RegionCursor, RegionCursorMut, RegionKind, diff --git a/hir/src/ir/reachability.rs b/hir/src/ir/reachability.rs new file mode 100644 index 000000000..dbd4a2c06 --- /dev/null +++ b/hir/src/ir/reachability.rs @@ -0,0 +1,852 @@ +use crate::{ + AttributeRef, BlockRef, CallableOpInterface, FxHashMap, FxHashSet, Operation, OperationRef, + Region, RegionBranchOpInterface, RegionBranchPoint, RegionBranchTerminatorOpInterface, + RegionKindInterface, RegionRef, SmallVec, + cfg::Graph, + traits::{NoTerminator, ReturnLike}, +}; + +/// The answer to a control-flow reachability query between two operations. +/// +/// See [Operation::reachability]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reachability { + /// Provably unreachable, i.e. no control flow path exists between `a` and `b` + Impossible, + /// Provably reachable, i.e. there is at least one control flow path guaranteed to reach + /// from `a` to `b` + Guaranteed, + /// Reachability is not proven, but there is at least one control flow path that reaches + /// from `a` to `b`; full reachability analysis is required to prove whether the path(s) are + /// truly executable + Maybe, + /// Cannot be determined without global reachability analysis, because the two ops are in + /// different functions + MaybeInterprocedurally, + /// Cannot be determined because control flow between the two ops is not well-defined (i.e. + /// both belong to a graph-like region, or their common ancestor region is graph-like) + Indeterminate, +} + +/// A lazily-populated cache of forward block reachability, for callers issuing many +/// [Operation::reachability_cached] queries over one body of IR. +/// +/// The first query from a given block computes and stores that block's full forward closure; +/// subsequent queries from the same block are set lookups. The cache assumes the block +/// structure of the IR does not change between queries: discard it after splitting, erasing, +/// or rewiring blocks. +#[derive(Default)] +pub struct ReachabilityCache { + forward: FxHashMap>, +} + +impl ReachabilityCache { + /// Returns true if control leaving the end of `from` can reach the start of `to` by + /// following block successors. + /// + /// The walk is not reflexive: `from == to` returns true only when a cycle leads back into + /// the block, which is what [region_can_re_execute] relies on for cycle detection. + fn leads_to(&mut self, from: BlockRef, to: BlockRef) -> bool { + self.forward + .entry(from) + .or_insert_with(|| { + let mut reachable = FxHashSet::default(); + let mut worklist = SmallVec::<[BlockRef; 8]>::from_iter(BlockRef::children(from)); + while let Some(block) = worklist.pop() { + if reachable.insert(block) { + worklist.extend(BlockRef::children(block)); + } + } + reachable + }) + .contains(&to) + } +} + +/// Queries +impl Operation { + /// Computes whether some control-flow path from `from` can reach `to`. + /// + /// This is a conservative, purely structural query over the CFG and region graph: it does + /// not consider whether paths are actually executable at runtime (that requires a proper + /// reachability analysis performed in concert with SCCP/DCE), so paths that exist but may + /// never execute are reported as [Reachability::Maybe]. The precise answers are reliable in + /// both directions: [Reachability::Impossible] means no control-flow path exists at all, + /// and [Reachability::Guaranteed] means a path exists that control cannot branch away from. + /// + /// Queries that relate positions in different functions, or in regions where operation + /// order does not define control flow, are answered with + /// [Reachability::MaybeInterprocedurally] and [Reachability::Indeterminate] respectively, + /// leaving the interpretation of such queries to the caller. + pub fn reachability(from: OperationRef, to: OperationRef) -> Reachability { + Self::reachability_cached(from, to, &mut ReachabilityCache::default()) + } + + /// Like [Operation::reachability], reusing `cache` across queries. + /// + /// Use this when issuing many queries over one body of IR (e.g. pairing spills with + /// reloads), so that block-reachability walks are shared between them. + pub fn reachability_cached( + from: OperationRef, + to: OperationRef, + cache: &mut ReachabilityCache, + ) -> Reachability { + // One operation enclosing the other relates the two positions through the chain of + // regions between them: control entering the enclosing op can reach the nested + // position, and control leaving the nested position flows back through the enclosing + // op — unless the chain crosses a graph-like region (e.g. a module enclosing a + // function), where no control-flow order is defined. This must precede the scope + // comparison, which would otherwise misclassify enclosure by an op residing in a + // graph-like region as interprocedural. + if let Some(result) = enclosure_reachability(from, to, EnclosureDirection::Entering, cache) + { + return result; + } + if let Some(result) = enclosure_reachability(to, from, EnclosureDirection::Exiting, cache) { + return result; + } + + // Intra-procedural reasoning is bounded by the nearest ancestor residing in a + // graph-like region (in practice, the enclosing function): positions under different + // such ancestors can only be related interprocedurally. + if control_flow_scope(from) != control_flow_scope(to) { + return Reachability::MaybeInterprocedurally; + } + + // Without a common ancestor region no control-flow path can exist: any path from `from` + // to `to` would itself lie in a region containing both. + let Some(common_region) = Region::find_common_ancestor(&[from, to]) else { + return Reachability::Impossible; + }; + + // In a graph-like region operation order does not define control flow, so positional + // queries within it are meaningless. + if !region_has_ssa_dominance(common_region) { + return Reachability::Indeterminate; + } + + // Normalize both operations to the common region: an operation nested in a sub-region + // (e.g. structured control flow) is represented by its ancestor op in the common region. + let common_region_ref = common_region; + let common_region = common_region.borrow(); + let (Some(from_ancestor), Some(to_ancestor)) = + (common_region.find_ancestor_op(from), common_region.find_ancestor_op(to)) + else { + // Unreachable per find_common_ancestor's postcondition (the returned region + // contains every queried op); kept as a defensive fallback. + return Reachability::Maybe; + }; + + // Both operations normalize to the same ancestor op, i.e. they sit in different + // sub-regions of it (enclosure was handled above). Whether control can transfer from + // one sub-region to a sibling is decided by the op's own region graph, or by the op + // executing more than once. + if from_ancestor == to_ancestor { + return sibling_region_reachability(from_ancestor, from, to, cache); + } + + let (Some(from_block), Some(to_block)) = + (from_ancestor.borrow().parent(), to_ancestor.borrow().parent()) + else { + return Reachability::Maybe; + }; + + // Within one block an earlier operation may flow into a later one, but order alone does + // not prove total fallthrough: the source or an intervening operation may loop + // indefinitely, return from the function, or abort. Positions that were normalized are + // likewise conditional on their ancestor op's semantics. + if from_block == to_block && from_ancestor.borrow().is_before_in_block(&to_ancestor) { + return Reachability::Maybe; + } + + // A forward path may exist through block successors; earlier positions are only + // reachable through a cycle, either via block successors, or by re-entry of the common + // region itself. + if cache.leads_to(from_block, to_block) { + return Reachability::Maybe; + } + if region_can_re_execute(common_region_ref, cache) { + return Reachability::Maybe; + } + + Reachability::Impossible + } +} + +/// Returns the nearest proper ancestor of `op` that resides in a graph-like region (or has no +/// parent at all), i.e. the operation whose body bounds any intra-procedural control-flow +/// reasoning about `op`. In practice this is the enclosing function, whose parent module body +/// is a graph-like region. +fn control_flow_scope(op: OperationRef) -> Option { + let mut current = op.borrow().parent_op(); + while let Some(ancestor) = current { + let Some(parent_block) = ancestor.borrow().parent() else { + return Some(ancestor); + }; + if !parent_block.borrow().has_ssa_dominance() { + return Some(ancestor); + } + current = ancestor.borrow().parent_op(); + } + None +} + +#[derive(Clone, Copy)] +enum EnclosureDirection { + Entering, + Exiting, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum BoundaryReachability { + CanCross, + CannotCross, + Unknown, +} + +struct EnclosureStep { + owner: OperationRef, + region: RegionRef, + position: OperationRef, +} + +struct RegionExitAnalysis { + successors: SmallVec<[RegionBranchPoint; 4]>, + has_unknown_exit: bool, +} + +enum RegionGraphStart { + Parent, + Child { region: RegionRef, block: BlockRef }, +} + +/// If `ancestor` properly encloses `descendant`, classify whether control can enter or leave the +/// descendant position through every SSA region separating the two operations. +/// +/// A graph-like region makes the positional query indeterminate. For SSA regions, `Impossible` +/// is returned only when a fully modeled CFG or region boundary proves that no path exists; +/// incomplete terminator or owner semantics remain conservatively `Maybe`. +fn enclosure_reachability( + ancestor: OperationRef, + descendant: OperationRef, + direction: EnclosureDirection, + cache: &mut ReachabilityCache, +) -> Option { + if !ancestor.borrow().is_proper_ancestor_of(&descendant.borrow()) { + return None; + } + + // Collect the direct owner/child-region boundaries from the descendant out to the ancestor. + // `position` is always directly contained in `region`, which gives each boundary analysis a + // concrete starting or target block. + let mut steps = SmallVec::<[EnclosureStep; 4]>::new(); + let mut position = descendant; + loop { + let Some(region) = position.borrow().parent_region() else { + // Defensive fallback for malformed ancestry. + return Some(Reachability::Maybe); + }; + if !region_has_ssa_dominance(region) { + return Some(Reachability::Indeterminate); + } + let Some(owner) = region.parent() else { + return Some(Reachability::Maybe); + }; + steps.push(EnclosureStep { + owner, + region, + position, + }); + if owner == ancestor { + break; + } + position = owner; + } + + match direction { + EnclosureDirection::Entering => { + for step in steps.iter().rev() { + match region_entry_reachability(step, cache) { + BoundaryReachability::CanCross => {} + BoundaryReachability::CannotCross => { + return Some(Reachability::Impossible); + } + BoundaryReachability::Unknown => return Some(Reachability::Maybe), + } + } + } + EnclosureDirection::Exiting => { + for step in &steps { + match region_exit_reachability(step, cache) { + BoundaryReachability::CanCross => {} + BoundaryReachability::CannotCross => { + return Some(Reachability::Impossible); + } + BoundaryReachability::Unknown => return Some(Reachability::Maybe), + } + } + } + } + + Some(Reachability::Maybe) +} + +fn region_entry_reachability( + step: &EnclosureStep, + cache: &mut ReachabilityCache, +) -> BoundaryReachability { + let owner_is_region_branch = step.owner.borrow().implements::(); + let owner_is_callable = is_callable_body(step.owner, step.region); + if !owner_is_region_branch && !owner_is_callable { + return BoundaryReachability::Unknown; + } + + let Some(target_block) = step.position.borrow().parent() else { + return BoundaryReachability::Unknown; + }; + let Some(entry_block) = step.region.borrow().entry_block_ref() else { + return BoundaryReachability::CannotCross; + }; + if !block_reaches(entry_block, target_block, cache) { + return BoundaryReachability::CannotCross; + } + + if owner_is_region_branch { + return region_branch_reachability( + step.owner, + RegionGraphStart::Parent, + RegionBranchPoint::Child(step.region), + cache, + ); + } + + if owner_is_callable { + BoundaryReachability::CanCross + } else { + BoundaryReachability::Unknown + } +} + +fn region_exit_reachability( + step: &EnclosureStep, + cache: &mut ReachabilityCache, +) -> BoundaryReachability { + let Some(start_block) = step.position.borrow().parent() else { + return BoundaryReachability::Unknown; + }; + + if step.owner.borrow().implements::() { + return region_branch_reachability( + step.owner, + RegionGraphStart::Child { + region: step.region, + block: start_block, + }, + RegionBranchPoint::Parent, + cache, + ); + } + + if is_callable_body(step.owner, step.region) { + return callable_region_exit_reachability(step.owner, step.region, start_block, cache); + } + + BoundaryReachability::Unknown +} + +fn region_branch_reachability( + owner: OperationRef, + start: RegionGraphStart, + target: RegionBranchPoint, + cache: &mut ReachabilityCache, +) -> BoundaryReachability { + let mut worklist = SmallVec::<[(RegionRef, Option); 8]>::new(); + + match start { + RegionGraphStart::Parent => { + let successors = { + let owner = owner.borrow(); + let operands = unknown_operands(owner.num_operands()); + let branch = owner + .as_trait::() + .expect("expected a region branch operation"); + branch + .get_entry_successor_regions(&operands) + .map(RegionBranchPoint::from) + .collect::>() + }; + for successor in successors { + if successor == target { + return BoundaryReachability::CanCross; + } + if let RegionBranchPoint::Child(region) = successor { + worklist.push((region, None)); + } + } + } + RegionGraphStart::Child { region, block } => { + worklist.push((region, Some(block))); + } + } + + // A child reached from another child starts at its entry. The starting exit child instead + // starts at the descendant's block; track both states independently so a later region-graph + // cycle may legitimately re-enter that same child through its entry block. + let mut visited = SmallVec::<[(RegionRef, Option); 8]>::new(); + let mut has_unknown_path = false; + + while let Some((region, start_block)) = worklist.pop() { + if visited.contains(&(region, start_block)) { + continue; + } + visited.push((region, start_block)); + + let start_block = match start_block.or_else(|| region.borrow().entry_block_ref()) { + Some(block) => block, + None => { + has_unknown_path = true; + continue; + } + }; + let exits = reachable_region_exits(owner, region, start_block, cache); + has_unknown_path |= exits.has_unknown_exit; + + for successor in exits.successors { + if successor == target { + return BoundaryReachability::CanCross; + } + if let RegionBranchPoint::Child(region) = successor { + worklist.push((region, None)); + } + // Parent reached after a child is terminal for this execution of the region owner; + // it must not be expanded as though the operation were being entered again. + } + } + + if has_unknown_path { + BoundaryReachability::Unknown + } else { + BoundaryReachability::CannotCross + } +} + +fn reachable_region_exits( + owner: OperationRef, + region: RegionRef, + start_block: BlockRef, + cache: &mut ReachabilityCache, +) -> RegionExitAnalysis { + let mut analysis = RegionExitAnalysis { + successors: SmallVec::new(), + has_unknown_exit: false, + }; + + for block in region.borrow().body().iter() { + let block = block.as_block_ref(); + if !block_reaches(start_block, block, cache) { + continue; + } + let terminator = block.borrow().terminator(); + let Some(terminator) = terminator else { + if owner.borrow().implements::() { + let successors = { + let owner = owner.borrow(); + let branch = owner + .as_trait::() + .expect("expected a region branch operation"); + branch + .get_successor_regions(RegionBranchPoint::Child(region)) + .map(RegionBranchPoint::from) + .collect::>() + }; + append_unique_successors(&mut analysis.successors, successors); + } else { + analysis.has_unknown_exit = true; + } + continue; + }; + + let terminator = terminator.borrow(); + if let Some(region_terminator) = + terminator.as_trait::() + { + let operands = unknown_operands(terminator.num_operands()); + let successors = region_terminator + .get_successor_regions(&operands) + .into_iter() + .map(|successor| successor.successor()); + append_unique_successors(&mut analysis.successors, successors); + } else if terminator.implements::() { + // Plain returns are only valid as direct terminators of a callable body. A + // ReturnLike under a RegionBranch owner is malformed or otherwise unmodeled. + analysis.has_unknown_exit = true; + } else if terminator.num_successors() == 0 { + // A terminator with no CFG or region successors may abort, throw, or transfer control + // by semantics unavailable here. It cannot justify a false `Impossible`. + analysis.has_unknown_exit = true; + } + } + + analysis +} + +fn callable_region_exit_reachability( + owner: OperationRef, + region: RegionRef, + start_block: BlockRef, + cache: &mut ReachabilityCache, +) -> BoundaryReachability { + let mut has_unknown_exit = false; + + for block in region.borrow().body().iter() { + let block = block.as_block_ref(); + if !block_reaches(start_block, block, cache) { + continue; + } + let Some(terminator) = block.borrow().terminator() else { + if owner.borrow().implements::() { + return BoundaryReachability::CanCross; + } + has_unknown_exit = true; + continue; + }; + let terminator = terminator.borrow(); + if terminator.implements::() { + // Such a terminator only has a defined destination under a RegionBranch owner. + has_unknown_exit = true; + } else if terminator.implements::() { + return BoundaryReachability::CanCross; + } else if terminator.num_successors() == 0 { + has_unknown_exit = true; + } + } + + if has_unknown_exit { + BoundaryReachability::Unknown + } else { + BoundaryReachability::CannotCross + } +} + +fn is_callable_body(owner: OperationRef, region: RegionRef) -> bool { + owner + .borrow() + .as_trait::() + .is_some_and(|callable| callable.get_callable_region() == Some(region)) +} + +fn block_reaches(from: BlockRef, to: BlockRef, cache: &mut ReachabilityCache) -> bool { + from == to || cache.leads_to(from, to) +} + +fn unknown_operands(count: usize) -> SmallVec<[Option; 4]> { + core::iter::repeat_n(None, count).collect() +} + +fn append_unique_successors( + successors: &mut SmallVec<[RegionBranchPoint; 4]>, + additional: impl IntoIterator, +) { + for successor in additional { + if !successors.contains(&successor) { + successors.push(successor); + } + } +} + +/// Classifies reachability between two positions in different sub-regions of one `owner` op. +/// +/// The sibling region is reachable when the region graph of `owner` can transfer control from +/// the region holding `from` to the region holding `to` within one execution of `owner` (e.g. +/// from the `before` region of a while to its `after` region), or when `owner` itself can +/// execute more than once (e.g. the arms of an if that an enclosing loop re-enters); otherwise +/// it is provably unreachable (e.g. the arms of an if outside any loop). +fn sibling_region_reachability( + owner: OperationRef, + from: OperationRef, + to: OperationRef, + cache: &mut ReachabilityCache, +) -> Reachability { + if !owner.borrow().implements::() { + // Unknown region semantics: conservatively treat transfer as possible. + return Reachability::Maybe; + } + let (Some(from_region), Some(to_region)) = + (child_region_containing(owner, from), child_region_containing(owner, to)) + else { + // Unreachable given the normalization above; defensively treat as reachable. + return Reachability::Maybe; + }; + if to_region.borrow().is_reachable_from(&from_region.borrow()) { + return Reachability::Maybe; + } + if op_can_re_execute(owner, cache) { + return Reachability::Maybe; + } + Reachability::Impossible +} + +/// Returns the direct child region of `owner` that contains `op`. +fn child_region_containing(owner: OperationRef, op: OperationRef) -> Option { + let mut region = op.borrow().parent_region(); + while let Some(r) = region { + let parent = r.parent()?; + if parent == owner { + return Some(r); + } + region = parent.borrow().parent_region(); + } + None +} + +/// Returns true if `op` can execute more than once: its block lies on a CFG cycle, or the +/// region containing it can re-execute. +fn op_can_re_execute(op: OperationRef, cache: &mut ReachabilityCache) -> bool { + let (block, region) = { + let op = op.borrow(); + (op.parent(), op.parent_region()) + }; + if let Some(block) = block + && cache.leads_to(block, block) + { + return true; + } + region.is_some_and(|region| region_can_re_execute(region, cache)) +} + +/// Returns true if `region` requires SSA dominance, i.e. operation order within it defines +/// control flow. Regions of operations that do not declare a region kind default to SSA. +fn region_has_ssa_dominance(region: RegionRef) -> bool { + region + .parent() + .and_then(|op| { + op.borrow() + .as_trait::() + .map(|rki| rki.has_ssa_dominance()) + }) + .unwrap_or(true) +} + +/// Returns true if `region` can execute more than once within a single execution of the +/// operation bounding control-flow reasoning about it (see [control_flow_scope]). +/// +/// That is the case when an enclosing region is repetitive (e.g. the regions of an `scf.while`, +/// whose back edges are expressed in the region graph of the owning op rather than as block +/// successors), or when an enclosing op itself sits on a CFG cycle in its parent region. +fn region_can_re_execute(region: RegionRef, cache: &mut ReachabilityCache) -> bool { + let mut current = Some(region); + while let Some(r) = current { + let Some(owner) = r.parent() else { + return false; + }; + let owner_op = owner.borrow(); + let Some(owner_block) = owner_op.parent() else { + // A top-level owner cannot be re-entered from anywhere. + return false; + }; + if !owner_block.borrow().has_ssa_dominance() { + // The owner resides in a graph-like region: control-flow reasoning stops here (in + // practice the owner is the enclosing function, and each execution of its body is + // a separate invocation). + return false; + } + if !owner_op.implements::() { + // Unknown region semantics: conservatively treat re-entry as possible. + return true; + } + if r.borrow().is_repetitive_region() { + return true; + } + if cache.leads_to(owner_block, owner_block) { + return true; + } + current = owner_op.parent_region(); + } + false +} + +#[cfg(test)] +mod tests { + use alloc::vec::Vec; + + use super::*; + use crate::{ + Builder, BuilderExt, Op, RegionSuccessorInfo, RegionSuccessorIter, SourceSpan, + SuccessorOperandRange, SuccessorOperandRangeMut, ValueRef, + derive::operation, + dialects::test::TestDialect, + testing::Test, + traits::{AnyType, BranchOpInterface, Terminator}, + }; + + #[operation(dialect = TestDialect, implements(RegionBranchOpInterface))] + pub struct TestRegionBranch { + #[region] + first: Region, + #[region] + second: Region, + } + + impl RegionBranchOpInterface for TestRegionBranch { + fn get_successor_regions(&self, point: RegionBranchPoint) -> RegionSuccessorIter<'_> { + let first = self.first().as_region_ref(); + let second = self.second().as_region_ref(); + let successors = match point { + RegionBranchPoint::Parent => { + SmallVec::from_buf([RegionSuccessorInfo::Entering(first)]) + } + RegionBranchPoint::Child(region) if region == first => { + SmallVec::from_buf([RegionSuccessorInfo::Entering(second)]) + } + RegionBranchPoint::Child(_) => { + SmallVec::from_buf([RegionSuccessorInfo::Returning(SmallVec::new())]) + } + }; + RegionSuccessorIter::new(self.as_operation(), successors) + } + } + + #[operation( + dialect = TestDialect, + traits(Terminator), + implements(BranchOpInterface) + )] + pub struct TestBranch { + #[successor] + target: Successor, + } + + impl BranchOpInterface for TestBranch {} + + #[operation( + dialect = TestDialect, + traits(Terminator), + implements(RegionBranchTerminatorOpInterface) + )] + pub struct TestRegionYield { + #[operands] + yielded: AnyType, + } + + impl RegionBranchTerminatorOpInterface for TestRegionYield { + fn get_successor_operands(&self, _point: RegionBranchPoint) -> SuccessorOperandRange<'_> { + SuccessorOperandRange::forward(self.yielded()) + } + + fn get_mutable_successor_operands( + &mut self, + _point: RegionBranchPoint, + ) -> SuccessorOperandRangeMut<'_> { + SuccessorOperandRangeMut::forward(self.yielded_mut()) + } + + fn get_successor_regions( + &self, + _operands: &[Option], + ) -> SmallVec<[RegionSuccessorInfo; 2]> { + let region = self.parent_region().expect("test yield must be in a region"); + let owner = self.parent_op().expect("test yield region must have an owner"); + let owner = owner.borrow(); + let owner = owner + .downcast_ref::() + .expect("test yield must be nested in TestRegionBranch"); + if region == owner.first().as_region_ref() { + core::iter::once(RegionSuccessorInfo::Entering(owner.second().as_region_ref())) + .collect() + } else { + core::iter::once(RegionSuccessorInfo::Returning(SmallVec::new())).collect() + } + } + } + + #[operation(dialect = TestDialect)] + pub struct TestUnknownRegionOwner { + #[region] + body: Region, + } + + #[operation(dialect = TestDialect)] + pub struct TestMarker {} + + #[test] + fn region_branch_entry_does_not_bypass_an_intermediate_child_sink() { + let mut test = + Test::new("region_branch_entry_does_not_bypass_an_intermediate_child_sink", &[], &[]); + let (owner, target) = { + let mut builder = test.function_builder(); + let owner = builder.builder_mut().create::(SourceSpan::UNKNOWN)() + .unwrap(); + let first = owner.borrow().first().as_region_ref(); + let second = owner.borrow().second().as_region_ref(); + + let sink = builder.builder_mut().create_block(first, None, &[]); + builder.builder_mut().set_insertion_point_to_end(sink); + builder + .builder_mut() + .create::)>(SourceSpan::UNKNOWN)( + sink, + Vec::new(), + ) + .unwrap(); + + // The abstract owner graph advertises first -> second, but this concrete terminator is + // disconnected from the first region's entry and therefore cannot enable that edge. + let disconnected = builder.builder_mut().create_block(first, None, &[]); + builder.builder_mut().set_insertion_point_to_end(disconnected); + builder + .builder_mut() + .create::,)>(SourceSpan::UNKNOWN)( + Vec::new() + ) + .unwrap(); + + let second_entry = builder.builder_mut().create_block(second, None, &[]); + builder.builder_mut().set_insertion_point_to_end(second_entry); + let target = builder + .builder_mut() + .create::,)>(SourceSpan::UNKNOWN)( + Vec::new() + ) + .unwrap(); + (owner.as_operation_ref(), target.as_operation_ref()) + }; + + assert_eq!( + Operation::reachability(owner, target), + Reachability::Impossible, + "an abstract child-to-child edge must not bypass a sink in the intermediate child" + ); + } + + #[test] + fn unknown_region_owner_semantics_remain_maybe() { + let mut test = Test::new("unknown_region_owner_semantics_remain_maybe", &[], &[]); + let (owner, nested) = { + let mut builder = test.function_builder(); + let owner = + builder.builder_mut().create::(SourceSpan::UNKNOWN)() + .unwrap(); + let body = owner.borrow().body().as_region_ref(); + let entry = builder.builder_mut().create_block(body, None, &[]); + builder.builder_mut().set_insertion_point_to_end(entry); + builder + .builder_mut() + .create::)>(SourceSpan::UNKNOWN)( + entry, + Vec::new(), + ) + .unwrap(); + let disconnected = builder.builder_mut().create_block(body, None, &[]); + builder.builder_mut().set_insertion_point_to_end(disconnected); + let nested = + builder.builder_mut().create::(SourceSpan::UNKNOWN)().unwrap(); + (owner.as_operation_ref(), nested.as_operation_ref()) + }; + + assert_eq!( + Operation::reachability(owner, nested), + Reachability::Maybe, + "an unmodeled region owner must stay conservative even for a disconnected block" + ); + } +} diff --git a/tests/lit/hir-opt/transform-spills-join-covered-reload.hir b/tests/lit/hir-opt/transform-spills-join-covered-reload.hir new file mode 100644 index 000000000..a24ed05de --- /dev/null +++ b/tests/lit/hir-opt/transform-spills-join-covered-reload.hir @@ -0,0 +1,74 @@ +// RUN: hir-opt %s --pass-pipeline='builtin.module(builtin.function(transform-spills))' | filecheck %s + +// Both arms of the diamond below spill the same value (each contains a copy of the +// high-pressure call), while the only use of that value after the spills lies after the join. +// The spills analysis places one spill per arm, and the single reload lands at the join, so the +// reload is covered by the *set* of spills, none of which individually dominates it. This is a +// regression test for spill pruning: pruning spills that do not dominate a live reload erased +// both arm spills while keeping the reload, and reload materialization then failed because no +// procedure local was ever allocated for the spilled value. +// +// Operand stack pressure at each call (the analysis models pressure in felts with K = 16): the +// call arguments require 15 felts (%8: ptr = 1, %5: u128 = 4, %9: u128 passed twice = 8, +// u64 = 2), and %1: u32 and %3: u32 are live across both calls (used only in the join block), +// for a total of 15 + 2 = 17 > K, so exactly one of {%1, %3} is spilled in each arm. + +// COM: No spill materializes before the branch +// CHECK-NOT: hir.store_local +// CHECK: cf.cond_br %{{\d+}} ^[[THEN:block\d+]], ^[[ELSE:block\d+]] + +// COM: The then-arm spills the value live across both calls, before its call +// CHECK: ^[[THEN]]: +// CHECK: hir.store_local %[[V:\d+]] <{ local = #builtin.local_variable<[[L:\d+]], u32> }> +// CHECK-NEXT: hir.exec ::@test::@example +// CHECK-NOT: hir.store_local +// CHECK: cf.br ^[[JOIN:block\d+]]( + +// COM: The else-arm spills the same value to the same local +// CHECK: ^[[ELSE]]: +// CHECK: hir.store_local %[[V]] <{ local = #builtin.local_variable<[[L]], u32> }> +// CHECK-NEXT: hir.exec ::@test::@example +// CHECK-NOT: hir.store_local +// CHECK: cf.br ^[[JOIN]]( + +// COM: The join block holds the single reload of the spilled value +// CHECK: ^[[JOIN]]( +// CHECK-NOT: hir.load_local +// CHECK-NOT: hir.store_local +// CHECK: hir.load_local <{ local = #builtin.local_variable<[[L]], u32> }> +// CHECK-NOT: hir.load_local +// CHECK-NOT: hir.store_local +// CHECK: builtin.ret % +builtin.module public @test { + builtin.function public extern("C") @join_covered_reload(%0: ptr) -> u32 { + %1 = hir.ptr_to_int %0 <{ ty = #builtin.type }>; + %2 = arith.constant 32 : u32; + %3 = arith.add %1, %2 <{ overflow = #builtin.overflow }>; + %4 = hir.int_to_ptr %3 <{ ty = #builtin.type> }>; + %5 = hir.load %4; + %6 = arith.constant 64 : u32; + %7 = arith.add %1, %6 <{ overflow = #builtin.overflow }>; + %8 = hir.int_to_ptr %7 <{ ty = #builtin.type> }>; + %9 = hir.load %8; + %10 = arith.constant 0 : u32; + %11 = arith.eq %1, %10; + cf.cond_br %11 ^then, ^else : (i1); + ^then: + %12 = arith.constant 1 : u64; + %13 = hir.exec ::@test::@example(%8, %5, %9, %9, %12) : extern("C") (ptr, u128, u128, u128, u64) -> u32; + cf.br ^join(%13 : u32); + ^else: + %14 = arith.constant 1 : u64; + %15 = hir.exec ::@test::@example(%8, %5, %9, %9, %14) : extern("C") (ptr, u128, u128, u128, u64) -> u32; + cf.br ^join(%15 : u32); + ^join(%16: u32): + %17 = arith.constant 5 : u32; + %18 = arith.add %1, %17 <{ overflow = #builtin.overflow }>; + %19 = arith.add %18, %16 <{ overflow = #builtin.overflow }>; + %20 = arith.add %19, %3 <{ overflow = #builtin.overflow }>; + builtin.ret %20 : (u32); + }; + builtin.function public extern("C") @example(%p: ptr, %x: u128, %y: u128, %z: u128, %w: u64) -> u32 { + builtin.ret_imm 42 : u32; + }; +};