diff --git a/enzyme/Enzyme/MLIR/Interfaces/GradientUtils.cpp b/enzyme/Enzyme/MLIR/Interfaces/GradientUtils.cpp index 0c157581e31..d2fadbc0896 100644 --- a/enzyme/Enzyme/MLIR/Interfaces/GradientUtils.cpp +++ b/enzyme/Enzyme/MLIR/Interfaces/GradientUtils.cpp @@ -225,8 +225,20 @@ void mlir::enzyme::MGradientUtils::setDiffe(mlir::Value val, mlir::Value toset, } assert(!isConstantValue(val)); + // Reverse mode also uses invertedPointers for mutable values. For allocation + // results, forceAugmentedReturns() installs a placeholder shadow; the + // allocation derivative later calls setDiffe with the real shadow, so replace + // it here. + bool isMutable = cast(val.getType()).isMutable(); if (mode == DerivativeMode::ForwardMode || - mode == DerivativeMode::ForwardModeSplit) { + mode == DerivativeMode::ForwardModeSplit || isMutable) { + if (isMutable) { + // Mutable arguments may already be mapped to a real user-provided shadow + // rather than a placeholder. Only replace placeholders here. + auto existing = invertedPointers.lookupOrNull(val); + if (existing && !existing.getDefiningOp()) + return; + } setInvertedPointer(val, toset); } /* diff --git a/enzyme/test/MLIR/ReverseMode/alloca.mlir b/enzyme/test/MLIR/ReverseMode/alloca.mlir new file mode 100644 index 00000000000..5ca576c674b --- /dev/null +++ b/enzyme/test/MLIR/ReverseMode/alloca.mlir @@ -0,0 +1,110 @@ +// RUN: %eopt --enzyme %s | FileCheck --implicit-check-not=enzyme.placeholder %s + +func.func @foo_flat(%x : f64) -> f64 { + %buf = memref.alloca() : memref + memref.store %x, %buf[] : memref + %y = memref.load %buf[] : memref + return %y : f64 +} + +func.func @dfoo_flat(%x: f64, %dout : f64) -> f64 { + %dx = enzyme.autodiff @foo_flat(%x, %dout) { + activity = [#enzyme], + ret_activity = [#enzyme] + } : (f64, f64) -> (f64) + return %dx : f64 +} + +// CHECK-LABEL: func.func private @diffefoo_flat( +// CHECK-SAME: %[[X:[^,]+]]: f64, +// CHECK-SAME: %[[DOUT:[^)]+]]: f64) -> f64 { + +// CHECK: %[[GX:.+]] = "enzyme.init"() : () -> !enzyme.Gradient +// CHECK: "enzyme.set"(%[[GX]], %{{.*}}) : (!enzyme.Gradient, f64) -> () +// CHECK: %[[CS:.+]] = "enzyme.init"() : () -> !enzyme.Cache> +// CHECK: %[[CL:.+]] = "enzyme.init"() : () -> !enzyme.Cache> +// CHECK: %[[GY:.+]] = "enzyme.init"() : () -> !enzyme.Gradient +// CHECK: "enzyme.set"(%[[GY]], %{{.*}}) : (!enzyme.Gradient, f64) -> () + +// CHECK: %[[DBUF:.+]] = memref.alloca() : memref +// CHECK: %[[ZERO_INIT:.+]] = arith.constant 0.000000e+00 : f64 +// CHECK: linalg.fill ins(%[[ZERO_INIT]] : f64) outs(%[[DBUF]] : memref) + +// CHECK: %[[BUF:.+]] = memref.alloca() : memref +// CHECK: "enzyme.push"(%[[CS]], %[[DBUF]]) : (!enzyme.Cache>, memref) -> () +// CHECK: memref.store %[[X]], %[[BUF]][] : memref +// CHECK: "enzyme.push"(%[[CL]], %[[DBUF]]) : (!enzyme.Cache>, memref) -> () +// CHECK: memref.load %[[BUF]][] : memref +// CHECK: cf.br ^bb1 +// CHECK: ^bb1: + +// CHECK: %{{.+}} = "enzyme.get"(%[[GY]]) : (!enzyme.Gradient) -> f64 +// CHECK: arith.addf %{{.+}}, %[[DOUT]] : f64 +// CHECK: "enzyme.set"(%[[GY]], %{{.*}}) : (!enzyme.Gradient, f64) -> () + +// CHECK: %{{.+}} = "enzyme.get"(%[[GY]]) : (!enzyme.Gradient) -> f64 +// CHECK: %[[POPL:.+]] = "enzyme.pop"(%[[CL]]) : (!enzyme.Cache>) -> memref +// CHECK: memref.load %[[POPL]][] : memref +// CHECK: arith.addf +// CHECK: memref.store %{{.+}}, %[[POPL]][] : memref + +// CHECK: %[[POPS:.+]] = "enzyme.pop"(%[[CS]]) : (!enzyme.Cache>) -> memref +// CHECK: memref.load %[[POPS]][] : memref +// CHECK: %{{.+}} = "enzyme.get"(%[[GX]]) : (!enzyme.Gradient) -> f64 +// CHECK: arith.addf +// CHECK: "enzyme.set"(%[[GX]], %{{.*}}) : (!enzyme.Gradient, f64) -> () +// CHECK: %[[ZERO_CLEAR:.+]] = arith.constant 0.000000e+00 : f64 +// CHECK: memref.store %[[ZERO_CLEAR]], %[[POPS]][] : memref + +// CHECK: %{{.+}} = "enzyme.get"(%[[GX]]) : (!enzyme.Gradient) -> f64 +// CHECK: return %{{.+}} : f64 + + +// ----- + +func.func @load_arg(%mem: memref) -> f64 { + %buf = memref.alloca() : memref + %x = memref.load %mem[] : memref + memref.store %x, %buf[] : memref + %y = memref.load %buf[] : memref + return %y : f64 +} + +func.func @dload_arg(%mem: memref, %dmem: memref, %dout : f64) { + enzyme.autodiff @load_arg(%mem, %dmem, %dout) { + activity = [#enzyme], + ret_activity = [#enzyme] + } : (memref, memref, f64) -> () + return +} + +// CHECK-LABEL: func.func private @diffeload_arg( +// CHECK-SAME: %[[MEM:[^,]+]]: memref, +// CHECK-SAME: %[[DMEM:[^,]+]]: memref, +// CHECK-SAME: %[[DOUT:[^)]+]]: f64) { +// CHECK: %[[DBUF:.+]] = memref.alloca() : memref +// CHECK: linalg.fill ins(%{{.+}} : f64) outs(%[[DBUF]] : memref) +// CHECK: %[[BUF:.+]] = memref.alloca() : memref +// CHECK: "enzyme.push"(%{{.+}}, %[[DMEM]]) : (!enzyme.Cache>, memref) -> () +// CHECK: memref.load %[[MEM]][] : memref +// CHECK: "enzyme.push"(%{{.+}}, %[[DBUF]]) : (!enzyme.Cache>, memref) -> () +// CHECK: memref.store %{{.+}}, %[[BUF]][] : memref +// CHECK: "enzyme.push"(%{{.+}}, %[[DBUF]]) : (!enzyme.Cache>, memref) -> () +// CHECK: memref.load %[[BUF]][] : memref +// CHECK: arith.addf %{{.+}}, %[[DOUT]] : f64 +// CHECK: %[[POPL:.+]] = "enzyme.pop"(%{{.+}}) : (!enzyme.Cache>) -> memref +// CHECK: memref.load %[[POPL]][] : memref +// CHECK: arith.addf +// CHECK: memref.store %{{.+}}, %[[POPL]][] : memref +// CHECK: %[[POPS:.+]] = "enzyme.pop"(%{{.+}}) : (!enzyme.Cache>) -> memref +// CHECK: memref.load %[[POPS]][] : memref +// CHECK: "enzyme.get" +// CHECK: arith.addf +// CHECK: "enzyme.set" +// CHECK: memref.store %{{.+}}, %[[POPS]][] : memref +// CHECK: "enzyme.get" +// CHECK: %[[POPD:.+]] = "enzyme.pop"(%{{.+}}) : (!enzyme.Cache>) -> memref +// CHECK: memref.load %[[POPD]][] : memref +// CHECK: arith.addf +// CHECK: memref.store %{{.+}}, %[[POPD]][] : memref +// CHECK: return