From 121bef3f8adc262a0f7b010cfc01624da27f7438 Mon Sep 17 00:00:00 2001 From: Toni Boehnlein Date: Thu, 16 Jul 2026 12:42:32 +0200 Subject: [PATCH 1/5] fix(insert-sync): track explicit address provenance --- docs/designs/ptoas-auto-sync-design.md | 15 +- include/PTO/IR/PTOOps.td | 10 ++ .../PTO/Transforms/InsertSync/SyncCommon.h | 24 ++-- .../InsertSync/MemoryDependentAnalyzer.cpp | 30 +++- .../Transforms/InsertSync/PTOIRTranslator.cpp | 103 +++++++++++--- .../pto/alloc_tile_physical_overlap_sync.pto | 131 +++++++++++++++++- ...mory_address_provenance_no_false_alias.pto | 55 ++++++++ 7 files changed, 332 insertions(+), 36 deletions(-) create mode 100644 test/lit/pto/plan_memory_address_provenance_no_false_alias.pto diff --git a/docs/designs/ptoas-auto-sync-design.md b/docs/designs/ptoas-auto-sync-design.md index b8f5c0a2d5..5d67b69931 100644 --- a/docs/designs/ptoas-auto-sync-design.md +++ b/docs/designs/ptoas-auto-sync-design.md @@ -88,10 +88,11 @@ | `scope` | `pto::AddressSpace` | 地址空间(GM/MAT/VEC/ACC/LEFT/RIGHT 等) | | `baseAddresses` | `SmallVector` | 已知的偏移列表,配合 `allocateSize` 做精确区间重叠 | | `allocateSize` | `uint64_t` | 字节大小 | +| `addressProvenance` | `AddressProvenance` | 区分 root-relative 偏移、已知 absolute 物理地址、未知/动态 absolute 地址 | -`operator==`(第 111 行)要求 `baseAddresses`、`rootBuffer`、`scope`、 -`allocateSize`、`baseBuffer` 全部相等才算同一缓冲;这是别名分析里的「严格相等」 -判定,命中后可直接走依赖路径。 +`operator==` 要求 `baseAddresses`、`rootBuffer`、`scope`、 +`allocateSize`、`baseBuffer`、`addressProvenance` 全部相等才算同一缓冲; +这是别名分析里的「严格相等」判定,命中后可直接走依赖路径。 ### 3.4 同步指令:`SyncOperation` @@ -381,8 +382,12 @@ hazard 判定按三类关系展开: 1. `scope` 不同,直接不别名。 2. `GM` 先比较 root,必要时追踪 `realRoot`。 -3. 地址和大小都已知,就做区间重叠。 -4. 信息缺失(地址未知、size 未知),保守按“可能重叠”。 +3. 两侧都是已知 absolute 本地物理地址时,即使 allocation SSA root 不同, + 也按半开字节区间 `[base, base + size)` 判断重叠;区间端点相接不算 alias。 +4. 任一侧是未知/动态 absolute 本地地址时,保守按“可能重叠”。 +5. 其余 root-relative 地址保持原有 root/view 链分析;不同 root 下相同的相对 + offset(例如 level-2 规划前的 `baseAddresses={0}`)不会被误当作同一物理地址。 +6. size 未知,或 absolute view offset / 区间端点计算溢出时,保守按“可能重叠”。 这套规则在信息不足时会偏保守,但不会牺牲正确性。后续会继续补强动态 shape/offset 场景下的精细分析。 diff --git a/include/PTO/IR/PTOOps.td b/include/PTO/IR/PTOOps.td index e462ecded7..1ed0fd6132 100644 --- a/include/PTO/IR/PTOOps.td +++ b/include/PTO/IR/PTOOps.td @@ -319,6 +319,16 @@ def GetTensorViewStrideOp : PTO_Op<"get_tensor_view_stride", [Pure]> { def AllocTileOp : PTO_Op<"alloc_tile", [AttrSizedOperandSegments]> { let summary = "Allocates a tile buffer (logical buffer)."; + let description = [{ + Without `addr`, this op declares a logical local allocation whose physical + address is selected by PTOAS memory planning. + + Under `--pto-level=level3`, `addr` is required and denotes an absolute byte + address in the tile's local address space. Synchronization analysis compares + constant absolute ranges across distinct allocation SSA roots. If an + absolute address is dynamic, analysis conservatively treats it as possibly + aliasing any allocation in the same local address space. + }]; let arguments = (ins Optional:$addr, diff --git a/include/PTO/Transforms/InsertSync/SyncCommon.h b/include/PTO/Transforms/InsertSync/SyncCommon.h index 7b1970a634..d130f17cdd 100644 --- a/include/PTO/Transforms/InsertSync/SyncCommon.h +++ b/include/PTO/Transforms/InsertSync/SyncCommon.h @@ -80,6 +80,16 @@ enum class TCoreType { CUBE_OR_VECTOR, CUBE_AND_VECTOR }; + +/// Describes how BaseMemInfo::baseAddresses must be interpreted. +enum class AddressProvenance { + /// Addresses are byte offsets relative to rootBuffer. + RootRelative, + /// Addresses are known absolute byte addresses in the local address space. + KnownAbsolute, + /// The buffer has an absolute address, but its value is dynamic or unknown. + UnknownAbsolute +}; /// Meminfo of the target buffer /// 用于追踪 Buffer 的别名和根节点 @@ -87,10 +97,10 @@ struct BaseMemInfo { BaseMemInfo( Value baseBuffer, Value rootBuffer, pto::AddressSpace scope, SmallVector baseAddresses, uint64_t allocateSize, - bool hasKnownPhysicalAddresses = false) + AddressProvenance addressProvenance = AddressProvenance::RootRelative) : baseBuffer(baseBuffer), rootBuffer(rootBuffer), scope(scope), baseAddresses(std::move(baseAddresses)), allocateSize(allocateSize), - hasKnownPhysicalAddresses(hasKnownPhysicalAddresses) {} + addressProvenance(addressProvenance) {} /// baseBuffer: 当前操作直接使用的 Buffer (可能是 View 或 Alias) Value baseBuffer; @@ -100,9 +110,7 @@ struct BaseMemInfo { pto::AddressSpace scope; SmallVector baseAddresses; // 用于 Offset 分析 uint64_t allocateSize; - // PlanMemory materializes static local allocations as pointer_cast constants. - // This distinguishes their physical addresses from root-relative offsets. - bool hasKnownPhysicalAddresses; + AddressProvenance addressProvenance; bool areVectorEqual(const SmallVector& vec1, const SmallVector& vec2) const { @@ -117,7 +125,7 @@ struct BaseMemInfo { if (!areVectorEqual(baseAddresses, other.baseAddresses)) return false; if (rootBuffer != other.rootBuffer) return false; if (scope != other.scope) return false; - if (hasKnownPhysicalAddresses != other.hasKnownPhysicalAddresses) + if (addressProvenance != other.addressProvenance) return false; // allocateSize 和 baseBuffer 的严格相等性在某些别名分析中可能太强了, // 但为了保持原有逻辑,先保留。重点是 rootBuffer 必须一致。 @@ -129,13 +137,13 @@ struct BaseMemInfo { std::unique_ptr clone() const { return std::make_unique( baseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - hasKnownPhysicalAddresses); + addressProvenance); } std::unique_ptr clone(Value cloneBaseBuffer) const { return std::make_unique( cloneBaseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - hasKnownPhysicalAddresses); + addressProvenance); } }; diff --git a/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp b/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp index 5d8a5e9789..c700422539 100644 --- a/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp +++ b/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp @@ -16,6 +16,8 @@ #include "mlir/Interfaces/ViewLikeInterface.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "llvm/Support/Debug.h" + +#include #define DEBUG_TYPE "pto-inject-sync" @@ -154,11 +156,22 @@ bool MemoryDependentAnalyzer::MemAlias(const BaseMemInfo *a, } // 2. Local Memory (UB/L1) - // PTOPlanMemory turns each allocation into a distinct pointer_cast. Once an - // async MTE3 store has consumed the source SSA, a later allocation can reuse - // the same physical range with a different pointer_cast root. Compare those - // ranges directly when both sides carry known physical local addresses. - if (a->hasKnownPhysicalAddresses && b->hasKnownPhysicalAddresses) { + // An unknown absolute local address can overlap any other allocation in the + // same address space. Keep this case conservative instead of falling back to + // distinct-root identity and silently declaring no-alias. + if (a->addressProvenance == AddressProvenance::UnknownAbsolute || + b->addressProvenance == AddressProvenance::UnknownAbsolute) { + if (isTraceEnabled()) + llvm::errs() << " -> Unknown absolute local address. Conservatively " + "aliasing.\n"; + return true; + } + + // PTOPlanMemory and level-3 explicit allocations can produce distinct SSA + // roots for the same physical bytes. Compare known absolute ranges directly + // across roots. + if (a->addressProvenance == AddressProvenance::KnownAbsolute && + b->addressProvenance == AddressProvenance::KnownAbsolute) { if (isTraceEnabled()) llvm::errs() << " -> Comparing known physical local ranges.\n"; if (a->baseAddresses.empty() || b->baseAddresses.empty()) @@ -239,6 +252,13 @@ bool MemoryDependentAnalyzer::isBufferOverlap(const BaseMemInfo *a, int bIndex) { uint64_t aStart = a->baseAddresses[aIndex]; uint64_t bStart = b->baseAddresses[bIndex]; + + // Invalid/overflowing half-open ranges are not safe to classify as disjoint. + // Conservatively report overlap if either end cannot be represented. + if (a->allocateSize > std::numeric_limits::max() - aStart || + b->allocateSize > std::numeric_limits::max() - bStart) + return true; + uint64_t aEnd = aStart + a->allocateSize; uint64_t bEnd = bStart + b->allocateSize; diff --git a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp index 4c09745cbf..11d8a3037a 100644 --- a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp +++ b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp @@ -27,6 +27,7 @@ // [P0 新增] 引入副作用接口和 PTO 接口 #include "mlir/Interfaces/SideEffectInterfaces.h" +#include #include #define DEBUG_TYPE "pto-ir-translator" @@ -92,6 +93,29 @@ static std::optional getKnownPhysicalAddress(Value value) { return static_cast(address); } +static std::optional addByteOffset(uint64_t base, uint64_t offset) { + if (offset > std::numeric_limits::max() - base) + return std::nullopt; + return base + offset; +} + +static std::optional addByteOffset(uint64_t base, int64_t offset) { + if (offset >= 0) + return addByteOffset(base, static_cast(offset)); + + uint64_t magnitude = static_cast(-(offset + 1)) + 1; + if (magnitude > base) + return std::nullopt; + return base - magnitude; +} + +static void markAddressRangeUnknown(BaseMemInfo &info) { + info.baseAddresses.clear(); + info.allocateSize = 0; + if (info.addressProvenance == AddressProvenance::KnownAbsolute) + info.addressProvenance = AddressProvenance::UnknownAbsolute; +} + static bool isLocalAddressSpace(pto::AddressSpace space) { return space != pto::AddressSpace::GM && space != pto::AddressSpace::Zero; @@ -509,9 +533,15 @@ LogicalResult PTOIRTranslator::UpdateAllocTileOpMemInfo(pto::AllocTileOp op) { } // 3. 注册 Buffer 信息 + AddressProvenance addressProvenance = AddressProvenance::RootRelative; + if (op.getAddr() && isLocalAddressSpace(space)) { + addressProvenance = knownPhysicalAddress + ? AddressProvenance::KnownAbsolute + : AddressProvenance::UnknownAbsolute; + } auto newMemInfo = std::make_unique( res, res, space, SmallVector{baseAddr}, sizeInBytes, - knownPhysicalAddress.has_value() && isLocalAddressSpace(space)); + addressProvenance); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); @@ -549,21 +579,23 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { if (op.getAddrs().size() == 1) { // Keep the address SSA as the root so dynamic pointer casts retain their - // historical identity-based behavior. For a static local cast, also keep - // its byte address for cross-root physical range analysis; downstream view - // ops accumulate their delta into baseAddresses[0]. + // identity. Local pointer_cast operands are absolute physical addresses: + // keep a constant byte address for cross-root range analysis, and mark a + // dynamic address as unknown-absolute so alias analysis stays conservative. + // Downstream view ops accumulate their delta into baseAddresses[0]. Value rootSrc = op.getAddrs().front(); SmallVector baseAddresses{0}; - bool hasKnownPhysicalAddresses = false; + AddressProvenance addressProvenance = AddressProvenance::RootRelative; if (isLocalAddressSpace(space)) { + addressProvenance = AddressProvenance::UnknownAbsolute; if (std::optional address = getKnownPhysicalAddress(rootSrc)) { baseAddresses[0] = *address; - hasKnownPhysicalAddresses = true; + addressProvenance = AddressProvenance::KnownAbsolute; } } auto newMemInfo = std::make_unique( res, rootSrc, space, std::move(baseAddresses), sizeInBytes, - hasKnownPhysicalAddresses); + addressProvenance); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); } @@ -586,7 +618,9 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { // historical behavior. Value rootSrc = op.getAddrs().front(); auto newMemInfo = std::make_unique( - res, rootSrc, space, SmallVector{0}, sizeInBytes); + res, rootSrc, space, SmallVector{0}, sizeInBytes, + isLocalAddressSpace(space) ? AddressProvenance::UnknownAbsolute + : AddressProvenance::RootRelative); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); } @@ -595,7 +629,8 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { auto newMemInfo = std::make_unique( res, res, space, std::move(slotOffsets), sizeInBytes, - isLocalAddressSpace(space)); + isLocalAddressSpace(space) ? AddressProvenance::KnownAbsolute + : AddressProvenance::RootRelative); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); } @@ -975,14 +1010,26 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { for (auto &parentInfo : buffer2MemInfoMap_[source]) { auto newInfo = parentInfo->clone(result); + bool addressRangeUnknown = false; if (!newInfo->baseAddresses.empty()) { - newInfo->baseAddresses[0] += deltaOffset; + if (std::optional address = + addByteOffset(newInfo->baseAddresses[0], deltaOffset)) { + newInfo->baseAddresses[0] = *address; + } else { + markAddressRangeUnknown(*newInfo); + addressRangeUnknown = true; + } } else { - newInfo->baseAddresses.push_back(deltaOffset); + if (std::optional address = addByteOffset(0, deltaOffset)) + newInfo->baseAddresses.push_back(*address); + else { + markAddressRangeUnknown(*newInfo); + addressRangeUnknown = true; + } } - if (newSize > 0) { + if (newSize > 0 && !addressRangeUnknown) { newInfo->allocateSize = newSize; } @@ -1098,8 +1145,20 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { SmallVector addresses; addresses.reserve(subViewAddresses->size()); uint64_t parentBase = parentInfo->baseAddresses[0]; - for (uint64_t offset : *subViewAddresses) - addresses.push_back(parentBase + offset); + bool overflowed = false; + for (uint64_t offset : *subViewAddresses) { + std::optional address = addByteOffset(parentBase, offset); + if (!address) { + overflowed = true; + break; + } + addresses.push_back(*address); + } + if (overflowed) { + markAddressRangeUnknown(*newInfo); + resultMemInfoVec.emplace_back(std::move(newInfo)); + continue; + } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; resultMemInfoVec.emplace_back(std::move(newInfo)); @@ -1153,8 +1212,20 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { SmallVector addresses; addresses.reserve(subViewAddresses->size()); uint64_t parentBase = parentInfo->baseAddresses[0]; - for (uint64_t offset : *subViewAddresses) - addresses.push_back(parentBase + offset); + bool overflowed = false; + for (uint64_t offset : *subViewAddresses) { + std::optional address = addByteOffset(parentBase, offset); + if (!address) { + overflowed = true; + break; + } + addresses.push_back(*address); + } + if (overflowed) { + markAddressRangeUnknown(*newInfo); + resultMemInfoVec.emplace_back(std::move(newInfo)); + continue; + } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; resultMemInfoVec.emplace_back(std::move(newInfo)); diff --git a/test/lit/pto/alloc_tile_physical_overlap_sync.pto b/test/lit/pto/alloc_tile_physical_overlap_sync.pto index 8e6cc457d9..f2ccaad5fa 100644 --- a/test/lit/pto/alloc_tile_physical_overlap_sync.pto +++ b/test/lit/pto/alloc_tile_physical_overlap_sync.pto @@ -7,8 +7,9 @@ // See LICENSE in the root of the software repository for the full text of the License. // Explicit local alloc_tile addresses are physical UB addresses. Distinct SSA -// roots whose ranges overlap must synchronize an asynchronous MTE3 read before -// a later V-pipe write reuses the range. +// roots whose ranges overlap must synchronize cross-pipe reuse. Known disjoint +// and boundary-touching half-open ranges must remain independent, while dynamic +// absolute addresses conservatively alias. // // RUN: ptoas --pto-arch=a3 --pto-level=level3 --enable-insert-sync %s -o - 2>&1 | FileCheck %s @@ -42,6 +43,110 @@ module attributes {pto.target_arch = "a2a3"} { outs(%reused : !pto.tile_buf) return } + + // Models the gather failure directly: PIPE_V is still reading the assembled + // 4 KiB tile at [4096,8192) when PIPE_MTE2 overwrites the 32-byte index row + // at [4608,4640). The distinct alloc_tile roots must still produce a + // PIPE_V -> PIPE_MTE2 WAR dependency. + func.func @alloc_tile_explicit_overlap_v_to_mte2( + %assembled_src: memref<32x32xf32, #pto.address_space>, + %index_src: memref<1x8xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c4608_i64 = arith.constant 4608 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %assembled = pto.alloc_tile addr = %c4096_i64 : + !pto.tile_buf + %vector_out = pto.alloc_tile addr = %c8192_i64 : + !pto.tile_buf + %index_row = pto.alloc_tile addr = %c4608_i64 : + !pto.tile_buf + + pto.tload ins(%assembled_src : memref<32x32xf32, #pto.address_space>) + outs(%assembled : !pto.tile_buf) + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%index_row : !pto.tile_buf) + return + } + + func.func @alloc_tile_explicit_disjoint_no_sync( + %assembled_src: memref<32x32xf32, #pto.address_space>, + %index_src: memref<1x8xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %assembled = pto.alloc_tile addr = %c4096_i64 : + !pto.tile_buf + %vector_out = pto.alloc_tile addr = %c8192_i64 : + !pto.tile_buf + %index_row = pto.alloc_tile addr = %c16384_i64 : + !pto.tile_buf + + pto.tload ins(%assembled_src : memref<32x32xf32, #pto.address_space>) + outs(%assembled : !pto.tile_buf) + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%index_row : !pto.tile_buf) + return + } + + func.func @alloc_tile_explicit_boundary_no_sync( + %assembled_src: memref<32x32xf32, #pto.address_space>, + %index_src: memref<1x8xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c12288_i64 = arith.constant 12288 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %assembled = pto.alloc_tile addr = %c4096_i64 : + !pto.tile_buf + %vector_out = pto.alloc_tile addr = %c12288_i64 : + !pto.tile_buf + %index_row = pto.alloc_tile addr = %c8192_i64 : + !pto.tile_buf + + pto.tload ins(%assembled_src : memref<32x32xf32, #pto.address_space>) + outs(%assembled : !pto.tile_buf) + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%index_row : !pto.tile_buf) + return + } + + func.func @alloc_tile_unknown_absolute_sync( + %assembled_src: memref<32x32xf32, #pto.address_space>, + %index_src: memref<1x8xf32, #pto.address_space>, + %assembled_addr: i64, + %index_addr: i64) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c8192_i64 = arith.constant 8192 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %assembled = pto.alloc_tile addr = %assembled_addr : + !pto.tile_buf + %vector_out = pto.alloc_tile addr = %c8192_i64 : + !pto.tile_buf + %index_row = pto.alloc_tile addr = %index_addr : + !pto.tile_buf + + pto.tload ins(%assembled_src : memref<32x32xf32, #pto.address_space>) + outs(%assembled : !pto.tile_buf) + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%index_row : !pto.tile_buf) + return + } } // CHECK-LABEL: AICORE void alloc_tile_physical_overlap_sync( @@ -49,3 +154,25 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK-NEXT: set_flag(PIPE_MTE3, PIPE_V, EVENT_ID[[MTE3_TO_V:[0-9]+]]); // CHECK-NEXT: wait_flag(PIPE_MTE3, PIPE_V, EVENT_ID[[MTE3_TO_V]]); // CHECK-NEXT: TMULS( + +// CHECK-LABEL: AICORE void alloc_tile_explicit_overlap_v_to_mte2( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[V_TO_MTE2]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void alloc_tile_explicit_disjoint_no_sync( +// CHECK: TMULS( +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2 +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void alloc_tile_explicit_boundary_no_sync( +// CHECK: TMULS( +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2 +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void alloc_tile_unknown_absolute_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[UNKNOWN_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[UNKNOWN_V_TO_MTE2]]); +// CHECK: TLOAD( diff --git a/test/lit/pto/plan_memory_address_provenance_no_false_alias.pto b/test/lit/pto/plan_memory_address_provenance_no_false_alias.pto new file mode 100644 index 0000000000..95a5e873d5 --- /dev/null +++ b/test/lit/pto/plan_memory_address_provenance_no_false_alias.pto @@ -0,0 +1,55 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// Address-less level-2 allocations start as root-relative offsets. They must +// not all alias merely because their pre-planning baseAddresses are zero. +// PlanMemory assigns disjoint physical ranges here, and InsertSync must not add +// a spurious PIPE_V -> PIPE_MTE2 dependency between the first TMULS and the +// unrelated second TLOAD. +// +// RUN: ptoas --pto-arch=a3 --pto-level=level2 --enable-insert-sync --emit-pto-ir --mlir-print-ir-after=pto-plan-memory %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=PLAN +// RUN: ptoas --pto-arch=a3 --pto-level=level2 --enable-insert-sync %s -o - 2>&1 | FileCheck %s --check-prefix=SYNC + +module attributes {pto.target_arch = "a2a3"} { + func.func @plan_memory_address_provenance_no_false_alias( + %assembled_src: memref<32x32xf32, #pto.address_space>, + %index_src: memref<1x8xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %one = arith.constant 1.000000e+00 : f32 + + %assembled = pto.alloc_tile : + !pto.tile_buf + %vector_out = pto.alloc_tile : + !pto.tile_buf + %index_row = pto.alloc_tile : + !pto.tile_buf + + pto.tload ins(%assembled_src : memref<32x32xf32, #pto.address_space>) + outs(%assembled : !pto.tile_buf) + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%index_row : !pto.tile_buf) + // Keep the assembled tile live across the index load so PlanMemory cannot + // legally reuse its physical range for index_row. + pto.tmuls ins(%assembled, %one : !pto.tile_buf, f32) + outs(%vector_out : !pto.tile_buf) + return + } +} + +// PLAN-LABEL: func.func @plan_memory_address_provenance_no_false_alias( +// PLAN-DAG: arith.constant 0 : i64 +// PLAN-DAG: arith.constant 4096 : i64 +// PLAN-DAG: arith.constant 8192 : i64 +// PLAN: pto.pointer_cast + +// SYNC-LABEL: AICORE void plan_memory_address_provenance_no_false_alias( +// SYNC: TMULS( +// SYNC-NOT: set_flag(PIPE_V, PIPE_MTE2 +// SYNC: TLOAD( From 5d1169f48bbebd59c42e163c1c451bbb1732af5a Mon Sep 17 00:00:00 2001 From: Toni Boehnlein Date: Thu, 16 Jul 2026 13:17:35 +0200 Subject: [PATCH 2/5] fix(insert-sync): offset every physical slot --- .../PTO/Transforms/InsertSync/SyncCommon.h | 25 ++- .../Transforms/InsertSync/PTOIRTranslator.cpp | 44 +++++- .../pto/alloc_tile_physical_overlap_sync.pto | 142 ++++++++++++++++++ 3 files changed, 201 insertions(+), 10 deletions(-) diff --git a/include/PTO/Transforms/InsertSync/SyncCommon.h b/include/PTO/Transforms/InsertSync/SyncCommon.h index d130f17cdd..26c4a726c1 100644 --- a/include/PTO/Transforms/InsertSync/SyncCommon.h +++ b/include/PTO/Transforms/InsertSync/SyncCommon.h @@ -90,6 +90,14 @@ enum class AddressProvenance { /// The buffer has an absolute address, but its value is dynamic or unknown. UnknownAbsolute }; + +/// Describes what multiple entries in BaseMemInfo::baseAddresses represent. +enum class AddressListKind { + /// Each entry is the start of a discontiguous segment of one logical view. + Segments, + /// Each entry is an alternative physical slot for the same logical view. + AlternativeSlots +}; /// Meminfo of the target buffer /// 用于追踪 Buffer 的别名和根节点 @@ -97,10 +105,14 @@ struct BaseMemInfo { BaseMemInfo( Value baseBuffer, Value rootBuffer, pto::AddressSpace scope, SmallVector baseAddresses, uint64_t allocateSize, - AddressProvenance addressProvenance = AddressProvenance::RootRelative) + AddressProvenance addressProvenance = AddressProvenance::RootRelative, + AddressListKind addressListKind = AddressListKind::Segments, + bool hasAppliedReinterpret = false) : baseBuffer(baseBuffer), rootBuffer(rootBuffer), scope(scope), baseAddresses(std::move(baseAddresses)), allocateSize(allocateSize), - addressProvenance(addressProvenance) {} + addressProvenance(addressProvenance), + addressListKind(addressListKind), + hasAppliedReinterpret(hasAppliedReinterpret) {} /// baseBuffer: 当前操作直接使用的 Buffer (可能是 View 或 Alias) Value baseBuffer; @@ -111,6 +123,9 @@ struct BaseMemInfo { SmallVector baseAddresses; // 用于 Offset 分析 uint64_t allocateSize; AddressProvenance addressProvenance; + AddressListKind addressListKind; + /// True once a reinterpret_cast descriptor offset has been modeled. + bool hasAppliedReinterpret; bool areVectorEqual(const SmallVector& vec1, const SmallVector& vec2) const { @@ -127,6 +142,8 @@ struct BaseMemInfo { if (scope != other.scope) return false; if (addressProvenance != other.addressProvenance) return false; + if (addressListKind != other.addressListKind) + return false; // allocateSize 和 baseBuffer 的严格相等性在某些别名分析中可能太强了, // 但为了保持原有逻辑,先保留。重点是 rootBuffer 必须一致。 if (allocateSize != other.allocateSize) return false; @@ -137,13 +154,13 @@ struct BaseMemInfo { std::unique_ptr clone() const { return std::make_unique( baseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - addressProvenance); + addressProvenance, addressListKind, hasAppliedReinterpret); } std::unique_ptr clone(Value cloneBaseBuffer) const { return std::make_unique( cloneBaseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - addressProvenance); + addressProvenance, addressListKind, hasAppliedReinterpret); } }; diff --git a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp index 11d8a3037a..c87ecd41a7 100644 --- a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp +++ b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp @@ -112,6 +112,8 @@ static std::optional addByteOffset(uint64_t base, int64_t offset) { static void markAddressRangeUnknown(BaseMemInfo &info) { info.baseAddresses.clear(); info.allocateSize = 0; + info.addressListKind = AddressListKind::Segments; + info.hasAppliedReinterpret = false; if (info.addressProvenance == AddressProvenance::KnownAbsolute) info.addressProvenance = AddressProvenance::UnknownAbsolute; } @@ -630,7 +632,8 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { auto newMemInfo = std::make_unique( res, res, space, std::move(slotOffsets), sizeInBytes, isLocalAddressSpace(space) ? AddressProvenance::KnownAbsolute - : AddressProvenance::RootRelative); + : AddressProvenance::RootRelative, + AddressListKind::AlternativeSlots); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); } @@ -1012,11 +1015,36 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { auto newInfo = parentInfo->clone(result); bool addressRangeUnknown = false; - if (!newInfo->baseAddresses.empty()) { - if (std::optional address = - addByteOffset(newInfo->baseAddresses[0], deltaOffset)) { - newInfo->baseAddresses[0] = *address; - } else { + // A first reinterpret_cast offset applies independently to every + // alternative multi-buffer slot. A segmented subview is different: its + // entries are pieces of one logical view, and changing the reinterpret + // offset/strides can merge or repartition those pieces. Chained + // reinterpret_cast offsets also replace, rather than accumulate with, the + // previous descriptor offset. Fall back to conservative aliasing whenever + // either case cannot be represented exactly. + bool isReinterpret = + isa_and_nonnull(result.getDefiningOp()); + bool cannotModelReinterpretExactly = + isReinterpret && + ((newInfo->addressListKind == AddressListKind::Segments && + newInfo->baseAddresses.size() > 1) || + newInfo->hasAppliedReinterpret); + if (cannotModelReinterpretExactly) { + markAddressRangeUnknown(*newInfo); + addressRangeUnknown = true; + } else if (!newInfo->baseAddresses.empty()) { + bool addressOverflow = false; + const size_t addressCount = newInfo->baseAddresses.size(); + for (size_t i = 0; i < addressCount; ++i) { + if (std::optional address = + addByteOffset(newInfo->baseAddresses[i], deltaOffset)) { + newInfo->baseAddresses[i] = *address; + } else { + addressOverflow = true; + break; + } + } + if (addressOverflow) { markAddressRangeUnknown(*newInfo); addressRangeUnknown = true; } @@ -1032,6 +1060,8 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { if (newSize > 0 && !addressRangeUnknown) { newInfo->allocateSize = newSize; } + if (isReinterpret && !addressRangeUnknown) + newInfo->hasAppliedReinterpret = true; resultMemInfoVec.emplace_back(std::move(newInfo)); } @@ -1161,6 +1191,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; + newInfo->addressListKind = AddressListKind::Segments; resultMemInfoVec.emplace_back(std::move(newInfo)); } } @@ -1228,6 +1259,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; + newInfo->addressListKind = AddressListKind::Segments; resultMemInfoVec.emplace_back(std::move(newInfo)); } } diff --git a/test/lit/pto/alloc_tile_physical_overlap_sync.pto b/test/lit/pto/alloc_tile_physical_overlap_sync.pto index f2ccaad5fa..9243099e3c 100644 --- a/test/lit/pto/alloc_tile_physical_overlap_sync.pto +++ b/test/lit/pto/alloc_tile_physical_overlap_sync.pto @@ -147,6 +147,130 @@ module attributes {pto.target_arch = "a2a3"} { outs(%index_row : !pto.tile_buf) return } + + // A dynamic slot selection conservatively tracks every physical address. A + // downstream reinterpret_cast offset applies to every possible slot, not + // just slot 0. Here slot 1 moves from 8192 to 8704, where it overlaps the + // following MTE2 write. + func.func @multi_slot_alias_offset_sync( + %index_src: memref<1x8xf32, #pto.address_space>, + %slot: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c8704_i64 = arith.constant 8704 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %slots = pto.pointer_cast(%c4096_i64, %c8192_i64) + : memref<1x8xf32, #pto.address_space> + %selected = pto.slot_marker %slots[%slot] + : memref<1x8xf32, #pto.address_space> + -> memref<1x8xf32, #pto.address_space> + %shifted = memref.reinterpret_cast %selected to offset: [128], + sizes: [1, 8], strides: [8, 1] + : memref<1x8xf32, #pto.address_space> + to memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c8704_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%shifted, %one + : memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } + + // A subview uses one address entry per discontiguous row segment. A + // reinterpret_cast changes the logical range from [0, 32) + [64, 96) to + // [32, 96); treating both entries like alternative slots would instead + // invent [32, 64) + [96, 128) and miss the write at address 64. + func.func @segmented_alias_reinterpret_sync( + %index_src: memref<1x8xf32, #pto.address_space>, + %n: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c0_i64 = arith.constant 0 : i64 + %c64_i64 = arith.constant 64 : i64 + %c128_i64 = arith.constant 128 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c0_i64) + : memref<2x16xf32, #pto.address_space> + %segments = memref.subview %root[0, 0] [2, 8] [1, 1] + : memref<2x16xf32, #pto.address_space> + to memref<2x8xf32, strided<[16, 1]>, #pto.address_space> + %vector_out = pto.pointer_cast(%c128_i64) + : memref<2x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c64_i64) + : memref<1x8xf32, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 + iter_args(%iter = %segments) + -> (memref<2x8xf32, strided<[16, 1]>, #pto.address_space>) { + %reinterpreted = memref.reinterpret_cast %iter to offset: [8], + sizes: [2, 8], strides: [8, 1] + : memref<2x8xf32, strided<[16, 1]>, #pto.address_space> + to memref<2x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + pto.tmuls ins(%reinterpreted, %one + : memref<2x8xf32, strided<[8, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<2x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + scf.yield %iter + : memref<2x8xf32, strided<[16, 1]>, #pto.address_space> + } + return + } + + // reinterpret_cast offsets are relative to the underlying allocation, not + // the source descriptor's current offset. The final cast resets either + // selected offset back to 0, so slot 1 again overlaps the write at 8192. + func.func @chained_reinterpret_resets_slot_offset_sync( + %index_src: memref<1x8xf32, #pto.address_space>, + %slot: index, + %choose: i1) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %slots = pto.pointer_cast(%c4096_i64, %c8192_i64) + : memref<1x8xf32, #pto.address_space> + %selected = pto.slot_marker %slots[%slot] + : memref<1x8xf32, #pto.address_space> + -> memref<1x8xf32, #pto.address_space> + %shifted128 = memref.reinterpret_cast %selected to offset: [128], + sizes: [1, 8], strides: [8, 1] + : memref<1x8xf32, #pto.address_space> + to memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + %shifted256 = memref.reinterpret_cast %selected to offset: [256], + sizes: [1, 8], strides: [8, 1] + : memref<1x8xf32, #pto.address_space> + to memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + %descriptor = arith.select %choose, %shifted128, %shifted256 + : memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + %reset = memref.reinterpret_cast %descriptor to offset: [0], + sizes: [1, 8], strides: [16, 1] + : memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + to memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c8192_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%reset, %one + : memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } } // CHECK-LABEL: AICORE void alloc_tile_physical_overlap_sync( @@ -176,3 +300,21 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[UNKNOWN_V_TO_MTE2:[0-9]+]]); // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[UNKNOWN_V_TO_MTE2]]); // CHECK: TLOAD( + +// CHECK-LABEL: AICORE void multi_slot_alias_offset_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SLOT_OFFSET_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SLOT_OFFSET_V_TO_MTE2]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void segmented_alias_reinterpret_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_V_TO_MTE2]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void chained_reinterpret_resets_slot_offset_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_V_TO_MTE2]]); +// CHECK: TLOAD( From 4358fb324123cd51f41834b2c5e51e5781840026 Mon Sep 17 00:00:00 2001 From: Toni Boehnlein Date: Thu, 16 Jul 2026 15:15:27 +0200 Subject: [PATCH 3/5] fix(insert-sync): invalidate rebased reinterpret views --- .../Transforms/InsertSync/PTOIRTranslator.h | 1 + .../PTO/Transforms/InsertSync/SyncCommon.h | 14 +- .../InsertSync/InsertSyncAnalysis.cpp | 43 ++++-- .../Transforms/InsertSync/PTOIRTranslator.cpp | 63 +++++--- .../pto/alloc_tile_physical_overlap_sync.pto | 143 ++++++++++++++++++ 5 files changed, 226 insertions(+), 38 deletions(-) diff --git a/include/PTO/Transforms/InsertSync/PTOIRTranslator.h b/include/PTO/Transforms/InsertSync/PTOIRTranslator.h index a919f97441..dd6b2da14c 100644 --- a/include/PTO/Transforms/InsertSync/PTOIRTranslator.h +++ b/include/PTO/Transforms/InsertSync/PTOIRTranslator.h @@ -78,6 +78,7 @@ class PTOIRTranslator { // 处理 View/Alias (MakeTensorView, Subview, Mov) void UpdateAliasBufferInfo(Value result, Value source); void UpdateConservativeAliasBufferInfo(Value result, Value source); + void UpdateConservativeSubviewAliasBufferInfo(Value result, Value source); void UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op); void UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op); void UpdateSlotMarkerAliasBufferInfo(pto::SlotMarkerOp op); diff --git a/include/PTO/Transforms/InsertSync/SyncCommon.h b/include/PTO/Transforms/InsertSync/SyncCommon.h index 26c4a726c1..85768677cf 100644 --- a/include/PTO/Transforms/InsertSync/SyncCommon.h +++ b/include/PTO/Transforms/InsertSync/SyncCommon.h @@ -107,12 +107,14 @@ struct BaseMemInfo { SmallVector baseAddresses, uint64_t allocateSize, AddressProvenance addressProvenance = AddressProvenance::RootRelative, AddressListKind addressListKind = AddressListKind::Segments, - bool hasAppliedReinterpret = false) + bool hasAppliedReinterpret = false, + bool hasAppliedSubviewOffset = false) : baseBuffer(baseBuffer), rootBuffer(rootBuffer), scope(scope), baseAddresses(std::move(baseAddresses)), allocateSize(allocateSize), addressProvenance(addressProvenance), addressListKind(addressListKind), - hasAppliedReinterpret(hasAppliedReinterpret) {} + hasAppliedReinterpret(hasAppliedReinterpret), + hasAppliedSubviewOffset(hasAppliedSubviewOffset) {} /// baseBuffer: 当前操作直接使用的 Buffer (可能是 View 或 Alias) Value baseBuffer; @@ -126,6 +128,8 @@ struct BaseMemInfo { AddressListKind addressListKind; /// True once a reinterpret_cast descriptor offset has been modeled. bool hasAppliedReinterpret; + /// True once a subview offset has adjusted the physical address list. + bool hasAppliedSubviewOffset; bool areVectorEqual(const SmallVector& vec1, const SmallVector& vec2) const { @@ -154,13 +158,15 @@ struct BaseMemInfo { std::unique_ptr clone() const { return std::make_unique( baseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - addressProvenance, addressListKind, hasAppliedReinterpret); + addressProvenance, addressListKind, hasAppliedReinterpret, + hasAppliedSubviewOffset); } std::unique_ptr clone(Value cloneBaseBuffer) const { return std::make_unique( cloneBaseBuffer, rootBuffer, scope, baseAddresses, allocateSize, - addressProvenance, addressListKind, hasAppliedReinterpret); + addressProvenance, addressListKind, hasAppliedReinterpret, + hasAppliedSubviewOffset); } }; diff --git a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp index 2fd3a44526..c3522d367b 100644 --- a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp +++ b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp @@ -449,19 +449,35 @@ static bool isForwardDepDroppableBySlotAffine(const BaseMemInfo *a, const BaseMemInfo *b) { if (!a || !b) return false; - size_t aN = a->baseAddresses.size(); - size_t bN = b->baseAddresses.size(); - size_t n = std::max(aN, bN); - if (n < 2) + if (a->addressListKind != AddressListKind::AlternativeSlots || + b->addressListKind != AddressListKind::AlternativeSlots) return false; Value slotA = findSlotMarkerExpr(a->baseBuffer); Value slotB = findSlotMarkerExpr(b->baseBuffer); if (!slotA || !slotB) return false; + size_t aN = a->baseAddresses.size(); + size_t bN = b->baseAddresses.size(); + size_t n = std::max(aN, bN); + if (n < 2) + return false; return compareSlotSSA(slotA, slotB, static_cast(n)) == SlotRelation::kDisjoint; } +static size_t getAlternativeSlotPairCount(const BaseMemInfo *a, + const BaseMemInfo *b) { + if (!a || !b) + return 1; + if (a->addressListKind != AddressListKind::AlternativeSlots || + b->addressListKind != AddressListKind::AlternativeSlots) + return 1; + if (!findSlotMarkerExpr(a->baseBuffer) || + !findSlotMarkerExpr(b->baseBuffer)) + return 1; + return std::max(a->baseAddresses.size(), b->baseAddresses.size()); +} + void InsertSyncAnalysis::MemAnalyze( CompoundInstanceElement *nowCompound, CompoundInstanceElement *frontCompound, SyncRecordList &syncRecordList, @@ -628,6 +644,8 @@ void InsertSyncAnalysis::InsertSyncOperation( Value producerSlot; Value consumerSlot; for (auto &pair : depBaseMemInfosVec) { + if (getAlternativeSlotPairCount(pair.first, pair.second) <= 1) + continue; if (pair.second && pair.second->baseBuffer) producerSlot = findSlotMarkerExpr(pair.second->baseBuffer); if (pair.first && pair.first->baseBuffer) @@ -810,15 +828,9 @@ SmallVector InsertSyncAnalysis::GetMemInfoBuffers( int InsertSyncAnalysis::GetEventIdNum( const DepBaseMemInfoPairVec &depBaseMemInfosVec) { - // A back-edge dependency benefits from N dynamic event IDs whenever at - // least one side is a multi-buffer access. We detect that from the - // BaseMemInfo's `baseAddresses` size, which `UpdateSlotMarkerAliasBufferInfo` - // populated: - // - kSingle / const-slot : size == 1 - // - dyn-slot (PTOIRTranslator default) : size == N (all slots, conservative) - // For the alias to even reach this point both sides share a root, so the - // slot count derived from either side's full address set should be the - // same N. We pick the max to be robust against accidental narrowing. + // A back-edge dependency uses N dynamic event IDs only when both accesses + // are alternative-slot views with recoverable slot SSA expressions. + // Multiple segmented addresses are one logical view, not N event lanes. int eventIdNum = 1; for (const auto &pair : depBaseMemInfosVec) { bool isLocalA = @@ -829,9 +841,8 @@ int InsertSyncAnalysis::GetEventIdNum( pair.second->scope == pto::AddressSpace::VEC); if (!isLocalA && !isLocalB) continue; - size_t aN = pair.first ? pair.first->baseAddresses.size() : 1; - size_t bN = pair.second ? pair.second->baseAddresses.size() : 1; - int pairN = static_cast(std::max(aN, bN)); + int pairN = static_cast( + getAlternativeSlotPairCount(pair.first, pair.second)); if (pairN <= 1) continue; if (eventIdNum == 1) { diff --git a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp index c87ecd41a7..f99d4a7bd4 100644 --- a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp +++ b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp @@ -114,6 +114,7 @@ static void markAddressRangeUnknown(BaseMemInfo &info) { info.allocateSize = 0; info.addressListKind = AddressListKind::Segments; info.hasAppliedReinterpret = false; + info.hasAppliedSubviewOffset = false; if (info.addressProvenance == AddressProvenance::KnownAbsolute) info.addressProvenance = AddressProvenance::UnknownAbsolute; } @@ -344,7 +345,7 @@ static std::pair getStaticOffsetAndSize(Operation *op, if (auto castOp = dyn_cast(op)) { auto staticOffsets = castOp.getStaticOffsets(); if (staticOffsets.empty() || staticOffsets[0] == ShapedType::kDynamic) { - return {0, 0}; + return {-1, -1}; } return {staticOffsets[0] * elemSize, 0}; } @@ -999,6 +1000,7 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { int64_t deltaOffset = 0; int64_t newSize = -1; + bool isDynamicReinterpretOffset = false; if (auto op = result.getDefiningOp()) { auto info = getStaticOffsetAndSize(op, source); @@ -1007,6 +1009,12 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { if (info.second > 0) newSize = info.second; } + if (auto reinterpret = dyn_cast(op)) { + ArrayRef staticOffsets = reinterpret.getStaticOffsets(); + isDynamicReinterpretOffset = + staticOffsets.empty() || + staticOffsets.front() == ShapedType::kDynamic; + } } auto &resultMemInfoVec = buffer2MemInfoMap_[result]; @@ -1015,18 +1023,18 @@ void PTOIRTranslator::UpdateAliasBufferInfo(Value result, Value source) { auto newInfo = parentInfo->clone(result); bool addressRangeUnknown = false; - // A first reinterpret_cast offset applies independently to every - // alternative multi-buffer slot. A segmented subview is different: its - // entries are pieces of one logical view, and changing the reinterpret - // offset/strides can merge or repartition those pieces. Chained - // reinterpret_cast offsets also replace, rather than accumulate with, the - // previous descriptor offset. Fall back to conservative aliasing whenever - // either case cannot be represented exactly. + // A first static reinterpret_cast offset applies independently to every + // underlying alternative multi-buffer slot. It cannot be accumulated from + // addresses already adjusted by a subview, because reinterpret_cast + // replaces descriptor metadata relative to the underlying allocation. + // Dynamic offsets, segmented views, and chained reinterpret_casts likewise + // cannot be represented exactly by the current address list. bool isReinterpret = isa_and_nonnull(result.getDefiningOp()); bool cannotModelReinterpretExactly = isReinterpret && - ((newInfo->addressListKind == AddressListKind::Segments && + (isDynamicReinterpretOffset || newInfo->hasAppliedSubviewOffset || + (newInfo->addressListKind == AddressListKind::Segments && newInfo->baseAddresses.size() > 1) || newInfo->hasAppliedReinterpret); if (cannotModelReinterpretExactly) { @@ -1079,6 +1087,20 @@ void PTOIRTranslator::UpdateConservativeAliasBufferInfo(Value result, resultMemInfoVec.emplace_back(parentInfo->clone(result)); } +void PTOIRTranslator::UpdateConservativeSubviewAliasBufferInfo(Value result, + Value source) { + if (!result || !source) return; + if (!buffer2MemInfoMap_.contains(source)) return; + + auto &resultMemInfoVec = buffer2MemInfoMap_[result]; + for (auto &parentInfo : buffer2MemInfoMap_[source]) { + auto newInfo = parentInfo->clone(result); + markAddressRangeUnknown(*newInfo); + newInfo->hasAppliedSubviewOffset = true; + resultMemInfoVec.emplace_back(std::move(newInfo)); + } +} + void PTOIRTranslator::UpdateSlotMarkerAliasBufferInfo(pto::SlotMarkerOp op) { Value result = op.getResult(); Value source = op.getSource(); @@ -1102,7 +1124,8 @@ void PTOIRTranslator::UpdateSlotMarkerAliasBufferInfo(pto::SlotMarkerOp op) { // two const-slot uses pick different slots. For a dynamic slot index, // keep all addresses -- the runtime SSA could resolve to any slot, so // sync must conservatively treat the use as touching all of them. - if (isConstSlot && constSlotIdx >= 0 && + if (newInfo->addressListKind == AddressListKind::AlternativeSlots && + isConstSlot && constSlotIdx >= 0 && constSlotIdx < static_cast(newInfo->baseAddresses.size()) && newInfo->baseAddresses.size() > 1) { uint64_t pickAddr = @@ -1124,7 +1147,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { auto sourceType = dyn_cast(source.getType()); if (!sourceType) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } @@ -1135,7 +1158,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { : getMemrefSubViewBaseAddresses( op, sourceType, static_cast(elemBytes)); if (!subViewAddresses || subViewAddresses->empty()) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } @@ -1144,7 +1167,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { if (failed(mlir::pto::getPTOMemRefStridesAndOffset(sourceType, strides, baseOffset)) || strides.size() != 2) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } @@ -1157,14 +1180,14 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { segmentSize = static_cast(staticSizes[0] * static_cast(elemBytes)); } else { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } for (auto &parentInfo : buffer2MemInfoMap_[source]) { if (!parentInfo || parentInfo->baseAddresses.size() != 1 || parentInfo->allocateSize == 0) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } } @@ -1186,12 +1209,14 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { } if (overflowed) { markAddressRangeUnknown(*newInfo); + newInfo->hasAppliedSubviewOffset = true; resultMemInfoVec.emplace_back(std::move(newInfo)); continue; } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; newInfo->addressListKind = AddressListKind::Segments; + newInfo->hasAppliedSubviewOffset = true; resultMemInfoVec.emplace_back(std::move(newInfo)); } } @@ -1206,7 +1231,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { auto sourceType = dyn_cast(source.getType()); if (!sourceType) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } @@ -1217,7 +1242,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { : getPtoSubViewBaseAddresses( op, sourceType, static_cast(elemBytes)); if (!subViewAddresses || subViewAddresses->empty()) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } @@ -1232,7 +1257,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { for (auto &parentInfo : buffer2MemInfoMap_[source]) { if (!parentInfo || parentInfo->baseAddresses.size() != 1 || parentInfo->allocateSize == 0) { - UpdateConservativeAliasBufferInfo(result, source); + UpdateConservativeSubviewAliasBufferInfo(result, source); return; } } @@ -1254,12 +1279,14 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { } if (overflowed) { markAddressRangeUnknown(*newInfo); + newInfo->hasAppliedSubviewOffset = true; resultMemInfoVec.emplace_back(std::move(newInfo)); continue; } newInfo->baseAddresses = std::move(addresses); newInfo->allocateSize = segmentSize; newInfo->addressListKind = AddressListKind::Segments; + newInfo->hasAppliedSubviewOffset = true; resultMemInfoVec.emplace_back(std::move(newInfo)); } } diff --git a/test/lit/pto/alloc_tile_physical_overlap_sync.pto b/test/lit/pto/alloc_tile_physical_overlap_sync.pto index 9243099e3c..bd7ef86ffa 100644 --- a/test/lit/pto/alloc_tile_physical_overlap_sync.pto +++ b/test/lit/pto/alloc_tile_physical_overlap_sync.pto @@ -271,6 +271,119 @@ module attributes {pto.target_arch = "a2a3"} { outs(%target : memref<1x8xf32, #pto.address_space>) return } + + // A dynamic reinterpret offset cannot be modeled as zero. At runtime the + // offset may move the view from 4096 to 8192, so the resulting local address + // must conservatively alias the following MTE2 write. + func.func @dynamic_reinterpret_offset_sync( + %index_src: memref<1x8xf32, #pto.address_space>, + %offset: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<1x8xf32, #pto.address_space> + %reinterpreted = memref.reinterpret_cast %root to offset: [%offset], + sizes: [1, 8], strides: [8, 1] + : memref<1x8xf32, #pto.address_space> + to memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c8192_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%reinterpreted, %one + : memref<1x8xf32, strided<[8, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } + + // Even a one-segment subview has already adjusted its physical address. + // reinterpret_cast offset 0 resets to the underlying allocation base 4096; + // accumulating it from the subview address 4160 would miss this overlap. + func.func @contiguous_subview_reinterpret_sync( + %index_src: memref<1x8xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, #pto.address_space> + %subview = memref.subview %root[1, 0] [1, 8] [1, 1] + : memref<2x16xf32, #pto.address_space> + to memref<1x8xf32, strided<[16, 1], offset: 16>, #pto.address_space> + %reset = memref.reinterpret_cast %subview to offset: [0], + sizes: [1, 8], strides: [8, 1] + : memref<1x8xf32, strided<[16, 1], offset: 16>, #pto.address_space> + to memref<1x8xf32, strided<[8, 1]>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c4096_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%reset, %one + : memref<1x8xf32, strided<[8, 1]>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } + + // The dynamic access is a real two-slot buffer, while %segments is two + // discontiguous pieces of fixed slot 0. Across the loop back-edge, the + // dynamic access needs per-slot event IDs, but the segmented access is one + // logical allocation and still needs its own single-event V -> MTE2 sync. + func.func @segmented_backedge_uses_single_event( + %index_src: memref<2x16xf32, #pto.address_space>, + %n: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %c32768_i64 = arith.constant 32768 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %slots = pto.pointer_cast(%c4096_i64, %c8192_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + %slot0 = pto.slot_marker %slots[%c0] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + -> memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + %segments = memref.subview %slot0[0, 0] [2, 8] [1, 1] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + to memref<2x8xf32, strided<[16, 1]>, #pto.address_space> + %segment_out = pto.pointer_cast(%c16384_i64) + : memref<2x8xf32, strided<[16, 1]>, #pto.address_space> + %slot_out = pto.pointer_cast(%c32768_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %idx = arith.remui %i, %c2 : index + %dynamic = pto.slot_marker %slots[%idx] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + -> memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + pto.tload ins(%index_src : memref<2x16xf32, #pto.address_space>) + outs(%dynamic + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space>) + pto.tmuls ins(%segments, %one + : memref<2x8xf32, strided<[16, 1]>, #pto.address_space>, f32) + outs(%segment_out + : memref<2x8xf32, strided<[16, 1]>, #pto.address_space>) + pto.tmuls ins(%dynamic, %one + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space>, f32) + outs(%slot_out + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space>) + } + return + } } // CHECK-LABEL: AICORE void alloc_tile_physical_overlap_sync( @@ -318,3 +431,33 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_V_TO_MTE2:[0-9]+]]); // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_V_TO_MTE2]]); // CHECK: TLOAD( + +// CHECK-LABEL: AICORE void dynamic_reinterpret_offset_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[DYNAMIC_REINTERPRET_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[DYNAMIC_REINTERPRET_V_TO_MTE2]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void contiguous_subview_reinterpret_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SUBVIEW_REINTERPRET_V_TO_MTE2:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SUBVIEW_REINTERPRET_V_TO_MTE2]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void segmented_backedge_uses_single_event( +// Two events belong to the real dynamic two-slot dependency. The third is the +// independent single-event dependency for the segmented view. +// CHECK: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID0); +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID1); +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_EVENT:[2-9][0-9]*]]); +// CHECK: for ( +// CHECK: wait_flag(PIPE_V, PIPE_MTE2, {{[_A-Za-z][_A-Za-z0-9]*}}); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_EVENT]]); +// CHECK: TLOAD( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_EVENT]]); +// CHECK: TMULS( +// CHECK: set_flag(PIPE_V, PIPE_MTE2, {{[_A-Za-z][_A-Za-z0-9]*}}); +// CHECK: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID0); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID1); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_EVENT]]); From 6544ee219f5e5e39cbfe65118167181a9e64543e Mon Sep 17 00:00:00 2001 From: Toni Boehnlein Date: Thu, 16 Jul 2026 15:53:59 +0200 Subject: [PATCH 4/5] fix(insert-sync): cover mixed and inexact subview hazards --- .../ptoas-multi-buffer-explicit-design.md | 5 +- .../PTO/Transforms/InsertSync/SyncCommon.h | 12 +- .../InsertSync/InsertSyncAnalysis.cpp | 47 ++--- .../Transforms/InsertSync/PTOIRTranslator.cpp | 76 +++++++- .../pto/alloc_tile_physical_overlap_sync.pto | 162 ++++++++++++++++++ 5 files changed, 273 insertions(+), 29 deletions(-) diff --git a/docs/designs/ptoas-multi-buffer-explicit-design.md b/docs/designs/ptoas-multi-buffer-explicit-design.md index d9c2b87cbb..afa731789c 100644 --- a/docs/designs/ptoas-multi-buffer-explicit-design.md +++ b/docs/designs/ptoas-multi-buffer-explicit-design.md @@ -467,7 +467,7 @@ lit test/lit/pto/multi_tile_prefetch_gss.pto | **`MemAlias` 自动识别 const-slot disjoint**:不同常量 slot 的 access baseAddresses disjoint → 不发同步 | ✅ | `lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp` (复用既有 range-overlap 逻辑) | | **GSS 别名链穿透 `pto.bind_tile` / `pto.slot_marker`** | ✅ | `lib/PTO/Transforms/Utils.cpp` (`getOperationAliasInfo`) | | **InsertSync 处理 arith.select-on-memref**:保留为防御逻辑(Resolve 移到 sync 后实际不再产生此场景) | ✅ | `lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp` | -| **`GetEventIdNum` 按多 slot 推 N**:back-edge dep 双侧 `baseAddresses.size() == N` 时返回 N | ✅ | `lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp` | +| **`GetEventIdNum` 按多 slot 推 N**:仅当 back-edge 的每个 local dep pair 都是同 N 的 `AlternativeSlots` 且双侧 slot SSA 可恢复时返回 N;`Segments`、缺失 slot SSA 或混合依赖回退到单个静态 event | ✅ | `lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp` | | **`SyncOperation` 携带 slot SSA**:`slotSSAExpr` + `slotCount` 字段,set/wait 各自记录自己一侧的 slot SSA | ✅ | `include/PTO/Transforms/InsertSync/SyncCommon.h`, `lib/PTO/Transforms/InsertSync/{SyncCommon,InsertSyncAnalysis}.cpp` | | **dyn event-id codegen**:`CreateSetWaitOpForMultiBuffer` 发 `pto.set_flag_dyn` / `pto.wait_flag_dyn`,event id 由 N-way `arith.select` 链根据 `slot % N` 选自分配的 N 个静态 event id | ✅ | `lib/PTO/Transforms/InsertSync/SyncCodegen.cpp` | | **`SyncEventIdAllocation` N event ids 分配**:复用既有 `eventIdNum > 1` 路径(已支持 N),自动给 set/wait 对分配 N 个 hardware event id | ✅ pre-existing | `lib/PTO/Transforms/InsertSync/SyncEventIdAllocation.cpp` | @@ -480,11 +480,12 @@ lit test/lit/pto/multi_tile_prefetch_gss.pto | **GSS 同 SSA 行为对齐 InsertSync**:`getMultiBufferEventIdInfo` 不再因 all-equal slot SSA 早退,对同 SSA 的 producer/consumer 也走 N dyn event id 路径;GSS 同 SSA 现在 emit 与 InsertSync 完全一样的 prefetch pipeline | ✅ | `lib/PTO/Transforms/GraphSyncSolver/SyncSolver.cpp` | | **EmitC 穿透 `arith.select`-on-memref**:`PTOMaterializeTileHandles::computeExplicitAddress` 沿 select 两支递归求 i64 地址,配合 dyn slot 路径的 select 链;EmitC TASSIGN 拿到正确地址 | ✅ | `lib/PTO/Transforms/PTOMaterializeTileHandles.cpp` | | **`multi_tile_get` lowering 防御**:`op->getOperand(0)` 取 source(绕开类型 cast),因为 alloc_multi_tile replace 之后 source 已经变成 memref,typed accessor `getSource()` 会断言 | ✅ | `lib/PTO/Transforms/PTOViewToMemref.cpp` | -| lit 测试(17 个):parse/print、verifier、const slot lowering、dyn slot lowering、N=3 / N=4 端到端、无 loop unroll、const-slot sync disjoint、dyn-slot sync 编译、GSS multi-buffer compile、prefetch dyn event-id (InsertSync)、prefetch GSS dyn flag、affine disjoint slots、const preload + dyn loop select、preload + loop set/wait、unknown slot GSS 保守降级 | ✅ | `test/lit/pto/multi_tile_*.pto` | +| lit 测试(21 个):parse/print、verifier、const slot lowering、dyn slot lowering、N=3 / N=4 端到端、无 loop unroll、const-slot sync disjoint、dyn-slot sync 编译、GSS multi-buffer compile、prefetch dyn event-id (InsertSync)、prefetch GSS dyn flag、affine disjoint slots、const preload + dyn loop select、preload + loop set/wait、unknown slot GSS 保守降级等 | ✅ | `test/lit/pto/multi_tile_*.pto` | ### 当前限制 - **affine 分析仅覆盖核心几种形态**:`compareSlotSSA` 当前能证 `iv % N` / `(iv ± c) % N` / 同 SSA / 纯常量;不能证 `(iv * c) % N`、跨函数 / 跨循环的 SSA 等价、非 `arith.remui` 包装的 slot 表达式。命中不到时退回 kUnknown / 保守 N dyn event id。 +- **混合 back-edge 依赖回退到静态 event**:只要同一同步候选还包含 `Segments` 或其他不能与 slot lane 一一对应的 local dep pair,InsertSync 就使用单个静态 event 覆盖全部 hazard,而不是只按 multi-buffer slot 派发。 - **PlanMemory N>2 不复用 Stage1**:N>2 的兄弟 slot 不走 SPEC_LEVEL_1 "ping/pong 相邻摆放"优化,用更多内存。N=2 路径不变。 - 初版仅支持 `loc=vec` / `loc=mat` local memory。 - function argument / return 上的 `multi_tile_buf` 不支持(多 buffer 所有权限定在 ptoas 内)。 diff --git a/include/PTO/Transforms/InsertSync/SyncCommon.h b/include/PTO/Transforms/InsertSync/SyncCommon.h index 85768677cf..e36c603f03 100644 --- a/include/PTO/Transforms/InsertSync/SyncCommon.h +++ b/include/PTO/Transforms/InsertSync/SyncCommon.h @@ -108,13 +108,15 @@ struct BaseMemInfo { AddressProvenance addressProvenance = AddressProvenance::RootRelative, AddressListKind addressListKind = AddressListKind::Segments, bool hasAppliedReinterpret = false, - bool hasAppliedSubviewOffset = false) + bool hasAppliedSubviewOffset = false, + bool hasInexactSubviewRange = false) : baseBuffer(baseBuffer), rootBuffer(rootBuffer), scope(scope), baseAddresses(std::move(baseAddresses)), allocateSize(allocateSize), addressProvenance(addressProvenance), addressListKind(addressListKind), hasAppliedReinterpret(hasAppliedReinterpret), - hasAppliedSubviewOffset(hasAppliedSubviewOffset) {} + hasAppliedSubviewOffset(hasAppliedSubviewOffset), + hasInexactSubviewRange(hasInexactSubviewRange) {} /// baseBuffer: 当前操作直接使用的 Buffer (可能是 View 或 Alias) Value baseBuffer; @@ -130,6 +132,8 @@ struct BaseMemInfo { bool hasAppliedReinterpret; /// True once a subview offset has adjusted the physical address list. bool hasAppliedSubviewOffset; + /// True when the range is a conservative parent envelope, not an exact view. + bool hasInexactSubviewRange; bool areVectorEqual(const SmallVector& vec1, const SmallVector& vec2) const { @@ -159,14 +163,14 @@ struct BaseMemInfo { return std::make_unique( baseBuffer, rootBuffer, scope, baseAddresses, allocateSize, addressProvenance, addressListKind, hasAppliedReinterpret, - hasAppliedSubviewOffset); + hasAppliedSubviewOffset, hasInexactSubviewRange); } std::unique_ptr clone(Value cloneBaseBuffer) const { return std::make_unique( cloneBaseBuffer, rootBuffer, scope, baseAddresses, allocateSize, addressProvenance, addressListKind, hasAppliedReinterpret, - hasAppliedSubviewOffset); + hasAppliedSubviewOffset, hasInexactSubviewRange); } }; diff --git a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp index c3522d367b..8590af626f 100644 --- a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp +++ b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp @@ -456,26 +456,28 @@ static bool isForwardDepDroppableBySlotAffine(const BaseMemInfo *a, Value slotB = findSlotMarkerExpr(b->baseBuffer); if (!slotA || !slotB) return false; - size_t aN = a->baseAddresses.size(); - size_t bN = b->baseAddresses.size(); - size_t n = std::max(aN, bN); - if (n < 2) + if (a->baseAddresses.size() < 2 || + a->baseAddresses.size() != b->baseAddresses.size()) return false; - return compareSlotSSA(slotA, slotB, static_cast(n)) == + return compareSlotSSA(slotA, slotB, + static_cast(a->baseAddresses.size())) == SlotRelation::kDisjoint; } -static size_t getAlternativeSlotPairCount(const BaseMemInfo *a, - const BaseMemInfo *b) { +static std::optional +getAlternativeSlotPairCount(const BaseMemInfo *a, const BaseMemInfo *b) { if (!a || !b) - return 1; + return std::nullopt; if (a->addressListKind != AddressListKind::AlternativeSlots || b->addressListKind != AddressListKind::AlternativeSlots) - return 1; + return std::nullopt; if (!findSlotMarkerExpr(a->baseBuffer) || !findSlotMarkerExpr(b->baseBuffer)) - return 1; - return std::max(a->baseAddresses.size(), b->baseAddresses.size()); + return std::nullopt; + if (a->baseAddresses.size() < 2 || + a->baseAddresses.size() != b->baseAddresses.size()) + return std::nullopt; + return a->baseAddresses.size(); } void InsertSyncAnalysis::MemAnalyze( @@ -644,7 +646,7 @@ void InsertSyncAnalysis::InsertSyncOperation( Value producerSlot; Value consumerSlot; for (auto &pair : depBaseMemInfosVec) { - if (getAlternativeSlotPairCount(pair.first, pair.second) <= 1) + if (!getAlternativeSlotPairCount(pair.first, pair.second)) continue; if (pair.second && pair.second->baseBuffer) producerSlot = findSlotMarkerExpr(pair.second->baseBuffer); @@ -829,9 +831,10 @@ SmallVector InsertSyncAnalysis::GetMemInfoBuffers( int InsertSyncAnalysis::GetEventIdNum( const DepBaseMemInfoPairVec &depBaseMemInfosVec) { // A back-edge dependency uses N dynamic event IDs only when both accesses - // are alternative-slot views with recoverable slot SSA expressions. - // Multiple segmented addresses are one logical view, not N event lanes. - int eventIdNum = 1; + // in every local dependency pair are compatible alternative-slot views with + // recoverable slot SSA expressions. A segmented or otherwise incompatible + // pair forces one static event so the synchronization covers every hazard. + std::optional eventIdNum; for (const auto &pair : depBaseMemInfosVec) { bool isLocalA = pair.first && (pair.first->scope == pto::AddressSpace::MAT || @@ -841,20 +844,20 @@ int InsertSyncAnalysis::GetEventIdNum( pair.second->scope == pto::AddressSpace::VEC); if (!isLocalA && !isLocalB) continue; - int pairN = static_cast( - getAlternativeSlotPairCount(pair.first, pair.second)); - if (pairN <= 1) - continue; - if (eventIdNum == 1) { + auto pairCount = getAlternativeSlotPairCount(pair.first, pair.second); + if (!pairCount) + return 1; + int pairN = static_cast(*pairCount); + if (!eventIdNum) { eventIdNum = pairN; - } else if (eventIdNum != pairN) { + } else if (*eventIdNum != pairN) { // Multiple dep pairs disagreeing on N: fall back to single event id // for safety. With more work this could be relaxed by per-pair // multi-buffer reasoning. return 1; } } - return eventIdNum; + return eventIdNum.value_or(1); } bool InsertSyncAnalysis::IsGMHazard( diff --git a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp index f99d4a7bd4..796307fa5c 100644 --- a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp +++ b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp @@ -109,12 +109,68 @@ static std::optional addByteOffset(uint64_t base, int64_t offset) { return base - magnitude; } +static std::optional +getContiguousMemRefSizeInBytes(MemRefType type) { + if (!type.hasStaticShape() || type.getRank() != kTileRank2D) + return std::nullopt; + + SmallVector strides; + int64_t offset = ShapedType::kDynamic; + if (failed(mlir::pto::getPTOMemRefStridesAndOffset(type, strides, offset)) || + strides.size() != kTileRank2D || + llvm::is_contained(strides, ShapedType::kDynamic)) + return std::nullopt; + if (offset != 0) + return std::nullopt; + + ArrayRef shape = type.getShape(); + if (shape[0] <= 0 || shape[1] <= 0 || strides[0] < 0 || strides[1] < 0) + return std::nullopt; + bool rowMajorContiguous = + strides[1] == 1 && (shape[0] == 1 || strides[0] == shape[1]); + bool colMajorContiguous = + strides[0] == 1 && (shape[1] == 1 || strides[1] == shape[0]); + if (!rowMajorContiguous && !colMajorContiguous) + return std::nullopt; + + uint64_t elemBytes = pto::getPTOStorageElemByteSize(type.getElementType()); + if (elemBytes == 0) + return std::nullopt; + uint64_t rows = static_cast(shape[0]); + uint64_t cols = static_cast(shape[1]); + if (cols > std::numeric_limits::max() / rows) + return std::nullopt; + uint64_t elements = rows * cols; + if (elemBytes > std::numeric_limits::max() / elements) + return std::nullopt; + return elements * elemBytes; +} + +static bool canRetainSubviewParentRange(Value source, + const BaseMemInfo &parentInfo) { + if (parentInfo.baseAddresses.empty() || parentInfo.allocateSize == 0 || + parentInfo.addressProvenance == AddressProvenance::UnknownAbsolute) + return false; + if (parentInfo.hasInexactSubviewRange) + return true; + if (parentInfo.hasAppliedSubviewOffset && + parentInfo.addressListKind == AddressListKind::Segments) + return true; + + auto sourceType = dyn_cast(source.getType()); + if (!sourceType) + return false; + auto contiguousSize = getContiguousMemRefSizeInBytes(sourceType); + return contiguousSize && parentInfo.allocateSize >= *contiguousSize; +} + static void markAddressRangeUnknown(BaseMemInfo &info) { info.baseAddresses.clear(); info.allocateSize = 0; info.addressListKind = AddressListKind::Segments; info.hasAppliedReinterpret = false; info.hasAppliedSubviewOffset = false; + info.hasInexactSubviewRange = false; if (info.addressProvenance == AddressProvenance::KnownAbsolute) info.addressProvenance = AddressProvenance::UnknownAbsolute; } @@ -1095,8 +1151,10 @@ void PTOIRTranslator::UpdateConservativeSubviewAliasBufferInfo(Value result, auto &resultMemInfoVec = buffer2MemInfoMap_[result]; for (auto &parentInfo : buffer2MemInfoMap_[source]) { auto newInfo = parentInfo->clone(result); - markAddressRangeUnknown(*newInfo); + if (!canRetainSubviewParentRange(source, *parentInfo)) + markAddressRangeUnknown(*newInfo); newInfo->hasAppliedSubviewOffset = true; + newInfo->hasInexactSubviewRange = true; resultMemInfoVec.emplace_back(std::move(newInfo)); } } @@ -1150,6 +1208,12 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { UpdateConservativeSubviewAliasBufferInfo(result, source); return; } + if (llvm::any_of(buffer2MemInfoMap_[source], [](const auto &parentInfo) { + return !parentInfo || parentInfo->hasInexactSubviewRange; + })) { + UpdateConservativeSubviewAliasBufferInfo(result, source); + return; + } unsigned elemBytes = pto::getPTOStorageElemByteSize(sourceType.getElementType()); @@ -1210,6 +1274,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { if (overflowed) { markAddressRangeUnknown(*newInfo); newInfo->hasAppliedSubviewOffset = true; + newInfo->hasInexactSubviewRange = true; resultMemInfoVec.emplace_back(std::move(newInfo)); continue; } @@ -1217,6 +1282,7 @@ void PTOIRTranslator::UpdateMemrefSubViewAliasBufferInfo(memref::SubViewOp op) { newInfo->allocateSize = segmentSize; newInfo->addressListKind = AddressListKind::Segments; newInfo->hasAppliedSubviewOffset = true; + newInfo->hasInexactSubviewRange = false; resultMemInfoVec.emplace_back(std::move(newInfo)); } } @@ -1234,6 +1300,12 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { UpdateConservativeSubviewAliasBufferInfo(result, source); return; } + if (llvm::any_of(buffer2MemInfoMap_[source], [](const auto &parentInfo) { + return !parentInfo || parentInfo->hasInexactSubviewRange; + })) { + UpdateConservativeSubviewAliasBufferInfo(result, source); + return; + } unsigned elemBytes = pto::getPTOStorageElemByteSize(sourceType.getElementType()); @@ -1280,6 +1352,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { if (overflowed) { markAddressRangeUnknown(*newInfo); newInfo->hasAppliedSubviewOffset = true; + newInfo->hasInexactSubviewRange = true; resultMemInfoVec.emplace_back(std::move(newInfo)); continue; } @@ -1287,6 +1360,7 @@ void PTOIRTranslator::UpdateTileSubViewAliasBufferInfo(pto::SubViewOp op) { newInfo->allocateSize = segmentSize; newInfo->addressListKind = AddressListKind::Segments; newInfo->hasAppliedSubviewOffset = true; + newInfo->hasInexactSubviewRange = false; resultMemInfoVec.emplace_back(std::move(newInfo)); } } diff --git a/test/lit/pto/alloc_tile_physical_overlap_sync.pto b/test/lit/pto/alloc_tile_physical_overlap_sync.pto index bd7ef86ffa..3e59bb20c0 100644 --- a/test/lit/pto/alloc_tile_physical_overlap_sync.pto +++ b/test/lit/pto/alloc_tile_physical_overlap_sync.pto @@ -335,6 +335,99 @@ module attributes {pto.target_arch = "a2a3"} { return } + // An unsupported dynamic subview is still contained by its known parent + // range. Retaining that range avoids inventing an alias with the disjoint + // target at 8192 while remaining conservative about the exact subview. + func.func @dynamic_subview_parent_range_disjoint( + %index_src: memref<1x8xf32, #pto.address_space>, + %offset: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + %subview = memref.subview %root[0, %offset] [1, 8] [1, 1] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + to memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c8192_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%subview, %one + : memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } + + // A static child of an inexact dynamic subview must retain the conservative + // parent envelope instead of narrowing from the root base. At runtime + // %offset = 8 places %child at the target address 4128. + func.func @chained_dynamic_subview_remains_conservative( + %index_src: memref<1x8xf32, #pto.address_space>, + %offset: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c4128_i64 = arith.constant 4128 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + %dynamic = memref.subview %root[0, %offset] [1, 8] [1, 1] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + to memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space> + %child = memref.subview %dynamic[0, 0] [1, 8] [1, 1] + : memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space> + to memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x8xf32, #pto.address_space> + %target = pto.pointer_cast(%c4128_i64) + : memref<1x8xf32, #pto.address_space> + + pto.tmuls ins(%child, %one + : memref<1x8xf32, strided<[16, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x8xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x8xf32, #pto.address_space>) + outs(%target : memref<1x8xf32, #pto.address_space>) + return + } + + // The logical byte count of a padded/strided root is not a physical + // envelope. Row 1 starts at 4224, outside the recorded 128 logical bytes, so + // an unsupported dynamic subview of this root must remain unknown. + func.func @strided_dynamic_subview_unknown_sync( + %index_src: memref<1x16xf32, #pto.address_space>, + %row: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c4224_i64 = arith.constant 4224 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space> + %subview = memref.subview %root[%row, 0] [1, 16] [1, 1] + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space> + to memref<1x16xf32, strided<[32, 1], offset: ?>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<1x16xf32, #pto.address_space> + %target = pto.pointer_cast(%c4224_i64) + : memref<1x16xf32, #pto.address_space> + + pto.tmuls ins(%subview, %one + : memref<1x16xf32, strided<[32, 1], offset: ?>, #pto.address_space>, f32) + outs(%vector_out : memref<1x16xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x16xf32, #pto.address_space>) + outs(%target : memref<1x16xf32, #pto.address_space>) + return + } + // The dynamic access is a real two-slot buffer, while %segments is two // discontiguous pieces of fixed slot 0. Across the loop back-edge, the // dynamic access needs per-slot event IDs, but the segmented access is one @@ -384,6 +477,48 @@ module attributes {pto.target_arch = "a2a3"} { } return } + + // The V operation has both a lane-compatible dynamic-slot dependency and an + // incompatible segmented dependency with the following iteration's TLOAD. + // A dynamic event keyed only by %idx would not cover both hazards, so the + // combined back-edge must fall back to one static event. + func.func @mixed_segmented_backedge_falls_back_to_single_event( + %index_src: memref<2x16xf32, #pto.address_space>, + %n: index) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096_i64 = arith.constant 4096 : i64 + %c8192_i64 = arith.constant 8192 : i64 + %c16384_i64 = arith.constant 16384 : i64 + + %slots = pto.pointer_cast(%c4096_i64, %c8192_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + %wide_slot0 = pto.pointer_cast(%c4096_i64) + : memref<2x32xf32, strided<[32, 1]>, #pto.address_space> + %segments = memref.subview %wide_slot0[0, 0] [2, 16] [1, 1] + : memref<2x32xf32, strided<[32, 1]>, #pto.address_space> + to memref<2x16xf32, strided<[32, 1]>, #pto.address_space> + %out = pto.pointer_cast(%c16384_i64) + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %idx = arith.remui %i, %c2 : index + %dynamic = pto.slot_marker %slots[%idx] + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + -> memref<2x16xf32, strided<[16, 1]>, #pto.address_space> + pto.tload ins(%index_src : memref<2x16xf32, #pto.address_space>) + outs(%dynamic + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space>) + pto.tadd ins(%segments, %dynamic + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space>, + memref<2x16xf32, strided<[16, 1]>, #pto.address_space>) + outs(%out + : memref<2x16xf32, strided<[16, 1]>, #pto.address_space>) + } + return + } } // CHECK-LABEL: AICORE void alloc_tile_physical_overlap_sync( @@ -444,6 +579,23 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SUBVIEW_REINTERPRET_V_TO_MTE2]]); // CHECK: TLOAD( +// CHECK-LABEL: AICORE void dynamic_subview_parent_range_disjoint( +// CHECK: TMULS( +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2 +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void chained_dynamic_subview_remains_conservative( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_SUBVIEW_EVENT:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_SUBVIEW_EVENT]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void strided_dynamic_subview_unknown_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_SUBVIEW_EVENT:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_SUBVIEW_EVENT]]); +// CHECK: TLOAD( + // CHECK-LABEL: AICORE void segmented_backedge_uses_single_event( // Two events belong to the real dynamic two-slot dependency. The third is the // independent single-event dependency for the segmented view. @@ -461,3 +613,13 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID0); // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID1); // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[SEGMENT_EVENT]]); + +// CHECK-LABEL: AICORE void mixed_segmented_backedge_falls_back_to_single_event( +// CHECK: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT:[0-9]+]]); +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2, +// CHECK: for ( +// CHECK: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT]]); +// CHECK: TLOAD( +// CHECK: TADD( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT]]); +// CHECK: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT]]); From 99c64ec53a5fc73cc12c3ceed7c7a1b7075d627e Mon Sep 17 00:00:00 2001 From: Toni Boehnlein Date: Mon, 20 Jul 2026 20:11:23 +0200 Subject: [PATCH 5/5] fix(insert-sync): harden physical range and slot mapping --- .../InsertSync/InsertSyncAnalysis.cpp | 164 +++++++++------- .../InsertSync/MemoryDependentAnalyzer.cpp | 11 ++ .../Transforms/InsertSync/PTOIRTranslator.cpp | 182 ++++++++++++------ .../pto/alloc_tile_physical_overlap_sync.pto | 112 ++++++++++- .../pto/multi_slot_physical_lane_mapping.pto | 179 +++++++++++++++++ 5 files changed, 521 insertions(+), 127 deletions(-) create mode 100644 test/lit/pto/multi_slot_physical_lane_mapping.pto diff --git a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp index 8590af626f..5e3d019264 100644 --- a/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp +++ b/lib/PTO/Transforms/InsertSync/InsertSyncAnalysis.cpp @@ -445,20 +445,54 @@ void InsertSyncAnalysis::InsertSync( // back-edge path still needs to sync per-slot via dyn event id (the // prefetch idiom). When the analysis is inconclusive (kUnknown / kEqual) // the dep is kept and the existing conservative path runs. +static bool rangesOverlapConservatively(uint64_t aStart, uint64_t aSize, + uint64_t bStart, uint64_t bSize) { + if (aSize == 0 || bSize == 0 || + aSize > std::numeric_limits::max() - aStart || + bSize > std::numeric_limits::max() - bStart) + return true; + return std::max(aStart, bStart) < std::min(aStart + aSize, bStart + bSize); +} + +// Slot-indexed synchronization is safe only when slot i on one side can +// physically alias slot i on the other side, and every off-diagonal pair is +// disjoint. Equal vector lengths alone do not establish that mapping for +// shifted, reordered, or partially overlapping physical allocations. +static bool hasIdentityPhysicalSlotMapping(const BaseMemInfo *a, + const BaseMemInfo *b) { + if (!a || !b || a->addressListKind != AddressListKind::AlternativeSlots || + b->addressListKind != AddressListKind::AlternativeSlots || + a->baseAddresses.size() < 2 || + a->baseAddresses.size() != b->baseAddresses.size() || + a->addressProvenance == AddressProvenance::UnknownAbsolute || + b->addressProvenance == AddressProvenance::UnknownAbsolute || + a->addressProvenance != b->addressProvenance) + return false; + + if (a->addressProvenance == AddressProvenance::RootRelative && + a->rootBuffer != b->rootBuffer) + return false; + + for (size_t i = 0; i < a->baseAddresses.size(); ++i) { + for (size_t j = 0; j < b->baseAddresses.size(); ++j) { + bool overlaps = + rangesOverlapConservatively(a->baseAddresses[i], a->allocateSize, + b->baseAddresses[j], b->allocateSize); + if ((i == j) != overlaps) + return false; + } + } + return true; +} + static bool isForwardDepDroppableBySlotAffine(const BaseMemInfo *a, const BaseMemInfo *b) { - if (!a || !b) - return false; - if (a->addressListKind != AddressListKind::AlternativeSlots || - b->addressListKind != AddressListKind::AlternativeSlots) + if (!hasIdentityPhysicalSlotMapping(a, b)) return false; Value slotA = findSlotMarkerExpr(a->baseBuffer); Value slotB = findSlotMarkerExpr(b->baseBuffer); if (!slotA || !slotB) return false; - if (a->baseAddresses.size() < 2 || - a->baseAddresses.size() != b->baseAddresses.size()) - return false; return compareSlotSSA(slotA, slotB, static_cast(a->baseAddresses.size())) == SlotRelation::kDisjoint; @@ -466,20 +500,60 @@ static bool isForwardDepDroppableBySlotAffine(const BaseMemInfo *a, static std::optional getAlternativeSlotPairCount(const BaseMemInfo *a, const BaseMemInfo *b) { - if (!a || !b) - return std::nullopt; - if (a->addressListKind != AddressListKind::AlternativeSlots || - b->addressListKind != AddressListKind::AlternativeSlots) + if (!hasIdentityPhysicalSlotMapping(a, b)) return std::nullopt; if (!findSlotMarkerExpr(a->baseBuffer) || !findSlotMarkerExpr(b->baseBuffer)) return std::nullopt; - if (a->baseAddresses.size() < 2 || - a->baseAddresses.size() != b->baseAddresses.size()) - return std::nullopt; return a->baseAddresses.size(); } +struct DynamicEventSlotMapping { + int eventIdNum; + Value consumerSlot; + Value producerSlot; +}; + +static bool isLocalMemInfo(const BaseMemInfo *info) { + return info && (info->scope == pto::AddressSpace::MAT || + info->scope == pto::AddressSpace::VEC); +} + +// A single set/wait pair can carry only one producer and one consumer slot +// expression. Require every local dependency pair to agree on both expressions +// modulo N as well as on the physical lane mapping. +static std::optional +getDynamicEventSlotMapping(const DepBaseMemInfoPairVec &depPairs) { + std::optional mapping; + for (const auto &pair : depPairs) { + if (!isLocalMemInfo(pair.first) && !isLocalMemInfo(pair.second)) + continue; + + auto pairCount = getAlternativeSlotPairCount(pair.first, pair.second); + if (!pairCount) + return std::nullopt; + int pairN = static_cast(*pairCount); + Value consumerSlot = findSlotMarkerExpr(pair.first->baseBuffer); + Value producerSlot = findSlotMarkerExpr(pair.second->baseBuffer); + if (!consumerSlot || !producerSlot) + return std::nullopt; + + if (!mapping) { + mapping = DynamicEventSlotMapping{pairN, consumerSlot, producerSlot}; + continue; + } + if (mapping->eventIdNum != pairN) + return std::nullopt; + uint32_t n = static_cast(pairN); + if (compareSlotSSA(mapping->consumerSlot, consumerSlot, n) != + SlotRelation::kEqual || + compareSlotSSA(mapping->producerSlot, producerSlot, n) != + SlotRelation::kEqual) + return std::nullopt; + } + return mapping; +} + void InsertSyncAnalysis::MemAnalyze( CompoundInstanceElement *nowCompound, CompoundInstanceElement *frontCompound, SyncRecordList &syncRecordList, @@ -638,35 +712,13 @@ void InsertSyncAnalysis::InsertSyncOperation( // dyn event IDs are warranted, also plumb the per-side slot SSA so // codegen can lower into `pto.set_flag_dyn` / `pto.wait_flag_dyn`. if (forEndIndex.has_value()) { - int eventIdNum = GetEventIdNum(depBaseMemInfosVec); - if (eventIdNum > 1) { - // Each dep pair has (now=consumer, front=producer). The producer's - // slot SSA gates the `set_flag_dyn`; the consumer's gates the - // `wait_flag_dyn`. Walk the first viable dep pair to extract them. - Value producerSlot; - Value consumerSlot; - for (auto &pair : depBaseMemInfosVec) { - if (!getAlternativeSlotPairCount(pair.first, pair.second)) - continue; - if (pair.second && pair.second->baseBuffer) - producerSlot = findSlotMarkerExpr(pair.second->baseBuffer); - if (pair.first && pair.first->baseBuffer) - consumerSlot = findSlotMarkerExpr(pair.first->baseBuffer); - if (producerSlot && consumerSlot) - break; - } - if (!producerSlot || !consumerSlot) { - // No slot SSA threaded through -- fall back to single event id. - // This keeps non-multi-buffer codepaths untouched even if their - // baseAddresses happen to have multiple entries for some other - // reason (e.g. memref subview). - eventIdNum = 1; - } else { - setOp->slotSSAExpr = producerSlot; - setOp->slotCount = static_cast(eventIdNum); - waitOp->slotSSAExpr = consumerSlot; - waitOp->slotCount = static_cast(eventIdNum); - } + int eventIdNum = 1; + if (auto mapping = getDynamicEventSlotMapping(depBaseMemInfosVec)) { + eventIdNum = mapping->eventIdNum; + setOp->slotSSAExpr = mapping->producerSlot; + setOp->slotCount = static_cast(eventIdNum); + waitOp->slotSSAExpr = mapping->consumerSlot; + waitOp->slotCount = static_cast(eventIdNum); } setOp->eventIdNum = eventIdNum; waitOp->eventIdNum = eventIdNum; @@ -834,30 +886,8 @@ int InsertSyncAnalysis::GetEventIdNum( // in every local dependency pair are compatible alternative-slot views with // recoverable slot SSA expressions. A segmented or otherwise incompatible // pair forces one static event so the synchronization covers every hazard. - std::optional eventIdNum; - for (const auto &pair : depBaseMemInfosVec) { - bool isLocalA = - pair.first && (pair.first->scope == pto::AddressSpace::MAT || - pair.first->scope == pto::AddressSpace::VEC); - bool isLocalB = - pair.second && (pair.second->scope == pto::AddressSpace::MAT || - pair.second->scope == pto::AddressSpace::VEC); - if (!isLocalA && !isLocalB) - continue; - auto pairCount = getAlternativeSlotPairCount(pair.first, pair.second); - if (!pairCount) - return 1; - int pairN = static_cast(*pairCount); - if (!eventIdNum) { - eventIdNum = pairN; - } else if (*eventIdNum != pairN) { - // Multiple dep pairs disagreeing on N: fall back to single event id - // for safety. With more work this could be relaxed by per-pair - // multi-buffer reasoning. - return 1; - } - } - return eventIdNum.value_or(1); + auto mapping = getDynamicEventSlotMapping(depBaseMemInfosVec); + return mapping ? mapping->eventIdNum : 1; } bool InsertSyncAnalysis::IsGMHazard( diff --git a/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp b/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp index c700422539..9f23c10d86 100644 --- a/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp +++ b/lib/PTO/Transforms/InsertSync/MemoryDependentAnalyzer.cpp @@ -181,6 +181,17 @@ bool MemoryDependentAnalyzer::MemAlias(const BaseMemInfo *a, return isBufferAddressRangeOverlap(a, b); } + // A known physical address and a root-relative offset are not directly + // comparable. This mixed state is not expected after level-3 planning, but + // treating distinct roots as disjoint would be an unsafe default if a future + // pipeline combines planned and symbolic local buffers. + if (a->addressProvenance != b->addressProvenance) { + if (isTraceEnabled()) + llvm::errs() << " -> Mixed local address provenance. " + "Conservatively aliasing.\n"; + return true; + } + if (a->rootBuffer == b->rootBuffer) { if (a->baseAddresses.empty() || b->baseAddresses.empty()) return true; if (a->allocateSize == 0 || b->allocateSize == 0) return true; diff --git a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp index 796307fa5c..61235d651b 100644 --- a/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp +++ b/lib/PTO/Transforms/InsertSync/PTOIRTranslator.cpp @@ -99,6 +99,33 @@ static std::optional addByteOffset(uint64_t base, uint64_t offset) { return base + offset; } +static std::optional checkedMultiply(uint64_t lhs, uint64_t rhs) { + if (lhs != 0 && rhs > std::numeric_limits::max() / lhs) + return std::nullopt; + return lhs * rhs; +} + +static std::optional +getStaticLogicalMemRefSizeInBytes(MemRefType type) { + if (!type.hasStaticShape()) + return std::nullopt; + + uint64_t elements = 1; + for (int64_t dim : type.getShape()) { + if (dim <= 0) + return std::nullopt; + auto next = checkedMultiply(elements, static_cast(dim)); + if (!next) + return std::nullopt; + elements = *next; + } + + uint64_t elemBytes = pto::getPTOStorageElemByteSize(type.getElementType()); + if (elemBytes == 0) + return std::nullopt; + return checkedMultiply(elements, elemBytes); +} + static std::optional addByteOffset(uint64_t base, int64_t offset) { if (offset >= 0) return addByteOffset(base, static_cast(offset)); @@ -109,41 +136,63 @@ static std::optional addByteOffset(uint64_t base, int64_t offset) { return base - magnitude; } -static std::optional -getContiguousMemRefSizeInBytes(MemRefType type) { - if (!type.hasStaticShape() || type.getRank() != kTileRank2D) +struct StaticMemRefPhysicalRange { + uint64_t startOffsetInBytes; + uint64_t sizeInBytes; +}; + +// Return the conservative physical envelope touched by a statically shaped +// memref. The envelope includes padding between strided elements; using the +// logical element count here can classify a real overlap as disjoint. +// Dynamic or negative strides cannot be represented safely by one unsigned +// half-open range and therefore fall back to unknown-address handling. +static std::optional +getStaticMemRefPhysicalRange(MemRefType type) { + if (!type.hasStaticShape()) return std::nullopt; SmallVector strides; int64_t offset = ShapedType::kDynamic; if (failed(mlir::pto::getPTOMemRefStridesAndOffset(type, strides, offset)) || - strides.size() != kTileRank2D || + strides.size() != static_cast(type.getRank()) || llvm::is_contained(strides, ShapedType::kDynamic)) return std::nullopt; - if (offset != 0) - return std::nullopt; - - ArrayRef shape = type.getShape(); - if (shape[0] <= 0 || shape[1] <= 0 || strides[0] < 0 || strides[1] < 0) - return std::nullopt; - bool rowMajorContiguous = - strides[1] == 1 && (shape[0] == 1 || strides[0] == shape[1]); - bool colMajorContiguous = - strides[0] == 1 && (shape[1] == 1 || strides[1] == shape[0]); - if (!rowMajorContiguous && !colMajorContiguous) - return std::nullopt; uint64_t elemBytes = pto::getPTOStorageElemByteSize(type.getElementType()); if (elemBytes == 0) return std::nullopt; - uint64_t rows = static_cast(shape[0]); - uint64_t cols = static_cast(shape[1]); - if (cols > std::numeric_limits::max() / rows) + + uint64_t maxElementOffset = 0; + for (auto [dim, stride] : llvm::zip(type.getShape(), strides)) { + if (dim <= 0 || stride < 0) + return std::nullopt; + auto extent = checkedMultiply(static_cast(dim - 1), + static_cast(stride)); + if (!extent || + *extent > std::numeric_limits::max() - maxElementOffset) + return std::nullopt; + maxElementOffset += *extent; + } + + if (maxElementOffset == std::numeric_limits::max()) return std::nullopt; - uint64_t elements = rows * cols; - if (elemBytes > std::numeric_limits::max() / elements) + auto sizeInBytes = checkedMultiply(maxElementOffset + 1, elemBytes); + if (!sizeInBytes) return std::nullopt; - return elements * elemBytes; + + // A dynamic layout offset on pointer_cast denotes the runtime descriptor's + // index-zero pointer, so the explicit address remains the analysis base. + // A static offset is representable and can be folded into that base. + uint64_t elementOffset = 0; + if (offset != ShapedType::kDynamic) { + if (offset < 0) + return std::nullopt; + elementOffset = static_cast(offset); + } + auto startOffsetInBytes = checkedMultiply(elementOffset, elemBytes); + if (!startOffsetInBytes) + return std::nullopt; + return StaticMemRefPhysicalRange{*startOffsetInBytes, *sizeInBytes}; } static bool canRetainSubviewParentRange(Value source, @@ -160,8 +209,8 @@ static bool canRetainSubviewParentRange(Value source, auto sourceType = dyn_cast(source.getType()); if (!sourceType) return false; - auto contiguousSize = getContiguousMemRefSizeInBytes(sourceType); - return contiguousSize && parentInfo.allocateSize >= *contiguousSize; + auto physicalRange = getStaticMemRefPhysicalRange(sourceType); + return physicalRange && parentInfo.allocateSize >= physicalRange->sizeInBytes; } static void markAddressRangeUnknown(BaseMemInfo &info) { @@ -171,6 +220,9 @@ static void markAddressRangeUnknown(BaseMemInfo &info) { info.hasAppliedReinterpret = false; info.hasAppliedSubviewOffset = false; info.hasInexactSubviewRange = false; + // Root-relative addresses stay root-relative: losing their precise offsets + // still aliases conservatively within that root, while distinct allocation + // roots remain independent. if (info.addressProvenance == AddressProvenance::KnownAbsolute) info.addressProvenance = AddressProvenance::UnknownAbsolute; } @@ -617,17 +669,20 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { return op.emitError("PointerCast must have at least one address operand"); } - uint64_t sizeInBytes = 0; - if (memRefType.hasStaticShape()) { - int64_t elemSize = static_cast( - pto::getPTOStorageElemByteSize(memRefType.getElementType())); - if (elemSize == 0) - return failure(); - int64_t numElements = 1; - for (auto dim : memRefType.getShape()) - numElements *= dim; - sizeInBytes = numElements * elemSize; + std::optional physicalRange; + if (op.getConfig()) { + // PTO tile memrefs use strides to encode blocked/fractal access layout + // (for example [16, 32] for a 32x32 tile), not a conventional injective + // strided memref envelope. TileBufConfig defines that storage, whose + // existing allocation contract is the logical tile byte size. + if (auto logicalSize = getStaticLogicalMemRefSizeInBytes(memRefType)) + physicalRange = StaticMemRefPhysicalRange{0, *logicalSize}; + } else { + physicalRange = getStaticMemRefPhysicalRange(memRefType); } + uint64_t sizeInBytes = physicalRange ? physicalRange->sizeInBytes : 0; + uint64_t baseAdjustment = + physicalRange ? physicalRange->startOffsetInBytes : 0; pto::AddressSpace space = pto::AddressSpace::GM; if (auto attr = memRefType.getMemorySpace()) { @@ -643,14 +698,22 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { // dynamic address as unknown-absolute so alias analysis stays conservative. // Downstream view ops accumulate their delta into baseAddresses[0]. Value rootSrc = op.getAddrs().front(); - SmallVector baseAddresses{0}; + SmallVector baseAddresses; AddressProvenance addressProvenance = AddressProvenance::RootRelative; if (isLocalAddressSpace(space)) { addressProvenance = AddressProvenance::UnknownAbsolute; - if (std::optional address = getKnownPhysicalAddress(rootSrc)) { - baseAddresses[0] = *address; - addressProvenance = AddressProvenance::KnownAbsolute; + if (physicalRange) { + if (std::optional address = + getKnownPhysicalAddress(rootSrc)) { + if (std::optional adjusted = + addByteOffset(*address, baseAdjustment)) { + baseAddresses.push_back(*adjusted); + addressProvenance = AddressProvenance::KnownAbsolute; + } + } } + } else { + baseAddresses.push_back(baseAdjustment); } auto newMemInfo = std::make_unique( res, rootSrc, space, std::move(baseAddresses), sizeInBytes, @@ -666,6 +729,14 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { // AllocToPointerCast). `pto.slot_marker` then narrows or keeps these // offsets according to its slot SSA; MemAlias's existing // `isBufferAddressRangeOverlap` does the per-slot disambiguation. + if (isLocalAddressSpace(space) && !physicalRange) { + auto newMemInfo = std::make_unique( + res, op.getAddrs().front(), space, SmallVector{}, 0, + AddressProvenance::UnknownAbsolute); + buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); + return success(); + } + SmallVector slotOffsets; slotOffsets.reserve(op.getAddrs().size()); for (Value a : op.getAddrs()) { @@ -683,7 +754,16 @@ PTOIRTranslator::UpdatePointerCastOpMemInfo(pto::PointerCastOp op) { buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); return success(); } - slotOffsets.push_back(*address); + std::optional adjusted = addByteOffset(*address, baseAdjustment); + if (!adjusted) { + auto newMemInfo = std::make_unique( + res, op.getAddrs().front(), space, SmallVector{}, 0, + isLocalAddressSpace(space) ? AddressProvenance::UnknownAbsolute + : AddressProvenance::RootRelative); + buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); + return success(); + } + slotOffsets.push_back(*adjusted); } auto newMemInfo = std::make_unique( @@ -1374,19 +1454,10 @@ LogicalResult PTOIRTranslator::UpdateMemrefAllocOpMemInfo(memref::AllocOp op) { if (!memRefType) return failure(); - // 1. 计算大小 (Bytes) - uint64_t sizeInBytes = 0; - if (memRefType.hasStaticShape()) { - int64_t elemSize = static_cast( - pto::getPTOStorageElemByteSize(memRefType.getElementType())); - if (elemSize == 0) - return failure(); - - int64_t numElements = 1; - for (auto dim : memRefType.getShape()) - numElements *= dim; - sizeInBytes = numElements * elemSize; - } + // 1. 计算保守的物理范围 (Bytes),包括静态 stride 产生的 padding。 + auto physicalRange = getStaticMemRefPhysicalRange(memRefType); + uint64_t sizeInBytes = physicalRange ? physicalRange->sizeInBytes : 0; + uint64_t baseOffset = physicalRange ? physicalRange->startOffsetInBytes : 0; // 2. 解析地址空间 (Scope) // 默认视为 MAT/UB (Local Memory),这是 alloc 的常见用途 @@ -1402,9 +1473,10 @@ LogicalResult PTOIRTranslator::UpdateMemrefAllocOpMemInfo(memref::AllocOp op) { // 3. 注册 Buffer 信息 // 对于 alloc,它自己就是 Root auto newMemInfo = std::make_unique( - res, // baseBuffer - res, // rootBuffer (Self is root) - space, SmallVector{0}, // Base Addresses (Offset 0) + res, // baseBuffer + res, // rootBuffer (Self is root) + space, + SmallVector{baseOffset}, // Base address of the physical span. sizeInBytes); buffer2MemInfoMap_[res].emplace_back(newMemInfo->clone()); diff --git a/test/lit/pto/alloc_tile_physical_overlap_sync.pto b/test/lit/pto/alloc_tile_physical_overlap_sync.pto index 3e59bb20c0..4dd06e0cc6 100644 --- a/test/lit/pto/alloc_tile_physical_overlap_sync.pto +++ b/test/lit/pto/alloc_tile_physical_overlap_sync.pto @@ -398,10 +398,10 @@ module attributes {pto.target_arch = "a2a3"} { return } - // The logical byte count of a padded/strided root is not a physical - // envelope. Row 1 starts at 4224, outside the recorded 128 logical bytes, so - // an unsupported dynamic subview of this root must remain unknown. - func.func @strided_dynamic_subview_unknown_sync( + // A padded/strided parent records its full physical envelope. Even though + // the dynamic row cannot be narrowed exactly, retaining [4096, 4288) still + // proves that it may overlap the target beginning at 4224. + func.func @strided_dynamic_subview_parent_envelope_sync( %index_src: memref<1x16xf32, #pto.address_space>, %row: index) attributes {pto.kernel_kind = #pto.kernel_kind} { @@ -428,6 +428,58 @@ module attributes {pto.target_arch = "a2a3"} { return } + // The logical element count is 128 bytes, but the second row starts 128 + // bytes after the first and extends the physical envelope to [4096, 4288). + // A direct use of the root therefore overlaps the separate range beginning + // at 4224 and requires a cross-pipeline WAR synchronization. + func.func @strided_root_physical_envelope_sync( + %index_src: memref<1x16xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c4224_i64 = arith.constant 4224 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<2x16xf32, #pto.address_space> + %target = pto.pointer_cast(%c4224_i64) + : memref<1x16xf32, #pto.address_space> + + pto.tmuls ins(%root, %one + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space>, f32) + outs(%vector_out : memref<2x16xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x16xf32, #pto.address_space>) + outs(%target : memref<1x16xf32, #pto.address_space>) + return + } + + // The same physical envelope ends exactly at 4288. Half-open ranges that + // only touch at that boundary remain disjoint. + func.func @strided_root_physical_boundary_no_sync( + %index_src: memref<1x16xf32, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c4288_i64 = arith.constant 4288 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.000000e+00 : f32 + + %root = pto.pointer_cast(%c4096_i64) + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space> + %vector_out = pto.pointer_cast(%c16384_i64) + : memref<2x16xf32, #pto.address_space> + %target = pto.pointer_cast(%c4288_i64) + : memref<1x16xf32, #pto.address_space> + + pto.tmuls ins(%root, %one + : memref<2x16xf32, strided<[32, 1]>, #pto.address_space>, f32) + outs(%vector_out : memref<2x16xf32, #pto.address_space>) + pto.tload ins(%index_src : memref<1x16xf32, #pto.address_space>) + outs(%target : memref<1x16xf32, #pto.address_space>) + return + } + // The dynamic access is a real two-slot buffer, while %segments is two // discontiguous pieces of fixed slot 0. Across the loop back-edge, the // dynamic access needs per-slot event IDs, but the segmented access is one @@ -519,6 +571,40 @@ module attributes {pto.target_arch = "a2a3"} { } return } + + // TileBufConfig memrefs use strides to describe blocked/fractal hardware + // access, not the allocation's conventional physical envelope. The 32x32 + // f16 tile occupies [4096,6144), so the target starting at 6144 is disjoint + // even though the synthetic [16,32] stride envelope would extend farther. + func.func @tile_config_blocked_stride_boundary_no_sync( + %tile_src: memref<32x32xf16, #pto.address_space>, + %target_src: memref<1x16xf16, #pto.address_space>) + attributes {pto.kernel_kind = #pto.kernel_kind} { + %c4096_i64 = arith.constant 4096 : i64 + %c6144_i64 = arith.constant 6144 : i64 + %c16384_i64 = arith.constant 16384 : i64 + %one = arith.constant 1.0 : f16 + + %tile = pto.pointer_cast(%c4096_i64) + {config = #pto.tile_buf_config} + : memref<32x32xf16, strided<[16, 32]>, #pto.address_space> + %out = pto.pointer_cast(%c16384_i64) + : memref<32x32xf16, strided<[16, 32]>, #pto.address_space> + %target = pto.pointer_cast(%c6144_i64) + : memref<1x16xf16, #pto.address_space> + + pto.tload ins(%tile_src : memref<32x32xf16, #pto.address_space>) + outs(%tile + : memref<32x32xf16, strided<[16, 32]>, #pto.address_space>) + pto.tmuls ins(%tile, %one + : memref<32x32xf16, strided<[16, 32]>, #pto.address_space>, f16) + outs(%out + : memref<32x32xf16, strided<[16, 32]>, #pto.address_space>) + pto.tload ins(%target_src : memref<1x16xf16, #pto.address_space>) + outs(%target : memref<1x16xf16, #pto.address_space>) + return + } } // CHECK-LABEL: AICORE void alloc_tile_physical_overlap_sync( @@ -590,12 +676,23 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[CHAINED_SUBVIEW_EVENT]]); // CHECK: TLOAD( -// CHECK-LABEL: AICORE void strided_dynamic_subview_unknown_sync( +// CHECK-LABEL: AICORE void strided_dynamic_subview_parent_envelope_sync( // CHECK: TMULS( // CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_SUBVIEW_EVENT:[0-9]+]]); // CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_SUBVIEW_EVENT]]); // CHECK: TLOAD( +// CHECK-LABEL: AICORE void strided_root_physical_envelope_sync( +// CHECK: TMULS( +// CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_ROOT_EVENT:[0-9]+]]); +// CHECK-NEXT: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[STRIDED_ROOT_EVENT]]); +// CHECK: TLOAD( + +// CHECK-LABEL: AICORE void strided_root_physical_boundary_no_sync( +// CHECK: TMULS( +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2 +// CHECK: TLOAD( + // CHECK-LABEL: AICORE void segmented_backedge_uses_single_event( // Two events belong to the real dynamic two-slot dependency. The third is the // independent single-event dependency for the segmented view. @@ -623,3 +720,8 @@ module attributes {pto.target_arch = "a2a3"} { // CHECK: TADD( // CHECK-NEXT: set_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT]]); // CHECK: wait_flag(PIPE_V, PIPE_MTE2, EVENT_ID[[MIXED_EVENT]]); + +// CHECK-LABEL: AICORE void tile_config_blocked_stride_boundary_no_sync( +// CHECK: TMULS( +// CHECK-NOT: set_flag(PIPE_V, PIPE_MTE2 +// CHECK: TLOAD( diff --git a/test/lit/pto/multi_slot_physical_lane_mapping.pto b/test/lit/pto/multi_slot_physical_lane_mapping.pto new file mode 100644 index 0000000000..caa9a6d974 --- /dev/null +++ b/test/lit/pto/multi_slot_physical_lane_mapping.pto @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +// RUN: ptoas --pto-arch=a3 --pto-level=level3 --enable-insert-sync --mlir-print-ir-after=pto-insert-sync %s 2>&1 1>/dev/null | FileCheck %s --check-prefix=SHIFTED +// RUN: ptoas --pto-arch=a3 --pto-level=level3 --enable-insert-sync --mlir-print-ir-after=pto-insert-sync %s 2>&1 1>/dev/null | FileCheck %s --check-prefix=FORWARD +// RUN: ptoas --pto-arch=a3 --pto-level=level3 --enable-insert-sync --mlir-print-ir-after=pto-insert-sync %s 2>&1 1>/dev/null | FileCheck %s --check-prefix=REORDERED +// RUN: ptoas --pto-arch=a3 --pto-level=level3 --enable-insert-sync --mlir-print-ir-after=pto-insert-sync %s 2>&1 1>/dev/null | FileCheck %s --check-prefix=OVERLAP + +// Dynamic event IDs and slot-affine dependency pruning require slot index i on +// both sides to denote the same physical range. Shifted, reordered, or +// overlapping address vectors must use conservative static synchronization. + +module { + func.func @shifted_slots_use_static_backedge( + %gm: memref<16x16xf16, #pto.address_space>, + %dst: memref<16x16xf16, #pto.address_space>, %n: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096 = arith.constant 4096 : i64 + %c8192 = arith.constant 8192 : i64 + %c12288 = arith.constant 12288 : i64 + + %producer_slots = pto.pointer_cast(%c4096, %c8192) + : memref<16x16xf16, #pto.address_space> + %consumer_slots = pto.pointer_cast(%c8192, %c12288) + : memref<16x16xf16, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %idx = arith.remui %i, %c2 : index + %producer = pto.slot_marker %producer_slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + %consumer = pto.slot_marker %consumer_slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + pto.tload ins(%gm : memref<16x16xf16, #pto.address_space>) + outs(%producer : memref<16x16xf16, #pto.address_space>) + pto.tstore ins(%consumer : memref<16x16xf16, #pto.address_space>) + outs(%dst : memref<16x16xf16, #pto.address_space>) + } + return + } + + func.func @shifted_slots_keep_forward_dependency( + %gm: memref<16x16xf16, #pto.address_space>, + %dst: memref<16x16xf16, #pto.address_space>, %n: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096 = arith.constant 4096 : i64 + %c8192 = arith.constant 8192 : i64 + %c12288 = arith.constant 12288 : i64 + + %producer_slots = pto.pointer_cast(%c4096, %c8192) + : memref<16x16xf16, #pto.address_space> + %consumer_slots = pto.pointer_cast(%c8192, %c12288) + : memref<16x16xf16, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %next = arith.addi %i, %c1 : index + %producer_idx = arith.remui %next, %c2 : index + %consumer_idx = arith.remui %i, %c2 : index + %producer = pto.slot_marker %producer_slots[%producer_idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + %consumer = pto.slot_marker %consumer_slots[%consumer_idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + pto.tload ins(%gm : memref<16x16xf16, #pto.address_space>) + outs(%producer : memref<16x16xf16, #pto.address_space>) + pto.tstore ins(%consumer : memref<16x16xf16, #pto.address_space>) + outs(%dst : memref<16x16xf16, #pto.address_space>) + } + return + } + + func.func @reordered_slots_use_static_backedge( + %gm: memref<16x16xf16, #pto.address_space>, + %dst: memref<16x16xf16, #pto.address_space>, %n: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096 = arith.constant 4096 : i64 + %c8192 = arith.constant 8192 : i64 + + %producer_slots = pto.pointer_cast(%c4096, %c8192) + : memref<16x16xf16, #pto.address_space> + %consumer_slots = pto.pointer_cast(%c8192, %c4096) + : memref<16x16xf16, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %idx = arith.remui %i, %c2 : index + %producer = pto.slot_marker %producer_slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + %consumer = pto.slot_marker %consumer_slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + pto.tload ins(%gm : memref<16x16xf16, #pto.address_space>) + outs(%producer : memref<16x16xf16, #pto.address_space>) + pto.tstore ins(%consumer : memref<16x16xf16, #pto.address_space>) + outs(%dst : memref<16x16xf16, #pto.address_space>) + } + return + } + + func.func @overlapping_slots_use_static_backedge( + %gm: memref<16x16xf16, #pto.address_space>, + %dst: memref<16x16xf16, #pto.address_space>, %n: index) { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %c2 = arith.constant 2 : index + %c4096 = arith.constant 4096 : i64 + %c4352 = arith.constant 4352 : i64 + + // Each slot occupies 512 bytes, so [4096, 4608) and [4352, 4864) + // overlap even though they have distinct slot indices. + %slots = pto.pointer_cast(%c4096, %c4352) + : memref<16x16xf16, #pto.address_space> + + scf.for %i = %c0 to %n step %c1 { + %idx = arith.remui %i, %c2 : index + %producer = pto.slot_marker %slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + %consumer = pto.slot_marker %slots[%idx] + : memref<16x16xf16, #pto.address_space> + -> memref<16x16xf16, #pto.address_space> + pto.tload ins(%gm : memref<16x16xf16, #pto.address_space>) + outs(%producer : memref<16x16xf16, #pto.address_space>) + pto.tstore ins(%consumer : memref<16x16xf16, #pto.address_space>) + outs(%dst : memref<16x16xf16, #pto.address_space>) + } + return + } +} + +// SHIFTED-LABEL: func.func @shifted_slots_use_static_backedge +// SHIFTED-NOT: pto.set_flag_dyn +// SHIFTED-NOT: pto.wait_flag_dyn +// SHIFTED: pto.set_flag[, , <[[EVENT:EVENT_ID[0-9]+]]>] +// SHIFTED: scf.for +// SHIFTED: pto.wait_flag[, , <[[EVENT]]>] +// SHIFTED: pto.tload +// SHIFTED: pto.tstore +// SHIFTED: pto.set_flag[, , <[[EVENT]]>] + +// FORWARD-LABEL: func.func @shifted_slots_keep_forward_dependency +// FORWARD: scf.for +// FORWARD: pto.tload +// FORWARD-NEXT: pto.set_flag[, , <[[EVENT:EVENT_ID[0-9]+]]>] +// FORWARD-NEXT: pto.wait_flag[, , <[[EVENT]]>] +// FORWARD-NEXT: pto.tstore + +// REORDERED-LABEL: func.func @reordered_slots_use_static_backedge +// REORDERED-NOT: pto.set_flag_dyn +// REORDERED-NOT: pto.wait_flag_dyn +// REORDERED: pto.set_flag[, , <[[EVENT:EVENT_ID[0-9]+]]>] +// REORDERED: scf.for +// REORDERED: pto.wait_flag[, , <[[EVENT]]>] +// REORDERED: pto.tload +// REORDERED: pto.tstore +// REORDERED: pto.set_flag[, , <[[EVENT]]>] + +// OVERLAP-LABEL: func.func @overlapping_slots_use_static_backedge +// OVERLAP-NOT: pto.set_flag_dyn +// OVERLAP-NOT: pto.wait_flag_dyn +// OVERLAP: pto.set_flag[, , <[[EVENT:EVENT_ID[0-9]+]]>] +// OVERLAP: scf.for +// OVERLAP: pto.wait_flag[, , <[[EVENT]]>] +// OVERLAP: pto.tload +// OVERLAP: pto.tstore +// OVERLAP: pto.set_flag[, , <[[EVENT]]>]