Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -634,7 +634,7 @@ pending slot, promoted on completion). a5 implements this order inline in
| NORMAL (ready) | `ready_queues[MIX\|AIC\|AIV]` | *(same `ready_queues[]` — a5 has no `ready_sync_queues[]`)* |
| EARLY (speculative) | `early_dispatch_queues[MIX\|AIC\|AIV]` | `early_sync_start_queue` (single) |

A task routes to the early sync lane iff `active_mask.requires_sync_start()`. Early
A task routes to the early sync lane iff `task_attrs.requires_sync_start()`. Early
dispatch runs only once normal `ready_queues[]` are empty **and** the local
`CoreTracker` has a spare slot (`has_any_free_slot`, a2a3 #1288).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ static bool all_claimed_fanin_completed(const PTO2FaninBuilder &fanin_builder) {
static bool all_claimed_fanin_allow_early_resolve(const PTO2FaninBuilder &fanin_builder) {
if (fanin_builder.count == 0) return true;
return fanin_builder.for_each([](PTO2TaskSlotState *producer) -> bool {
return producer != nullptr && producer->allow_early_resolve;
return producer != nullptr && producer->task_attrs.allow_early_resolve();
});
}

Expand Down Expand Up @@ -390,7 +390,7 @@ void PTO2OrchestratorState::wire_fanin_task(PTO2TaskSlotState &slot_state, int32
for_each_fanin_slot_state(*payload, [&](PTO2TaskSlotState *producer) {
producer->lock_fanout();
int32_t pstate = producer->task_state.load(std::memory_order_acquire);
if (!early_disqualified && !producer->allow_early_resolve) {
if (!early_disqualified && !producer->task_attrs.allow_early_resolve()) {
early_disqualified = true;
}
if (pstate >= PTO2_TASK_COMPLETED) {
Expand Down Expand Up @@ -517,7 +517,7 @@ static void prefetch_payload(PTO2TaskPayload *payload, int32_t tensor_count, int

static bool prepare_task(
PTO2OrchestratorState *orch, const L0TaskArgs &args, int32_t total_output_size, ActiveMask active_mask,
PTO2PreparedTask *out
TaskAttrs task_attrs, PTO2PreparedTask *out
) {
uint8_t ring_id = orch->current_ring_id();
auto &allocator = orch->rings[ring_id].task_allocator;
Expand Down Expand Up @@ -576,6 +576,7 @@ static bool prepare_task(
static_cast<int16_t>(block_num * __builtin_popcount(active_mask.core_mask()));
out->slot_state->logical_block_num = block_num;
out->slot_state->active_mask = active_mask;
out->slot_state->task_attrs = task_attrs;
// fanin_count is set during Orch-side wiring
scope_tasks_push(orch, out->slot_state);

Expand Down Expand Up @@ -798,14 +799,14 @@ static bool ensure_tensormap_capacity(PTO2OrchestratorState *orch, int32_t neede
// computation (explicit_deps + auto), output registration, slot init, and
// Orch-side wiring/ready publication.
static TaskOutputTensors submit_task_common(
PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, int32_t aic_kernel_id,
int32_t aiv0_kernel_id, int32_t aiv1_kernel_id
PTO2OrchestratorState *orch, const L0TaskArgs &args, ActiveMask active_mask, TaskAttrs task_attrs,
int32_t aic_kernel_id, int32_t aiv0_kernel_id, int32_t aiv1_kernel_id
) {
CYCLE_COUNT_START();
TaskOutputTensors result;
PTO2OutputLayout layout = calculate_output_layout(args);
PTO2PreparedTask prepared;
if (!prepare_task(orch, args, layout.total_output_size, active_mask, &prepared)) {
if (!prepare_task(orch, args, layout.total_output_size, active_mask, task_attrs, &prepared)) {
return result;
}
uint8_t ring_id = prepared.task_id.ring();
Expand Down Expand Up @@ -937,7 +938,6 @@ static TaskOutputTensors submit_task_common(
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIC)] = aic_kernel_id;
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIV0)] = aiv0_kernel_id;
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIV1)] = aiv1_kernel_id;
task.task_timing_slot = args.task_timing_slot();
task.packed_buffer_base = prepared.alloc_result.packed_base;
task.packed_buffer_end = prepared.alloc_result.packed_end;

Expand Down Expand Up @@ -966,7 +966,6 @@ static TaskOutputTensors submit_task_common(
}

payload.init(args, result, prepared.alloc_result, layout);
cur_slot_state.set_allow_early_resolve(args.allow_early_resolve());

// Dispatch predicate: resolve the (tensor, indices) to an absolute GM address
// now so the scheduler can read it at the dispatch point with a single load,
Expand Down Expand Up @@ -1086,7 +1085,11 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k
active_mask = normalized.to_active_mask();
}

// Encode require_sync_start into active_mask bit 3 (only meaningful for tasks with block_num > 1)
TaskAttrs task_attrs;
task_attrs.set_early_resolve(args.allow_early_resolve());
task_attrs.set_timing_slot(args.task_timing_slot());

// sync_start is only meaningful for tasks with block_num > 1.
if (block_num > 1 && args.launch_spec.require_sync_start()) {
// Deadlock check: block_num >= total available slots of the required type.
// For MIX/AIC: limit is total_cluster_count (one AIC per cluster).
Expand All @@ -1100,15 +1103,16 @@ TaskOutputTensors PTO2OrchestratorState::submit_task(const MixedKernels &mixed_k
);
return TaskOutputTensors{};
}
active_mask.set_sync_start();
task_attrs.set_sync_start();
}

if (args.predicate().op != PredicateOp::NONE) {
active_mask.set_has_predicate();
task_attrs.set_predicate();
}

return submit_task_common(
orch, args, active_mask, normalized.aic_kernel_id, normalized.aiv0_kernel_id, normalized.aiv1_kernel_id
orch, args, active_mask, task_attrs, normalized.aic_kernel_id, normalized.aiv0_kernel_id,
normalized.aiv1_kernel_id
);
}

Expand Down Expand Up @@ -1136,7 +1140,15 @@ TaskOutputTensors PTO2OrchestratorState::submit_dummy_task(const L0TaskArgs &arg
}
always_assert(orch->scheduler != nullptr);

return submit_task_common(orch, args, ActiveMask{}, INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID);
// Dummy tasks never dispatch to an AICore, so sync_start / has_predicate do
// not apply; only the early-dispatch hint and timing tag carry over.
TaskAttrs task_attrs;
task_attrs.set_early_resolve(args.allow_early_resolve());
task_attrs.set_timing_slot(args.task_timing_slot());

return submit_task_common(
orch, args, ActiveMask{}, task_attrs, INVALID_KERNEL_ID, INVALID_KERNEL_ID, INVALID_KERNEL_ID
);
}

TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) {
Expand Down Expand Up @@ -1176,7 +1188,9 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) {

PTO2OutputLayout layout = calculate_output_layout(args);
PTO2PreparedTask prepared;
if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, &prepared)) {
// Kernel-less alloc task: no active subtasks, no dispatch-time attributes. The
// early-dispatch hint is force-set below (see the flag-the-creator note).
if (!prepare_task(orch, args, layout.total_output_size, ActiveMask{}, TaskAttrs{}, &prepared)) {
return TaskOutputTensors{};
}

Expand All @@ -1196,9 +1210,6 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) {
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIC)] = INVALID_KERNEL_ID;
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIV0)] = INVALID_KERNEL_ID;
task.kernel_id[static_cast<int>(PTO2SubtaskSlot::AIV1)] = INVALID_KERNEL_ID;
// alloc_tensors builds a kernel-less descriptor that never dispatches; keep
// the slot untagged so a recycled ring slot cannot leak a stale tag.
task.task_timing_slot = TASK_TIMING_SLOT_NONE;
task.packed_buffer_base = prepared.alloc_result.packed_base;
task.packed_buffer_end = prepared.alloc_result.packed_end;

Expand Down Expand Up @@ -1227,7 +1238,7 @@ TaskOutputTensors PTO2OrchestratorState::alloc_tensors(const L0TaskArgs &args) {
// creation — it should always be transparent, never a barrier. Unlike a
// codegen task there is no Arg-driven hint to honor here, so mark it
// unconditionally. (a2a3 #1285/#1292)
prepared.slot_state->allow_early_resolve = true;
prepared.slot_state->task_attrs.set_early_resolve(true);
prepared.slot_state->mark_completed();
}
orch->inline_completed_tasks++;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,26 +197,14 @@ struct PTO2TaskDescriptor {
// Per-slot kernel IDs (INVALID_KERNEL_ID = inactive)
int32_t kernel_id[PTO2_SUBTASK_SLOT_COUNT];

// Selective task-timing slot (TASK_TIMING_SLOT_NONE = untagged; 0..15 valid).
// Occupies the 4-byte alignment pad that already followed kernel_id[3], so
// the descriptor does not grow; the scheduler folds this task's dispatch/
// finish cycles into the tagged slot (see common/device_phase.h).
int32_t task_timing_slot;

// Packed output buffer (all outputs packed into single contiguous buffer)
void *packed_buffer_base; // Start of packed buffer in GM Heap
void *packed_buffer_end; // End of packed buffer (for heap reclamation)
};

// task_timing_slot must occupy the former padding after kernel_id[3] without
// growing the descriptor or shifting packed_buffer_base — the scheduler and
// shared-memory ABI depend on the existing size and pointer offset.
static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor must not grow: slot uses the existing pad");
static_assert(
offsetof(PTO2TaskDescriptor, task_timing_slot) ==
offsetof(PTO2TaskDescriptor, kernel_id) + sizeof(int32_t) * PTO2_SUBTASK_SLOT_COUNT,
"task_timing_slot must sit immediately after kernel_id in the former pad"
);
// A 4-byte alignment pad follows kernel_id[3]; the scheduler and shared-memory
// ABI depend on the descriptor size and packed_buffer_base offset staying fixed.
static_assert(sizeof(PTO2TaskDescriptor) == 40, "PTO2TaskDescriptor size is part of the shared-memory ABI");
static_assert(offsetof(PTO2TaskDescriptor, packed_buffer_base) == 24, "packed_buffer_base offset must be unchanged");

// =============================================================================
Expand Down Expand Up @@ -445,9 +433,14 @@ struct alignas(64) PTO2TaskSlotState {
// --- Set per-submit (depend on task inputs) ---
ActiveMask active_mask; // Bitmask of active subtask slots (set once)
uint8_t ring_id; // Ring layer (immutable after init)
// These one-byte flags live in the padding before dep_pool_mark to keep
// PTO2TaskSlotState at 64 bytes.
bool allow_early_resolve{false};
// Single per-task attributes byte (early-dispatch hint, sync_start,
// has_predicate, selective timing tag). Lives on slot_state (not payload) so
// fanin walks and the completion path read them off the already-hot producer
// slot_state cache line. Packed into the padding before dep_pool_mark to keep
// PTO2TaskSlotState at 64 bytes. Plain-write (set once at submit, before the
// slot is scheduler-visible), so it MUST NOT share a byte with the atomically
// mutated lifecycle_flags.
TaskAttrs task_attrs{};
std::atomic<uint8_t> lifecycle_flags{PTO2_LIFECYCLE_FLAGS_NONE};
int32_t dep_pool_mark{0}; // Dep pool top after Orch-side wiring

Expand Down Expand Up @@ -521,8 +514,6 @@ struct alignas(64) PTO2TaskSlotState {
return (lifecycle_flags.load(std::memory_order_acquire) & PTO2_SUBTASK_DEFERRED) != 0;
}

void set_allow_early_resolve(bool v) { allow_early_resolve = v; }

/**
* Reset dynamic scheduling fields for slot reuse.
* Called by advance_ring_pointers() after a slot transitions to CONSUMED
Expand All @@ -543,7 +534,8 @@ struct alignas(64) PTO2TaskSlotState {
completed_subtasks.store(0, std::memory_order_relaxed);
next_block_idx.store(0, std::memory_order_relaxed);
lifecycle_flags.store(PTO2_LIFECYCLE_FLAGS_NONE, std::memory_order_relaxed);
allow_early_resolve = false;
// Note: active_mask and task_attrs are per-submit-constant fields
// rewritten in prepare_task on every reuse, so they are not reset here.
}

// === Per-task fanout spinlock ===
Expand Down
84 changes: 69 additions & 15 deletions src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_submit_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,9 @@ enum class PTO2SubtaskSlot : uint8_t {
/**
* Subtask mask bits (for ActiveMask)
*/
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4
inline constexpr uint8_t PTO2_SUBTASK_FLAG_SYNC_START = (1u << 3); // 0x8: all blocks must launch atomically
inline constexpr uint8_t PTO2_SUBTASK_FLAG_HAS_PREDICATE = (1u << 4); // 0x10: task carries a dispatch predicate
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIC = (1u << 0); // 0x1
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV0 = (1u << 1); // 0x2
inline constexpr uint8_t PTO2_SUBTASK_MASK_AIV1 = (1u << 2); // 0x4

// Dispatch-predicate comparison operator. The scheduler evaluates the predicate
// at the dispatch point — the task is ready (fanin satisfied), so the predicate
Expand Down Expand Up @@ -120,7 +118,11 @@ enum class PTO2ResourceShape : uint8_t {
inline constexpr int32_t PTO2_NUM_RESOURCE_SHAPES = 3;

/**
* Bitmask of active subtask slots + flags, sizeof == 1.
* Bitmask of active subtask slots (AIC/AIV0/AIV1), sizeof == 1.
*
* Pure subtask-slot mask: only the low 3 bits (core_mask) are meaningful, so it
* can be OR/==-combined when building MIX clusters without dragging unrelated
* flag bits along. Per-task scheduling flags live on TaskAttrs instead.
*/
class ActiveMask {
public:
Expand All @@ -134,11 +136,6 @@ class ActiveMask {

uint8_t core_mask() const { return raw_ & 0x07u; }

bool requires_sync_start() const { return (raw_ & PTO2_SUBTASK_FLAG_SYNC_START) != 0; }

// True when the task carries a dispatch predicate.
bool has_predicate() const { return (raw_ & PTO2_SUBTASK_FLAG_HAS_PREDICATE) != 0; }

PTO2ResourceShape to_shape() const {
uint8_t cmask = core_mask();
if (cmask == 0) return PTO2ResourceShape::DUMMY;
Expand All @@ -148,10 +145,6 @@ class ActiveMask {
return PTO2ResourceShape::AIV;
}

void set_sync_start() { raw_ |= PTO2_SUBTASK_FLAG_SYNC_START; }

void set_has_predicate() { raw_ |= PTO2_SUBTASK_FLAG_HAS_PREDICATE; }

bool operator==(ActiveMask other) const { return raw_ == other.raw_; }
bool operator!=(ActiveMask other) const { return raw_ != other.raw_; }

Expand All @@ -173,6 +166,67 @@ class ActiveMask {

static_assert(sizeof(ActiveMask) == 1, "ActiveMask must be exactly 1 byte");

/**
* Per-task scheduling attributes, packed into one byte on PTO2TaskSlotState.
*
* Single home for the independent per-task flags: an early-dispatch hint, the
* two dispatch-time predicates (sync_start / has_predicate), and the selective
* timing tag. Consolidating them here keeps active_mask a pure subtask-slot mask
* and lands the timing tag on the scheduler's hot slot_state cache line.
*
* bit 0 allow_early_resolve
* bit 1 sync_start
* bit 2 has_predicate
* bit 3 is_timed (0 => untagged; timing_tag ignored)
* bits 4-7 timing_tag (0..15)
*/
class TaskAttrs {
public:
constexpr TaskAttrs() = default;

bool allow_early_resolve() const { return (raw_ & BIT_EARLY_RESOLVE) != 0; }
void set_early_resolve(bool v) {
if (v) {
raw_ |= BIT_EARLY_RESOLVE;
} else {
raw_ &= static_cast<uint8_t>(~BIT_EARLY_RESOLVE);
}
}

bool requires_sync_start() const { return (raw_ & BIT_SYNC_START) != 0; }
void set_sync_start() { raw_ |= BIT_SYNC_START; }

bool has_predicate() const { return (raw_ & BIT_HAS_PREDICATE) != 0; }
void set_predicate() { raw_ |= BIT_HAS_PREDICATE; }

bool is_timed() const { return (raw_ & BIT_IS_TIMED) != 0; }

// 0..15 timing tag when tagged, else -1 (matches TASK_TIMING_SLOT_NONE; the
// equality is guarded by a static_assert in pto_types.h).
int32_t timing_slot() const { return is_timed() ? static_cast<int32_t>(raw_ >> TIMING_TAG_SHIFT) : -1; }

// slot < 0 => untagged; 0..15 => tagged. Arg::set_task_timing_slot already
// range-checks 0..NUM_TASK_TIMING_SLOTS-1, so only the low 4 bits are stored.
void set_timing_slot(int32_t slot) {
raw_ &= static_cast<uint8_t>(~(BIT_IS_TIMED | TIMING_TAG_MASK));
if (slot >= 0) {
raw_ |= static_cast<uint8_t>(BIT_IS_TIMED | ((slot & 0x0F) << TIMING_TAG_SHIFT));
}
}

private:
static constexpr uint8_t BIT_EARLY_RESOLVE = 1u << 0;
static constexpr uint8_t BIT_SYNC_START = 1u << 1;
static constexpr uint8_t BIT_HAS_PREDICATE = 1u << 2;
static constexpr uint8_t BIT_IS_TIMED = 1u << 3;
static constexpr uint8_t TIMING_TAG_SHIFT = 4;
static constexpr uint8_t TIMING_TAG_MASK = 0xF0u;

uint8_t raw_{0};
};

static_assert(sizeof(TaskAttrs) == 1, "TaskAttrs must be exactly 1 byte");

/**
* Mixed-task submit contract.
*
Expand Down
5 changes: 5 additions & 0 deletions src/a5/runtime/tensormap_and_ringbuffer/runtime/pto_types.h
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@
#include "tensor.h"
#include "tensor_create_info.h" // runtime-only TensorCreateInfo + materialization helpers

// TaskAttrs packs the timing tag into a 4-bit field and reports "untagged" as
// -1, so the tag domain must fit 0..15 and the untagged sentinel must be -1.
static_assert(NUM_TASK_TIMING_SLOTS <= 16, "timing tag must fit TaskAttrs' 4-bit field");
static_assert(TASK_TIMING_SLOT_NONE == -1, "TaskAttrs::timing_slot() reports untagged as -1");

typedef enum {
ASYNC_ENGINE_SDMA = 0,
ASYNC_ENGINE_ROCE = 1,
Expand Down
Loading
Loading