diff --git a/xls/interpreter/proc_evaluator_test_base.cc b/xls/interpreter/proc_evaluator_test_base.cc index cdb40f7e79..7d89b82b7a 100644 --- a/xls/interpreter/proc_evaluator_test_base.cc +++ b/xls/interpreter/proc_evaluator_test_base.cc @@ -780,6 +780,34 @@ TEST_P(ProcEvaluatorTestBase, ConditionalNextProc) { EXPECT_THAT(queue.Read(), Optional(Value(UBits(2, 32)))); } +TEST_P(ProcEvaluatorTestBase, DecoupledNextProc) { + Package package(TestName()); + ProcBuilder pb("dec_next", &package); + XLS_ASSERT_OK_AND_ASSIGN( + StateElement * se, + pb.UnreadStateElement("state_element", Value(UBits(42, 32)), + /*non_synthesizable=*/false)); + pb.StateRead(se); + pb.Next(se, pb.Literal(UBits(56, 32))); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build()); + XLS_ASSERT_OK(package.SetTop(proc)); + + std::unique_ptr queue_manager = + GetParam().CreateQueueManager(&package); + std::unique_ptr evaluator = + GetParam().CreateEvaluator(proc, queue_manager.get()); + + std::unique_ptr continuation = evaluator->NewContinuation( + queue_manager->elaboration().GetUniqueInstance(proc).value()); + + EXPECT_THAT(continuation->GetState(), ElementsAre(Value(UBits(42, 32)))); + EXPECT_THAT( + evaluator->Tick(*continuation), + IsOkAndHolds(TickResult{.execution_state = TickExecutionState::kCompleted, + .channel_instance = std::nullopt, + .progress_made = true})); + EXPECT_THAT(continuation->GetState(), ElementsAre(Value(UBits(56, 32)))); +} TEST_P(ProcEvaluatorTestBase, CollidingNextValuesProc) { // Create an output-only proc which increments its counter value only every // other iteration - but also tries to set the counter value to a different diff --git a/xls/interpreter/proc_interpreter.cc b/xls/interpreter/proc_interpreter.cc index 8359e17a92..4175d78fb4 100644 --- a/xls/interpreter/proc_interpreter.cc +++ b/xls/interpreter/proc_interpreter.cc @@ -212,7 +212,10 @@ class ProcIrInterpreter : public IrInterpreter { return SetValueResult(next, Value::Tuple({})); } } - (*active_next_values_)[next->state_read()->As()->state_element()] + (*active_next_values_)[next->has_state_read() ? next->state_read() + ->As() + ->state_element() + : next->state_element()] .push_back(next); return SetValueResult(next, Value::Tuple({})); } diff --git a/xls/ir/nodes.cc b/xls/ir/nodes.cc index acb4447d71..ec21bfc2d2 100755 --- a/xls/ir/nodes.cc +++ b/xls/ir/nodes.cc @@ -985,9 +985,16 @@ bool StateRead::IsDefinitelyEqualTo(const Node* other) const { std::vector StateRead::GetNextValues() const { std::vector next_values; - for (Node* user : users()) { - if (user->Is() && user->As()->state_read() == this) { - next_values.push_back(user->As()); + if (function_base()->AsProcOrDie()->uses_decoupled_next()) { + const auto& next_values_se = + function_base()->AsProcOrDie()->next_values(state_element_); + next_values.insert(next_values.end(), next_values_se.begin(), + next_values_se.end()); + } else { + for (Node* user : users()) { + if (user->Is() && user->As()->state_read() == this) { + next_values.push_back(user->As()); + } } } return next_values; diff --git a/xls/ir/proc_test.cc b/xls/ir/proc_test.cc index a8df186c98..45f709e871 100644 --- a/xls/ir/proc_test.cc +++ b/xls/ir/proc_test.cc @@ -288,6 +288,41 @@ TEST_F(ProcTest, RemoveStateThatStillHasUse) { HasSubstr("state read st has uses"))); } +TEST_F(ProcTest, GetNextStateReadDecoupled) { + auto p = CreatePackage(); + TokenlessProcBuilder pb(TestName(), "tkn", p.get()); + XLS_ASSERT_OK_AND_ASSIGN(StateElement * state_elem, + pb.UnreadStateElement("x", Value(UBits(42, 32)), + /*non_synthesizable=*/false)); + BValue read = pb.StateRead(state_elem); + BValue next_val = pb.Add(read, pb.Literal(Value(UBits(1, 32)))); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build({})); + + StateRead* state_read = proc->GetStateReadByStateElement(state_elem); + EXPECT_EQ(state_read->GetNextValues().size(), 0); + + XLS_ASSERT_OK(proc->MakeNodeWithName(SourceInfo(), state_elem, + next_val.node(), + /*predicate=*/std::nullopt, + /*label=*/std::nullopt, "x_next") + .status()); + EXPECT_EQ(state_read->GetNextValues().size(), 1); + + XLS_ASSERT_OK_AND_ASSIGN( + Literal * next_value, + proc->MakeNode(SourceInfo(), Value(UBits(43, 32)))); + XLS_ASSERT_OK( + proc->MakeNodeWithName(SourceInfo(), state_elem, next_value, + /*predicate=*/std::nullopt, + /*label=*/std::nullopt, "second_next") + .status()); + EXPECT_EQ(state_read->GetNextValues().size(), 2); + EXPECT_THAT(state_read->GetNextValues(), + UnorderedElementsAre( + m::NextWithStateElement(state_elem, m::Add()), + m::NextWithStateElement(state_elem, m::Literal(43)))); +} + TEST_F(ProcTest, InsertStateElementDecoupled) { auto p = CreatePackage(); ProcBuilder pb("p", p.get()); diff --git a/xls/jit/function_base_jit.cc b/xls/jit/function_base_jit.cc index 212cb5f3f2..95326f38e3 100644 --- a/xls/jit/function_base_jit.cc +++ b/xls/jit/function_base_jit.cc @@ -593,7 +593,13 @@ absl::StatusOr BuildPartitionFunction( if (node->Is()) { // next_value nodes store their output in the state-read's location, and // return nothing themselves. - StateRead* state_read = node->As()->state_read()->As(); + StateRead* state_read = + node->As()->has_state_read() + ? node->As()->state_read()->As() + : node->function_base() + ->AsProcOrDie() + ->GetStateReadByStateElement( + node->As()->state_element()); XLS_RET_CHECK(allocator.GetAllocationKind(state_read) == AllocationKind::kNone); output_buffers = wrapper.GetOutputBuffers(state_read, b); diff --git a/xls/jit/ir_builder_visitor.cc b/xls/jit/ir_builder_visitor.cc index 41692e8739..1bd42a0511 100644 --- a/xls/jit/ir_builder_visitor.cc +++ b/xls/jit/ir_builder_visitor.cc @@ -670,9 +670,13 @@ absl::Status InvokeAssertCallback(llvm::IRBuilder<>* builder, absl::Status InvokeNextValueCallback(llvm::IRBuilder<>* builder, Next* next, llvm::Value* instance_ctx) { llvm::Type* void_type = llvm::Type::getVoidTy(builder->getContext()); + StateElement* state_element = + next->has_state_read() + ? next->state_read()->As()->state_element() + : next->state_element(); llvm::Value* state_element_idx = builder->getInt64( *next->function_base()->AsProcOrDie()->GetStateElementIndex( - next->state_read()->As()->state_element())); + state_element)); llvm::Value* next_value = builder->getInt64(next->id()); InvokeCallback( builder, void_type, instance_ctx, {state_element_idx, next_value}); @@ -2851,7 +2855,11 @@ absl::Status IrBuilderVisitor::HandleNeg(UnOp* neg) { } absl::Status IrBuilderVisitor::HandleNext(Next* next) { - std::vector param_names({"param", "value"}); + std::vector param_names; + if (next->has_state_read()) { + param_names.push_back("param"); + } + param_names.push_back("value"); if (next->predicate().has_value()) { param_names.push_back("predicate"); } @@ -2860,7 +2868,8 @@ absl::Status IrBuilderVisitor::HandleNext(Next* next) { NewNodeIrContext(next, param_names, /*include_wrapper_args=*/true)); llvm::IRBuilder<>& b = node_context.entry_builder(); - llvm::Value* value_ptr = node_context.GetOperandPtr(Next::kValueOperand); + llvm::Value* value_ptr = + node_context.GetOperandPtr(next->value_operand_number()); if (!next->predicate().has_value()) { LlvmMemcpy(node_context.GetOutputPtr(0), value_ptr, @@ -2875,7 +2884,8 @@ absl::Status IrBuilderVisitor::HandleNext(Next* next) { } // If the predicate is true, emulate the `next_value` node's effects. - llvm::Value* predicate = Truthiness(node_context.LoadOperand(2), b); + llvm::Value* predicate = Truthiness( + node_context.LoadOperand(next->predicate_operand_number().value()), b); LlvmIfThen if_then = CreateIfThen(predicate, b, next->GetName()); LlvmMemcpy(node_context.GetOutputPtr(0), value_ptr, diff --git a/xls/passes/proc_state_narrowing_pass.cc b/xls/passes/proc_state_narrowing_pass.cc index e0a6d32111..009e0fcedf 100644 --- a/xls/passes/proc_state_narrowing_pass.cc +++ b/xls/passes/proc_state_narrowing_pass.cc @@ -59,7 +59,9 @@ struct ProcStateNarrowTransform : public Proc::StateElementTransformer { Next* old_next) final { XLS_RET_CHECK_EQ( new_state_read->GetType()->GetFlatBitCount() + known_leading_, - old_next->state_read()->GetType()->GetFlatBitCount()); + proc->GetStateReadByStateElement(old_next->state_element()) + ->GetType() + ->GetFlatBitCount()); return proc->MakeNodeWithName( old_next->loc(), old_next->value(), /*start=*/0, /*width=*/new_state_read->GetType()->GetFlatBitCount(), diff --git a/xls/passes/proc_state_narrowing_pass_test.cc b/xls/passes/proc_state_narrowing_pass_test.cc index 2bf858e732..aad25e4bb2 100644 --- a/xls/passes/proc_state_narrowing_pass_test.cc +++ b/xls/passes/proc_state_narrowing_pass_test.cc @@ -164,6 +164,40 @@ TEST_F(ProcStateNarrowingPassTest, BasicLoop) { EXPECT_THAT(proc->StateElements(), UnorderedElementsAre(m::StateElement( "the_state", p->GetBitsType(3)))); } + +TEST_F(ProcStateNarrowingPassTest, BasicLoopDecoupledNext) { + auto p = CreatePackage(); + XLS_ASSERT_OK_AND_ASSIGN( + auto* chan, p->CreateStreamingChannel("test_chan", ChannelOps::kSendOnly, + p->GetBitsType(32))); + ProcBuilder pb(TestName(), p.get()); + XLS_ASSERT_OK_AND_ASSIGN( + auto* state_element, + pb.UnreadStateElement("the_state", Value(UBits(1, 32)), + /*non_synthesizable=*/false)); + BValue state = pb.StateRead(state_element); + // State just counts up 1 to 6 then resets to 1. + // NB Limit is exactly 6 and comparison is LT so that however the transform is + // done the state fits in 3 bits. + auto in_loop = pb.ULt(state, pb.Literal(UBits(6, 32))); + pb.Send(chan, pb.Literal(Value::Token()), state); + pb.Next(state, pb.Add(state, pb.Literal(UBits(1, 32))), in_loop); + // Reset value is intentionally not something that could be removed by + // exploiting overflow + pb.Next(state, pb.Literal(UBits(1, 32)), pb.Not(in_loop)); + + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build()); + + solvers::ScopedVerifyProcEquivalence svpe(proc, /*activation_count=*/16, + /*include_state=*/false); + ScopedRecordIr sri(p.get()); + EXPECT_THAT(RunPass(proc), IsOkAndHolds(true)); + EXPECT_THAT(RunProcStateCleanup(proc), IsOkAndHolds(true)); + + EXPECT_THAT(proc->StateElements(), UnorderedElementsAre(m::StateElement( + "the_state", p->GetBitsType(3)))); +} + TEST_F(ProcStateNarrowingPassTest, BasicHalt) { auto p = CreatePackage(); XLS_ASSERT_OK_AND_ASSIGN( diff --git a/xls/passes/proc_state_range_query_engine.cc b/xls/passes/proc_state_range_query_engine.cc index f65623a14f..37651fcee4 100644 --- a/xls/passes/proc_state_range_query_engine.cc +++ b/xls/passes/proc_state_range_query_engine.cc @@ -539,7 +539,7 @@ absl::StatusOr> NarrowUsingSegments( for (Next* n : proc->next_values(state_element)) { // Nexts which don't update anything (either due to just being passthrough // or having a known-false predicate) don't need to be taken into account. - if (n->value() == n->state_read() || + if (n->value() == proc->GetStateReadByStateElement(state_element) || (n->predicate() && piqe.IsAllZeros(*n->predicate()))) { continue; } @@ -606,7 +606,7 @@ FindContextualRanges(Proc* proc, const QueryEngine& qe, // TODO(allight): We might want to use data-flow to better track whether // things have changed. This should probably be good enough in practice // however. - if (n->state_read() != n->value()) { + if (proc->GetStateReadByStateElement(state_element) != n->value()) { nexts.push_back(n); } } diff --git a/xls/passes/proc_state_range_query_engine_test.cc b/xls/passes/proc_state_range_query_engine_test.cc index cb5a57479e..5b262b22fb 100644 --- a/xls/passes/proc_state_range_query_engine_test.cc +++ b/xls/passes/proc_state_range_query_engine_test.cc @@ -76,6 +76,37 @@ TEST_F(ProcStateRangeQueryEngineTest, Negatives) { "[[0, 14]]"); } +TEST_F(ProcStateRangeQueryEngineTest, NegativesDecoupledNext) { + auto p = CreatePackage(); + ProcBuilder fb(TestName(), p.get()); + + XLS_ASSERT_OK_AND_ASSIGN(StateElement * state_element, + fb.UnreadStateElement("foo", Value(SBits(0, 32)), + /*non_synthesizable=*/false)); + + BValue st = fb.StateRead(state_element); + BValue res = fb.Add(st, fb.Literal(UBits(3, 32))); + BValue res_pos = fb.Add(st, fb.Literal(UBits(7, 32))); + + // Count -7 to 7 + auto in_loop = fb.SLt(st, fb.Literal(UBits(7, 32))); + fb.Next(state_element, fb.Add(st, fb.Literal(UBits(1, 32))), in_loop); + // If we aren't looping the value goes to -7 + fb.Next(state_element, fb.Literal(SBits(-7, 32)), fb.Not(in_loop)); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, fb.Build()); + + ProcStateRangeQueryEngine qe; + XLS_ASSERT_OK(qe.Populate(proc).status()); + EXPECT_EQ( + IntervalSetTreeToString(qe.GetIntervals(st.node())), + absl::StrFormat("[[0, 7], [%v, %v]]", SBits(-7, 32), SBits(-1, 32))); + EXPECT_EQ( + IntervalSetTreeToString(qe.GetIntervals(res.node())), + absl::StrFormat("[[0, 10], [%v, %v]]", SBits(-7 + 3, 32), SBits(-1, 32))); + EXPECT_EQ(IntervalSetTreeToString(qe.GetIntervals(res_pos.node())), + "[[0, 14]]"); +} + TEST_F(ProcStateRangeQueryEngineTest, NegativesWithEq) { auto p = CreatePackage(); ProcBuilder fb(TestName(), p.get()); diff --git a/xls/scheduling/pipeline_schedule.cc b/xls/scheduling/pipeline_schedule.cc index 3e0111a2f0..43fed07797 100644 --- a/xls/scheduling/pipeline_schedule.cc +++ b/xls/scheduling/pipeline_schedule.cc @@ -207,7 +207,8 @@ bool PipelineSchedule::IsLiveOutOfCycle(Node* node, int64_t c) const { } if (user->Is()) { Next* user_next = user->As(); - if (user_next->predicate() != node && user_next->value() != node) { + if (user_next->predicate() != node && user_next->value() != node && + user_next->has_state_read()) { CHECK_EQ(user_next->state_read(), node); // This Next node only uses this StateRead node to target the state // register it needs to write to; it doesn't actually need the value diff --git a/xls/scheduling/pipeline_schedule_test.cc b/xls/scheduling/pipeline_schedule_test.cc index 350c9b8f04..36c6e4e502 100644 --- a/xls/scheduling/pipeline_schedule_test.cc +++ b/xls/scheduling/pipeline_schedule_test.cc @@ -2372,6 +2372,44 @@ TEST_P(PipelineScheduleTest, ProcWithMultipleStateReads) { EXPECT_EQ(schedule.length(), 1); } +TEST_P(PipelineScheduleTest, ProcDecoupledFindMinimumCaseThroughput) { + Package p(TestName()); + Type* u32 = p.GetBitsType(32); + XLS_ASSERT_OK_AND_ASSIGN( + Channel * out_ch, + p.CreateStreamingChannel("out_ch", ChannelOps::kSendOnly, u32)); + + TokenlessProcBuilder pb("the_proc", "tkn", &p); + XLS_ASSERT_OK_AND_ASSIGN(StateElement * se, + pb.UnreadStateElement("state", Value(UBits(0, 32)), + /*non_synthesizable=*/false)); + BValue current = pb.StateRead(se); + BValue first_add = pb.Add(current, pb.Literal(UBits(1, 32))); + BValue second_add = pb.Add(first_add, pb.Literal(UBits(2, 32))); + BValue third_add = pb.Add(second_add, pb.Literal(UBits(3, 32))); + + pb.Send(out_ch, third_add); + BValue next = pb.Next(se, third_add); + + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build()); + + SchedulingOptions options = this->options(); + options.clock_period_ps(1); + options.minimize_worst_case_throughput(true); + + XLS_ASSERT_OK_AND_ASSIGN( + PipelineSchedule schedule, + RunPipelineSchedule(proc, TestDelayEstimator(), options)); + EXPECT_EQ(schedule.cycle(current.node()), 0); + EXPECT_EQ(schedule.cycle(next.node()), 2); + + ASSERT_TRUE(proc->GetInitiationInterval().has_value()); + EXPECT_EQ(proc->GetInitiationInterval().value(), 3); + + EXPECT_LE(schedule.cycle(next.node()) - schedule.cycle(current.node()), + *proc->GetInitiationInterval() - 1); +} + TEST_P(PipelineScheduleErrorTest, ProcWithZeroReadsErrors) { Package p(TestName()); TokenlessProcBuilder pb("the_proc", "tkn", &p); @@ -2816,7 +2854,36 @@ TEST_F(PipelineScheduleTest, EXPECT_EQ(schedule.cycle(next.node()), 1); EXPECT_EQ(schedule.cycle(read_mutually_exclusive.node()), 2); } +TEST_F(PipelineScheduleTest, DecoupledNextIsOutOfCycle) { + Package p(TestName()); + ProcBuilder pb("the_proc", &p); + XLS_ASSERT_OK_AND_ASSIGN(StateElement * se, + pb.UnreadStateElement("state", Value(UBits(0, 32)), + /*non_synthesizable=*/false)); + BValue read = pb.StateRead(se); + BValue next = pb.Next(se, read); + XLS_ASSERT_OK_AND_ASSIGN(Proc * proc, pb.Build()); + + // Run scheduler to get a valid base schedule first. + SchedulingOptions options; + options.clock_period_ps(1); + options.pipeline_stages(2); + XLS_ASSERT_OK_AND_ASSIGN( + PipelineSchedule schedule, + RunPipelineSchedule(proc, TestDelayEstimator(), options)); + ScheduleCycleMap cycle_map = schedule.GetCycleMap(); + cycle_map[read.node()] = 0; + cycle_map[next.node()] = 1; + XLS_ASSERT_OK_AND_ASSIGN(schedule, PipelineSchedule::Create(proc, cycle_map)); + EXPECT_EQ(schedule.cycle(read.node()), 0); + EXPECT_EQ(schedule.cycle(next.node()), 1); + // Read is consumed by Next in cycle 1, so read IS live out of cycle 0 + EXPECT_TRUE(schedule.IsLiveOutOfCycle(read.node(), 0)); + // By cycle 1 (where Next is), read no longer requires a pipeline register + // and isn't live out of cycle 1. + EXPECT_FALSE(schedule.IsLiveOutOfCycle(read.node(), 1)); +} TEST_F(PipelineScheduleTest, ProcWriteBeforeReadFailsVerification) { Package p(TestName()); ProcBuilder pb("the_proc", &p); diff --git a/xls/scheduling/run_pipeline_schedule.cc b/xls/scheduling/run_pipeline_schedule.cc index 094ef63029..e4ccef535b 100644 --- a/xls/scheduling/run_pipeline_schedule.cc +++ b/xls/scheduling/run_pipeline_schedule.cc @@ -232,7 +232,10 @@ absl::StatusOr FindMinimumWorstCaseThroughput( // Extract the worst-case throughput from this schedule as an upper bound. int64_t pessimistic_worst_case_throughput = 1; for (Next* next : proc->next_values()) { - Node* state_read = next->state_read(); + Node* state_read = + next->has_state_read() + ? next->state_read() + : proc->GetStateReadByStateElement(next->state_element()); const int64_t backedge_length = schedule_cycle_map[next] - schedule_cycle_map[state_read]; pessimistic_worst_case_throughput =