From da33159832c2eabfa43261a22e210cbbed028d93 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Wed, 8 Jul 2026 14:30:21 -0700 Subject: [PATCH 01/15] adding the new api --- .../assignor/streams}/GroupAssignment.java | 7 +- .../api/assignor/streams}/GroupSpec.java | 9 +- .../assignor/streams}/MemberAssignment.java | 7 +- .../assignor/streams/MemberSubscription.java} | 11 +- .../api/assignor/streams}/TaskAssignor.java | 7 +- .../streams}/TaskAssignorException.java | 6 +- .../group/api/assignor/streams}/TaskId.java | 7 +- .../assignor/streams}/TopologyDescriber.java | 7 +- .../api/assignor/streams/package-info.java | 21 ++ .../group/GroupMetadataManager.java | 4 +- .../group/streams/MemberTaskOffsets.java | 2 +- .../streams/TargetAssignmentBuilder.java | 18 +- .../group/streams/TopologyMetadata.java | 4 +- .../group/streams/assignor/GroupSpecImpl.java | 5 +- .../group/streams/assignor/MockAssignor.java | 12 +- .../group/streams/assignor/ProcessState.java | 3 + .../streams/assignor/StickyTaskAssignor.java | 13 +- .../group/GroupMetadataManagerTest.java | 16 +- .../GroupMetadataManagerTestContext.java | 2 +- .../group/streams/MemberTaskOffsetsTest.java | 2 +- .../group/streams/MockTaskAssignor.java | 12 +- .../group/streams/StreamsGroupTest.java | 2 +- .../streams/TargetAssignmentBuilderTest.java | 32 +- .../streams/assignor/GroupSpecImplTest.java | 6 +- .../streams/assignor/MockAssignorTest.java | 20 +- .../streams/assignor/ProcessStateTest.java | 2 + .../assignor/StickyTaskAssignorTest.java | 343 +++++++++--------- .../StreamsAssignorBenchmarkUtils.java | 8 +- .../StreamsStickyAssignorBenchmark.java | 20 +- 29 files changed, 353 insertions(+), 255 deletions(-) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/GroupAssignment.java (82%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/GroupSpec.java (79%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/MemberAssignment.java (87%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor/AssignmentMemberSpec.java => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java} (88%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/TaskAssignor.java (85%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/TaskAssignorException.java (83%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/TaskId.java (86%) rename group-coordinator/{src/main/java/org/apache/kafka/coordinator/group/streams/assignor => group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams}/TopologyDescriber.java (89%) create mode 100644 group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/package-info.java diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupAssignment.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupAssignment.java similarity index 82% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupAssignment.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupAssignment.java index c4ac5803b35d0..a9b931a9fda07 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupAssignment.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupAssignment.java @@ -14,7 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Map; import java.util.Objects; @@ -24,6 +27,8 @@ * * @param members The member assignments keyed by member ID. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public record GroupAssignment(Map members) { public GroupAssignment { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpec.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java similarity index 79% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpec.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java index 1a8e7edc01c63..3755bf1c4ab03 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpec.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java @@ -14,19 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Map; /** * The group metadata specifications required to compute the target assignment. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public interface GroupSpec { /** * @return Member metadata keyed by member Id. */ - Map members(); + Map members(); /** * @return Any configurations passed to the assignor. diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignment.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java similarity index 87% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignment.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java index 2902e647382f7..37a9cd9301546 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignment.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java @@ -14,7 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Map; import java.util.Objects; @@ -27,6 +30,8 @@ * @param standbyTasks The standby tasks assigned to this member keyed by subtopologyId. * @param warmupTasks The warm-up tasks assigned to this member keyed by subtopologyId. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public record MemberAssignment(Map> activeTasks, Map> standbyTasks, Map> warmupTasks) { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/AssignmentMemberSpec.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java similarity index 88% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/AssignmentMemberSpec.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java index fa41b5a8a3b4b..5f3d1859931c6 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/AssignmentMemberSpec.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java @@ -14,7 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Collections; import java.util.Map; @@ -34,7 +37,9 @@ * @param clientTags The client tags for a rack-aware assignment. * @param taskOffsets The last received cumulative task offsets of assigned tasks or dormant tasks. */ -public record AssignmentMemberSpec(Optional instanceId, +@InterfaceAudience.Public +@InterfaceStability.Evolving +public record MemberSubscription(Optional instanceId, Optional rackId, Map> activeTasks, Map> standbyTasks, @@ -45,7 +50,7 @@ public record AssignmentMemberSpec(Optional instanceId, Map taskEndOffsets ) { - public AssignmentMemberSpec { + public MemberSubscription { Objects.requireNonNull(instanceId); Objects.requireNonNull(rackId); activeTasks = Collections.unmodifiableMap(Objects.requireNonNull(activeTasks)); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignor.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java similarity index 85% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignor.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java index 73bff075ca648..895cfcbbf3637 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignor.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java @@ -14,11 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; /** * Server side task assignor used by streams groups. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public interface TaskAssignor { /** diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignorException.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java similarity index 83% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignorException.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java index 2fda6a9e9ec62..17a8a2974fdb3 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskAssignorException.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java @@ -14,13 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.errors.ApiException; /** * Exception thrown by {@link TaskAssignor#assign(GroupSpec, TopologyDescriber)}}. The exception is only used internally. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public class TaskAssignorException extends ApiException { public TaskAssignorException(String message) { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java similarity index 86% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java index 979c042df00c5..22497e378ec62 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java @@ -14,7 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Comparator; import java.util.Objects; @@ -25,6 +28,8 @@ * @param subtopologyId The unique identifier of the subtopology. * @param partition The partition of the input topics this task is processing. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public record TaskId(String subtopologyId, int partition) implements Comparable { public TaskId { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TopologyDescriber.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java similarity index 89% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TopologyDescriber.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java index d6f7a3ab579c6..5865779d6a2de 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TopologyDescriber.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java @@ -14,7 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.coordinator.group.streams.assignor; +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; import java.util.List; import java.util.NoSuchElementException; @@ -22,6 +25,8 @@ /** * The topology describer is used by the {@link TaskAssignor} to get topic and task metadata of the group's topology. */ +@InterfaceAudience.Public +@InterfaceStability.Evolving public interface TopologyDescriber { /** diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/package-info.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/package-info.java new file mode 100644 index 0000000000000..33483727c3fa7 --- /dev/null +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provides the public API for broker-side custom task assignors used by Kafka Streams groups. + */ +package org.apache.kafka.coordinator.group.api.assignor.streams; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index 38fee045c3d7c..41d5f455ce202 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -92,6 +92,8 @@ import org.apache.kafka.coordinator.group.api.assignor.PartitionAssignorException; import org.apache.kafka.coordinator.group.api.assignor.ShareGroupPartitionAssignor; import org.apache.kafka.coordinator.group.api.assignor.SubscriptionType; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.assignor.SimpleAssignor; import org.apache.kafka.coordinator.group.classic.ClassicGroup; import org.apache.kafka.coordinator.group.classic.ClassicGroupMember; @@ -162,8 +164,6 @@ import org.apache.kafka.coordinator.group.streams.TasksTuple; import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; import org.apache.kafka.coordinator.group.streams.assignor.StickyTaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignorException; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java index ed2434a79bffa..552d47eaf6ae8 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; -import org.apache.kafka.coordinator.group.streams.assignor.TaskId; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import java.util.HashMap; import java.util.List; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java index 6ce2632b6f986..2878b2b752e74 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java @@ -19,12 +19,12 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.coordinator.common.runtime.CoordinatorMetadataImage; import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; -import org.apache.kafka.coordinator.group.streams.assignor.AssignmentMemberSpec; -import org.apache.kafka.coordinator.group.streams.assignor.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignment; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignorException; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import java.util.ArrayList; @@ -116,12 +116,12 @@ public TargetAssignmentBuilder( this.assignmentConfigs = Objects.requireNonNull(assignmentConfigs); } - static AssignmentMemberSpec createAssignmentMemberSpec( + static MemberSubscription createMemberSubscription( StreamsGroupMember member, TasksTuple targetAssignment, MemberTaskOffsets taskOffsets ) { - return new AssignmentMemberSpec( + return new MemberSubscription( member.instanceId(), member.rackId(), targetAssignment.activeTasks(), @@ -217,10 +217,10 @@ public TargetAssignmentBuilder withTopology( * @throws TaskAssignorException if the target assignment cannot be computed. */ public TargetAssignmentResult build() throws TaskAssignorException { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. - members.forEach((memberId, member) -> memberSpecs.put(memberId, createAssignmentMemberSpec( + members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberSubscription( member, targetAssignment.getOrDefault(memberId, org.apache.kafka.coordinator.group.streams.TasksTuple.EMPTY), taskOffsets.getOrDefault(memberId, MemberTaskOffsets.EMPTY) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopologyMetadata.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopologyMetadata.java index e07a3f1eca42f..1a9cc58a66eb1 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopologyMetadata.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TopologyMetadata.java @@ -17,7 +17,7 @@ package org.apache.kafka.coordinator.group.streams; import org.apache.kafka.coordinator.common.runtime.CoordinatorMetadataImage; -import org.apache.kafka.coordinator.group.streams.assignor.TopologyDescriber; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import java.util.Collections; @@ -27,7 +27,7 @@ import java.util.SortedMap; /** - * The topology metadata class is used by the {@link org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor} to get topic and + * The topology metadata class is used by the {@link org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor} to get topic and * partition metadata for the topology that the streams group using. * * @param metadataImage The metadata image diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java index 7479fcae0fc2d..0f2cea510d96f 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; + import java.util.Map; import java.util.Objects; @@ -25,7 +28,7 @@ * @param members The member metadata keyed by member ID. * @param assignmentConfigs Any configurations passed to the assignor. */ -public record GroupSpecImpl(Map members, +public record GroupSpecImpl(Map members, Map assignmentConfigs) implements GroupSpec { public GroupSpecImpl { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java index 93827d29b1c75..c10fd943a4e1c 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java @@ -16,6 +16,14 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; + import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -56,9 +64,9 @@ public GroupAssignment assign( } // Copy existing assignment and fill temporary data structures - for (Map.Entry memberEntry : groupSpec.members().entrySet()) { + for (Map.Entry memberEntry : groupSpec.members().entrySet()) { final String memberId = memberEntry.getKey(); - final AssignmentMemberSpec memberSpec = memberEntry.getValue(); + final MemberSubscription memberSpec = memberEntry.getValue(); Map> activeTasks = new HashMap<>(memberSpec.activeTasks()); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java index 330a0cc0da3a7..9a73b2401ab1a 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; + import java.util.AbstractMap; import java.util.HashMap; import java.util.HashSet; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index 4828f0238b266..8c0075d85b665 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -17,6 +17,15 @@ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,11 +115,11 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol localState.processIdToState = new HashMap<>(localState.totalMembersWithActiveTaskCapacity); localState.activeTaskToPrevMember = new HashMap<>(localState.totalActiveTasks); localState.standbyTaskToPrevMember = new HashMap<>(localState.numStandbyReplicas > 0 ? (localState.totalTasks - localState.totalActiveTasks) / localState.numStandbyReplicas : 0); - for (final Map.Entry memberEntry : groupSpec.members().entrySet()) { + for (final Map.Entry memberEntry : groupSpec.members().entrySet()) { final String memberId = memberEntry.getKey(); final String processId = memberEntry.getValue().processId(); final Member member = new Member(processId, memberId); - final AssignmentMemberSpec memberSpec = memberEntry.getValue(); + final MemberSubscription memberSpec = memberEntry.getValue(); localState.processIdToState.putIfAbsent(processId, new ProcessState(processId)); localState.processIdToState.get(processId).addMember(memberId); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index f3568dc089b92..c704dfbad9e20 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -97,6 +97,8 @@ import org.apache.kafka.coordinator.group.api.assignor.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.GroupSpec; import org.apache.kafka.coordinator.group.api.assignor.PartitionAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.classic.ClassicGroup; import org.apache.kafka.coordinator.group.classic.ClassicGroupMember; import org.apache.kafka.coordinator.group.classic.ClassicGroupState; @@ -148,8 +150,6 @@ import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; import org.apache.kafka.coordinator.group.streams.TasksTuple; import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignorException; import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.image.MetadataProvenance; @@ -18746,8 +18746,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskId(subtopology1, 0), 10L), - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskId(subtopology1, 0), 20L) + Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 10L), + Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 20L) ), group.taskOffsets(memberId) ); @@ -18772,8 +18772,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) assertEquals(List.of(), result.records()); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskId(subtopology1, 0), 12L), - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskId(subtopology1, 0), 20L) + Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 12L), + Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 20L) ), group.taskOffsets(memberId) ); @@ -19347,8 +19347,8 @@ public void testStreamsGroupMemberJoiningWithStaleTopology() { ) .build(); - assignor.prepareGroupAssignment(new org.apache.kafka.coordinator.group.streams.assignor.GroupAssignment(Map.of( - memberId, org.apache.kafka.coordinator.group.streams.assignor.MemberAssignment.empty() + assignor.prepareGroupAssignment(new org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment(Map.of( + memberId, org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment.empty() ))); // Member joins the streams group. diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java index 6c81bcb03ec25..b042d94b59452 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTestContext.java @@ -60,6 +60,7 @@ import org.apache.kafka.coordinator.common.runtime.MockCoordinatorTimer; import org.apache.kafka.coordinator.group.api.assignor.ConsumerGroupPartitionAssignor; import org.apache.kafka.coordinator.group.api.assignor.ShareGroupPartitionAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.classic.ClassicGroup; import org.apache.kafka.coordinator.group.generated.ConsumerGroupCurrentMemberAssignmentKey; import org.apache.kafka.coordinator.group.generated.ConsumerGroupCurrentMemberAssignmentValue; @@ -114,7 +115,6 @@ import org.apache.kafka.coordinator.group.streams.StreamsGroupHeartbeatResult; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; import org.apache.kafka.coordinator.group.streams.TasksTupleWithEpochs; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; import org.apache.kafka.coordinator.group.streams.topics.InternalTopicManager; import org.apache.kafka.server.authorizer.Authorizer; import org.apache.kafka.server.common.ApiMessageAndVersion; diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java index 941c5c3cc97a0..1a889d20c7375 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; -import org.apache.kafka.coordinator.group.streams.assignor.TaskId; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.junit.jupiter.api.Test; diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java index 2b1f1fb551183..e3ffb14f9ac86 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java @@ -16,12 +16,12 @@ */ package org.apache.kafka.coordinator.group.streams; -import org.apache.kafka.coordinator.group.streams.assignor.GroupAssignment; -import org.apache.kafka.coordinator.group.streams.assignor.GroupSpec; -import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignment; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignorException; -import org.apache.kafka.coordinator.group.streams.assignor.TopologyDescriber; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import java.util.Map; import java.util.Map.Entry; diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java index a75c108314998..a0a0ea8da02f5 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java @@ -38,6 +38,7 @@ import org.apache.kafka.coordinator.group.OffsetAndMetadata; import org.apache.kafka.coordinator.group.OffsetExpirationCondition; import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; @@ -48,7 +49,6 @@ import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; -import org.apache.kafka.coordinator.group.streams.assignor.TaskId; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.timeline.SnapshotRegistry; diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index 0edb840870c19..333cc25119558 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -22,14 +22,14 @@ import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; import org.apache.kafka.coordinator.common.runtime.KRaftCoordinatorMetadataImage; import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; -import org.apache.kafka.coordinator.group.streams.assignor.AssignmentMemberSpec; -import org.apache.kafka.coordinator.group.streams.assignor.GroupAssignment; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignment; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskId; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; @@ -50,7 +50,7 @@ import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord; -import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createAssignmentMemberSpec; +import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createMemberSubscription; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,7 +88,7 @@ public void testBuildEmptyAssignmentWhenTopologyNotReady() { @ParameterizedTest @EnumSource(TaskRole.class) - public void testCreateAssignmentMemberSpec(TaskRole taskRole) { + public void testCreateMemberSubscription(TaskRole taskRole) { String fooSubtopologyId = Uuid.randomUuid().toString(); String barSubtopologyId = Uuid.randomUuid().toString(); @@ -105,13 +105,13 @@ public void testCreateAssignmentMemberSpec(TaskRole taskRole) { mkTasks(barSubtopologyId, 1, 2, 3) ); - AssignmentMemberSpec assignmentMemberSpec = createAssignmentMemberSpec( + MemberSubscription memberSubscription = createMemberSubscription( member, assignment, MemberTaskOffsets.EMPTY ); - assertEquals(new AssignmentMemberSpec( + assertEquals(new MemberSubscription( Optional.of("instanceId"), Optional.of("rackId"), assignment.activeTasks(), @@ -121,11 +121,11 @@ public void testCreateAssignmentMemberSpec(TaskRole taskRole) { clientTags, Map.of(), Map.of() - ), assignmentMemberSpec); + ), memberSubscription); } @Test - public void testCreateAssignmentMemberSpecPopulatesTaskOffsets() { + public void testCreateMemberSubscriptionPopulatesTaskOffsets() { String fooSubtopologyId = Uuid.randomUuid().toString(); StreamsGroupMember member = new StreamsGroupMember.Builder("member-id") @@ -138,14 +138,14 @@ public void testCreateAssignmentMemberSpecPopulatesTaskOffsets() { Map taskOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 10L); Map taskEndOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 20L); - AssignmentMemberSpec assignmentMemberSpec = createAssignmentMemberSpec( + MemberSubscription memberSubscription = createMemberSubscription( member, TasksTuple.EMPTY, new MemberTaskOffsets(taskOffsets, taskEndOffsets) ); - assertEquals(taskOffsets, assignmentMemberSpec.taskOffsets()); - assertEquals(taskEndOffsets, assignmentMemberSpec.taskEndOffsets()); + assertEquals(taskOffsets, memberSubscription.taskOffsets()); + assertEquals(taskEndOffsets, memberSubscription.taskEndOffsets()); } @Test @@ -429,9 +429,9 @@ public void prepareMemberAssignment( public org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.TargetAssignmentResult build() { // Prepare expected member specs. - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); members.forEach((memberId, member) -> - memberSpecs.put(memberId, createAssignmentMemberSpec( + memberSpecs.put(memberId, createMemberSubscription( member, targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY), MemberTaskOffsets.EMPTY diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java index be2aa696e8f79..c8da62c74e650 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -28,14 +30,14 @@ public class GroupSpecImplTest { - private Map members; + private Map members; private GroupSpecImpl groupSpec; @BeforeEach void setUp() { members = new HashMap<>(); - members.put("test-member", new AssignmentMemberSpec( + members.put("test-member", new MemberSubscription( Optional.of("test-instance"), Optional.of("test-rack"), Map.of(), diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java index da73e27e8c299..23e9aac07850b 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java @@ -16,6 +16,12 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; + import org.junit.jupiter.api.Test; import java.util.HashMap; @@ -60,7 +66,7 @@ public void testZeroMembers() { @Test public void testDoubleAssignment() { - final AssignmentMemberSpec memberSpec1 = new AssignmentMemberSpec( + final MemberSubscription memberSpec1 = new MemberSubscription( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -72,7 +78,7 @@ public void testDoubleAssignment() { Map.of() ); - final AssignmentMemberSpec memberSpec2 = new AssignmentMemberSpec( + final MemberSubscription memberSpec2 = new MemberSubscription( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -113,7 +119,7 @@ public void testBasicScenario() { @Test public void testSingleMember() { - final AssignmentMemberSpec memberSpec = new AssignmentMemberSpec( + final MemberSubscription memberSpec = new MemberSubscription( Optional.empty(), Optional.empty(), Map.of(), @@ -145,7 +151,7 @@ public void testSingleMember() { @Test public void testTwoMembersTwoSubtopologies() { - final AssignmentMemberSpec memberSpec1 = new AssignmentMemberSpec( + final MemberSubscription memberSpec1 = new MemberSubscription( Optional.empty(), Optional.empty(), Map.of(), @@ -157,7 +163,7 @@ public void testTwoMembersTwoSubtopologies() { Map.of() ); - final AssignmentMemberSpec memberSpec2 = new AssignmentMemberSpec( + final MemberSubscription memberSpec2 = new MemberSubscription( Optional.empty(), Optional.empty(), Map.of(), @@ -198,7 +204,7 @@ public void testTwoMembersTwoSubtopologies() { @Test public void testTwoMembersTwoSubtopologiesStickiness() { - final AssignmentMemberSpec memberSpec1 = new AssignmentMemberSpec( + final MemberSubscription memberSpec1 = new MemberSubscription( Optional.empty(), Optional.empty(), mkMap( @@ -213,7 +219,7 @@ public void testTwoMembersTwoSubtopologiesStickiness() { Map.of() ); - final AssignmentMemberSpec memberSpec2 = new AssignmentMemberSpec( + final MemberSubscription memberSpec2 = new MemberSubscription( Optional.empty(), Optional.empty(), mkMap( diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java index ba63afddd30a4..953970bb8ad0b 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java @@ -17,6 +17,8 @@ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; + import org.junit.jupiter.api.Test; import java.util.Set; diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java index 161f3e6a5d808..54725e9a243b2 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java @@ -17,6 +17,11 @@ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; + import org.junit.jupiter.api.Test; import org.mockito.internal.util.collections.Sets; @@ -53,9 +58,9 @@ public void testToStringReturnsName() { @Test public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCount() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); final GroupAssignment result = assignor.assign( new GroupSpecImpl( @@ -78,13 +83,13 @@ public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCoun @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { - final AssignmentMemberSpec memberSpec11 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec12 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec21 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec22 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec31 = createAssignmentMemberSpec("process3"); - final AssignmentMemberSpec memberSpec32 = createAssignmentMemberSpec("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberSubscription memberSpec11 = createMemberSubscription("process1"); + final MemberSubscription memberSpec12 = createMemberSubscription("process1"); + final MemberSubscription memberSpec21 = createMemberSubscription("process2"); + final MemberSubscription memberSpec22 = createMemberSubscription("process2"); + final MemberSubscription memberSpec31 = createMemberSubscription("process3"); + final MemberSubscription memberSpec32 = createMemberSubscription("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -106,13 +111,13 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { final Map> tasks = mkMap(mkEntry("test-subtopology1", Sets.newSet(0, 1, 2)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2))); - final AssignmentMemberSpec memberSpec11 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec12 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec21 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec22 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec31 = createAssignmentMemberSpec("process3"); - final AssignmentMemberSpec memberSpec32 = createAssignmentMemberSpec("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberSubscription memberSpec11 = createMemberSubscription("process1"); + final MemberSubscription memberSpec12 = createMemberSubscription("process1"); + final MemberSubscription memberSpec21 = createMemberSubscription("process2"); + final MemberSubscription memberSpec22 = createMemberSubscription("process2"); + final MemberSubscription memberSpec31 = createMemberSubscription("process3"); + final MemberSubscription memberSpec32 = createMemberSubscription("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -135,9 +140,9 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { @Test public void shouldNotMigrateActiveTaskToOtherProcess() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -154,8 +159,8 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { testMember1.activeTasks().get("test-subtopology").size() + testMember2.activeTasks().get("test-subtopology").size()); // flip the previous active tasks assignment around. - memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); members = mkMap(mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); result = assignor.assign( @@ -175,10 +180,10 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { @Test public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -203,10 +208,10 @@ public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() @Test public void shouldAssignBasedOnCapacity() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec21 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec22 = createAssignmentMemberSpec("process2"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec21 = createMemberSubscription("process2"); + final MemberSubscription memberSpec22 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -230,9 +235,9 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { final Map> activeTasks = mkMap( mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3, 4, 5)), mkEntry("test-subtopology2", Sets.newSet(0))); - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", activeTasks, Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", activeTasks, Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -257,12 +262,12 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { @Test public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { - AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - AssignmentMemberSpec memberSpec4 = createAssignmentMemberSpec("process4"); - AssignmentMemberSpec memberSpec5 = createAssignmentMemberSpec("process5"); - Map members = mkMap( + MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + MemberSubscription memberSpec4 = createMemberSubscription("process4"); + MemberSubscription memberSpec5 = createMemberSubscription("process5"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); @@ -291,11 +296,11 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { assertNull(testMember5.activeTasks().get("test-subtopology")); // change up the assignment and make sure it is still sticky - memberSpec1 = createAssignmentMemberSpec("process1"); - memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - memberSpec3 = createAssignmentMemberSpec("process3"); - memberSpec4 = createAssignmentMemberSpec("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - memberSpec5 = createAssignmentMemberSpec("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + memberSpec1 = createMemberSubscription("process1"); + memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + memberSpec3 = createMemberSubscription("process3"); + memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + memberSpec5 = createMemberSubscription("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); @@ -327,10 +332,10 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { @Test public void shouldAssignTasksToClientWithPreviousStandbyTasks() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + final MemberSubscription memberSpec3 = createMemberSubscription("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -354,9 +359,9 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() { @Test public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurrentActiveTasks() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -380,15 +385,15 @@ public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurren @Test public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final AssignmentMemberSpec memberSpec21 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec21 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final AssignmentMemberSpec memberSpec22 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec22 = createMemberSubscription("process2", Map.of(), Map.of()); - Map members = mkMap( + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); @@ -414,11 +419,11 @@ public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { @Test public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTaskIsAssignedTo() { final Map> tasks = mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))); - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - final AssignmentMemberSpec memberSpec4 = createAssignmentMemberSpec("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberSubscription memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); @@ -451,10 +456,10 @@ public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTask @Test public void shouldAssignMultipleReplicasOfStandbyTask() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -471,8 +476,8 @@ public void shouldAssignMultipleReplicasOfStandbyTask() { @Test public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHavingTheTaskAssigned() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + Map members = mkMap( mkEntry("member1", memberSpec1)); final GroupAssignment result = assignor.assign( @@ -486,10 +491,10 @@ public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHaving @Test public void shouldAssignActiveAndStandbyTasks() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -505,12 +510,12 @@ public void shouldAssignActiveAndStandbyTasks() { @Test public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { - final AssignmentMemberSpec memberSpec11 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec12 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec13 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - Map members = mkMap( + final MemberSubscription memberSpec11 = createMemberSubscription("process1"); + final MemberSubscription memberSpec12 = createMemberSubscription("process1"); + final MemberSubscription memberSpec13 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member1_3", memberSpec13), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -526,13 +531,13 @@ public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { @Test public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - final AssignmentMemberSpec memberSpec4 = createAssignmentMemberSpec("process4"); - final AssignmentMemberSpec memberSpec5 = createAssignmentMemberSpec("process5"); - final AssignmentMemberSpec memberSpec6 = createAssignmentMemberSpec("process6"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final MemberSubscription memberSpec4 = createMemberSubscription("process4"); + final MemberSubscription memberSpec5 = createMemberSubscription("process5"); + final MemberSubscription memberSpec6 = createMemberSubscription("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -547,13 +552,13 @@ public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { @Test public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - final AssignmentMemberSpec memberSpec4 = createAssignmentMemberSpec("process4"); - final AssignmentMemberSpec memberSpec5 = createAssignmentMemberSpec("process5"); - final AssignmentMemberSpec memberSpec6 = createAssignmentMemberSpec("process6"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final MemberSubscription memberSpec4 = createMemberSubscription("process4"); + final MemberSubscription memberSpec5 = createMemberSubscription("process5"); + final MemberSubscription memberSpec6 = createMemberSubscription("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -570,10 +575,10 @@ public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { @Test public void shouldAssignMoreTasksToClientWithMoreCapacity() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec21 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec22 = createAssignmentMemberSpec("process2"); - Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec21 = createMemberSubscription("process2"); + final MemberSubscription memberSpec22 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -587,11 +592,11 @@ public void shouldAssignMoreTasksToClientWithMoreCapacity() { @Test public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame() { - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final AssignmentMemberSpec memberSpec4 = createAssignmentMemberSpec("process4"); - Map members = mkMap( + final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final MemberSubscription memberSpec4 = createMemberSubscription("process4"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); GroupAssignment result = assignor.assign( @@ -607,10 +612,10 @@ public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame( @Test public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - Map members = mkMap( + final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -625,10 +630,10 @@ public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { @Test public void shouldRebalanceTasksToClientsBasedOnCapacity() { - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); - final AssignmentMemberSpec memberSpec31 = createAssignmentMemberSpec("process3"); - final AssignmentMemberSpec memberSpec32 = createAssignmentMemberSpec("process3"); - Map members = mkMap( + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); + final MemberSubscription memberSpec31 = createMemberSubscription("process3"); + final MemberSubscription memberSpec32 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member2", memberSpec2), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); GroupAssignment result = assignor.assign( @@ -644,10 +649,10 @@ public void shouldRebalanceTasksToClientsBasedOnCapacity() { public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClientAdded() { final Set p1PrevTasks = Sets.newSet(0, 2); final Set p2PrevTasks = Sets.newSet(1, 3); - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -666,9 +671,9 @@ public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClien @Test public void shouldNotMoveAnyTasksWhenNewTasksAdded() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -686,10 +691,10 @@ public void shouldNotMoveAnyTasksWhenNewTasksAdded() { @Test public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -710,19 +715,19 @@ public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients @Test public void shouldAssignTasksNotPreviouslyActiveToNewClient() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3", + final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(2)), mkEntry("test-subtopology1", Sets.newSet(2)))); - final AssignmentMemberSpec newMemberSpec = createAssignmentMemberSpec("process4", + final MemberSubscription newMemberSpec = createMemberSubscription("process4", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2, 3)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -742,19 +747,19 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() { @Test public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final AssignmentMemberSpec bounce1 = createAssignmentMemberSpec("bounce1", + final MemberSubscription bounce1 = createMemberSubscription("bounce1", Map.of(), mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final AssignmentMemberSpec bounce2 = createAssignmentMemberSpec("bounce2", + final MemberSubscription bounce2 = createMemberSubscription("bounce2", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(2, 3)), mkEntry("test-subtopology1", Sets.newSet(0)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("bounce_member1", bounce1), mkEntry("bounce_member2", bounce2)); GroupAssignment result = assignor.assign( @@ -774,9 +779,9 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { @Test public void shouldAssignTasksToNewClient() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -789,10 +794,10 @@ public void shouldAssignTasksToNewClient() { @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingClients() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); - final AssignmentMemberSpec newMemberSpec = createAssignmentMemberSpec("process3"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); + final MemberSubscription newMemberSpec = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -815,10 +820,10 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingAndBouncedClients() { - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); - final AssignmentMemberSpec newMemberSpec = createAssignmentMemberSpec("newProcess"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); + final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); + final MemberSubscription newMemberSpec = createMemberSubscription("newProcess"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -845,9 +850,9 @@ public void shouldHandleLargeNumberOfTasksWithStandbyAssignment() { final int numClients = 5; final int numStandbyReplicas = 2; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createAssignmentMemberSpec("process" + i)); + members.put("member" + i, createMemberSubscription("process" + i)); } GroupAssignment result = assignor.assign( @@ -916,9 +921,9 @@ public void shouldHandleOddNumberOfClientsWithStandbyTasks() { final int numClients = 7; final int numStandbyReplicas = 1; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createAssignmentMemberSpec("process" + i)); + members.put("member" + i, createMemberSubscription("process" + i)); } GroupAssignment result = assignor.assign( @@ -967,9 +972,9 @@ public void shouldHandleHighStandbyReplicaCount() { final int numClients = 3; final int numStandbyReplicas = 5; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createAssignmentMemberSpec("process" + i)); + members.put("member" + i, createMemberSubscription("process" + i)); } GroupAssignment result = assignor.assign( @@ -1011,9 +1016,9 @@ public void shouldHandleLargeNumberOfSubtopologiesWithStandbyTasks() { subtopologies.add("subtopology-" + i); } - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createAssignmentMemberSpec("process" + i)); + members.put("member" + i, createMemberSubscription("process" + i)); } GroupAssignment result = assignor.assign( @@ -1044,8 +1049,8 @@ public void shouldHandleEdgeCaseWithSingleClientAndMultipleStandbyReplicas() { final int numTasks = 10; final int numStandbyReplicas = 3; - Map members = mkMap( - mkEntry("member1", createAssignmentMemberSpec("process1")) + Map members = mkMap( + mkEntry("member1", createMemberSubscription("process1")) ); GroupAssignment result = assignor.assign( @@ -1068,9 +1073,9 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { final int numClients = 2; final int numStandbyReplicas = 5; // More than available clients - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createAssignmentMemberSpec("process" + i)); + members.put("member" + i, createMemberSubscription("process" + i)); } GroupAssignment result = assignor.assign( @@ -1101,18 +1106,18 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssignments() { // Initial setup: Node 1 has active tasks 0,1 and standby tasks 2,3 // Node 2 has active tasks 2,3 and standby tasks 0,1 - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1)))); // Node 3 joins as new client - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); final GroupAssignment result = assignor.assign( @@ -1148,11 +1153,11 @@ public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssig @Test public void shouldRangeAssignTasksWhenScalingUp() { // Two clients, the second one is new - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", Map.of("test-subtopology1", Set.of(0, 1), "test-subtopology2", Set.of(0, 1)), Map.of()); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final Map members = mkMap( + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1196,9 +1201,9 @@ public void shouldRangeAssignTasksWhenScalingUp() { @Test public void shouldRangeAssignTasksWhenStartingEmpty() { // Two clients starting empty (no previous tasks) - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1"); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2"); - final Map members = mkMap( + final MemberSubscription memberSpec1 = createMemberSubscription("process1"); + final MemberSubscription memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1257,15 +1262,15 @@ public void shouldAssignStandbyTaskToPreviousOwnerBasedOnBelowQuotaCondition() { // Process1: active=[0], standby=[1] (previously had both active and standby tasks) // Process2: active=[1] (had the active task that process1 had as standby) // Process3: no previous tasks - final AssignmentMemberSpec memberSpec1 = createAssignmentMemberSpec("process1", + final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0))), mkMap(mkEntry("test-subtopology", Sets.newSet(1)))); - final AssignmentMemberSpec memberSpec2 = createAssignmentMemberSpec("process2", + final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(1))), Map.of()); - final AssignmentMemberSpec memberSpec3 = createAssignmentMemberSpec("process3"); + final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -1429,8 +1434,8 @@ private Map> mergeAllStandbyTasks(GroupAssignment result) { return mergeAllStandbyTasks(result, result.members().keySet().toArray(memberIds)); } - private AssignmentMemberSpec createAssignmentMemberSpec(final String processId) { - return new AssignmentMemberSpec( + private MemberSubscription createMemberSubscription(final String processId) { + return new MemberSubscription( Optional.empty(), Optional.empty(), Map.of(), @@ -1442,9 +1447,9 @@ private AssignmentMemberSpec createAssignmentMemberSpec(final String processId) Map.of()); } - private AssignmentMemberSpec createAssignmentMemberSpec(final String processId, final Map> prevActiveTasks, + private MemberSubscription createMemberSubscription(final String processId, final Map> prevActiveTasks, final Map> prevStandbyTasks) { - return new AssignmentMemberSpec( + return new MemberSubscription( Optional.empty(), Optional.empty(), prevActiveTasks, diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java index fa3324d012e64..ffc05b7ddd847 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java @@ -16,9 +16,9 @@ */ package org.apache.kafka.jmh.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; -import org.apache.kafka.coordinator.group.streams.assignor.AssignmentMemberSpec; -import org.apache.kafka.coordinator.group.streams.assignor.GroupSpec; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredInternalTopic; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; @@ -46,14 +46,14 @@ public static GroupSpec createGroupSpec( Map members, Map assignmentConfigs ) { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. for (Map.Entry memberEntry : members.entrySet()) { String memberId = memberEntry.getKey(); StreamsGroupMember member = memberEntry.getValue(); - memberSpecs.put(memberId, new AssignmentMemberSpec( + memberSpecs.put(memberId, new MemberSubscription( member.instanceId(), member.rackId(), Map.of(), diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java index 22863fef5fa2e..f950c3c8d39be 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java @@ -17,16 +17,16 @@ package org.apache.kafka.jmh.assignor; import org.apache.kafka.coordinator.common.runtime.CoordinatorMetadataImage; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; import org.apache.kafka.coordinator.group.streams.TopologyMetadata; -import org.apache.kafka.coordinator.group.streams.assignor.AssignmentMemberSpec; -import org.apache.kafka.coordinator.group.streams.assignor.GroupAssignment; -import org.apache.kafka.coordinator.group.streams.assignor.GroupSpec; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignment; import org.apache.kafka.coordinator.group.streams.assignor.StickyTaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TaskAssignor; -import org.apache.kafka.coordinator.group.streams.assignor.TopologyDescriber; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.openjdk.jmh.annotations.Benchmark; @@ -132,15 +132,15 @@ private void simulateIncrementalRebalance() { GroupAssignment initialAssignment = new StickyTaskAssignor().assign(groupSpec, topologyDescriber); Map members = initialAssignment.members(); - Map updatedMemberSpec = new HashMap<>(); + Map updatedMemberSpec = new HashMap<>(); - for (Map.Entry member : groupSpec.members().entrySet()) { + for (Map.Entry member : groupSpec.members().entrySet()) { MemberAssignment memberAssignment = members.getOrDefault( member.getKey(), new MemberAssignment(Map.of(), Map.of(), Map.of()) ); - updatedMemberSpec.put(member.getKey(), new AssignmentMemberSpec( + updatedMemberSpec.put(member.getKey(), new MemberSubscription( Optional.empty(), Optional.empty(), memberAssignment.activeTasks(), @@ -153,7 +153,7 @@ private void simulateIncrementalRebalance() { )); } - updatedMemberSpec.put("newMember", new AssignmentMemberSpec( + updatedMemberSpec.put("newMember", new MemberSubscription( Optional.empty(), Optional.empty(), Map.of(), From 1a44a8b6d826d757934890137d5b9664bad86dd4 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Wed, 8 Jul 2026 16:11:21 -0700 Subject: [PATCH 02/15] reshape the api and fix the bugs --- docs/getting-started/upgrade.md | 2 +- .../group/api/assignor/streams/GroupSpec.java | 23 +- .../assignor/streams/MemberAssignment.java | 27 +- .../streams/MemberAssignmentState.java | 62 ++++ .../assignor/streams/MemberSubscription.java | 57 ++-- .../streams/TargetAssignmentBuilder.java | 16 +- .../group/streams/assignor/GroupSpecImpl.java | 38 ++- .../assignor/MemberAssignmentImpl.java | 46 +++ .../MemberSubscriptionAndAssignmentImpl.java | 66 ++++ .../group/streams/assignor/MockAssignor.java | 10 +- .../streams/assignor/StickyTaskAssignor.java | 28 +- .../group/GroupMetadataManagerTest.java | 3 +- .../group/streams/MockTaskAssignor.java | 7 +- .../streams/TargetAssignmentBuilderTest.java | 29 +- .../streams/assignor/GroupSpecImplTest.java | 37 +- .../streams/assignor/MockAssignorTest.java | 15 +- .../assignor/StickyTaskAssignorTest.java | 315 +++++++++--------- .../StreamsAssignorBenchmarkUtils.java | 6 +- .../StreamsStickyAssignorBenchmark.java | 20 +- 19 files changed, 523 insertions(+), 284 deletions(-) create mode 100644 group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignmentImpl.java create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java diff --git a/docs/getting-started/upgrade.md b/docs/getting-started/upgrade.md index f2594cdd3e89e..42a33fd782f79 100644 --- a/docs/getting-started/upgrade.md +++ b/docs/getting-started/upgrade.md @@ -99,7 +99,7 @@ Note: Apache Kafka 4.2 only supports KRaft mode - ZooKeeper mode has been remove ### Notable changes in 4.2.0 - * The `--max-partition-memory-bytes` option in `kafka-console-producer` is deprecated and will be removed in Kafka 5.0. Please use `--batch-size` instead. +m * The `--max-partition-memory-bytes` option in `kafka-console-producer` is deprecated and will be removed in Kafka 5.0. Please use `--batch-size` instead. * Queues for Kafka ([KIP-932](https://cwiki.apache.org/confluence/x/4hA0Dw)) is production-ready in Apache Kafka 4.2. This feature introduces a new kind of group called share groups, as an alternative to consumer groups. Consumers in a share group cooperatively consume records from topics, without assigning each partition to just one consumer. Share groups also introduce per-record acknowledgement and counting of delivery attempts. Use share groups in cases where records are processed one at a time, rather than as part of an ordered stream. * The Streams Rebalance Protocol ([KIP-1071](https://cwiki.apache.org/confluence/display/KAFKA/KIP-1071%3A+Streams+Rebalance+Protocol)) is now production-ready for its core feature set. This broker-driven rebalancing system designed specifically for Kafka Streams applications provides faster, more stable rebalances and better observability. For more information about the supported feature set, usage, and migration, please refer to the [Streams developer guide](/{version}/streams/developer-guide/streams-rebalance-protocol). Due to a critical broker-side bug in the offline migration code ([KAFKA-20254](https://issues.apache.org/jira/browse/KAFKA-20254)), we recommend against doing migrations from classic to streams groups in 4.2.0. Newly created streams groups are not impacted. The fix is available in 4.2.1. * The `org.apache.kafka.common.header.internals.RecordHeader` class has been updated to be read thread-safe. See [KIP-1205](https://cwiki.apache.org/confluence/x/nYmhFg) for details. In other words, each individual `Header` object within a `ConsumerRecord`'s `headers` can now be safely read from multiple threads concurrently. diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java index 3755bf1c4ab03..a1d64f92f2e15 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.annotation.InterfaceAudience; import org.apache.kafka.common.annotation.InterfaceStability; +import java.util.Collection; import java.util.Map; /** @@ -29,13 +30,29 @@ public interface GroupSpec { /** - * @return Member metadata keyed by member Id. + * @return The member Ids of all members in the group. */ - Map members(); + Collection memberIds(); + + /** + * Gets the static subscription metadata for a given member. + * + * @param memberId The member Id. + * @return The static member subscription. + */ + MemberSubscription memberSubscription(String memberId); + + /** + * Gets the current assignment state for a given member. + * + * @param memberId The member Id. + * @return The current member assignment state. + */ + MemberAssignmentState memberAssignmentState(String memberId); /** * @return Any configurations passed to the assignor. */ - Map assignmentConfigs(); + Map configs(); } diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java index 37a9cd9301546..a2fd2b4cfa8eb 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignment.java @@ -20,29 +20,26 @@ import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Map; -import java.util.Objects; import java.util.Set; /** * The task assignment for a streams group member. * - * @param activeTasks The active tasks assigned to this member keyed by subtopologyId. - * @param standbyTasks The standby tasks assigned to this member keyed by subtopologyId. - * @param warmupTasks The warm-up tasks assigned to this member keyed by subtopologyId. + *

Only active and standby tasks are assigned by the {@link TaskAssignor}. Warm-up tasks are + * not assigned by the assignor; they are decided during reconciliation. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public record MemberAssignment(Map> activeTasks, - Map> standbyTasks, - Map> warmupTasks) { +public interface MemberAssignment { - public MemberAssignment { - Objects.requireNonNull(activeTasks); - Objects.requireNonNull(standbyTasks); - Objects.requireNonNull(warmupTasks); - } + /** + * @return The active tasks assigned to this member keyed by subtopology Id. + */ + Map> activeTasks(); + + /** + * @return The standby tasks assigned to this member keyed by subtopology Id. + */ + Map> standbyTasks(); - public static MemberAssignment empty() { - return new MemberAssignment(Map.of(), Map.of(), Map.of()); - } } diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java new file mode 100644 index 0000000000000..d3199eff129dd --- /dev/null +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.api.assignor.streams; + +import org.apache.kafka.common.annotation.InterfaceAudience; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Map; +import java.util.Set; + +/** + * Interface representing the current assignment state for a streams group member. + * + *

This is the assignment the member currently owns and is used by the {@link TaskAssignor} + * to compute a new, sticky target assignment. Note that the assignor does not assign warm-up + * tasks itself (they are decided during reconciliation), but the currently-assigned warm-up + * tasks are exposed here so that the assignor can take them into account. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface MemberAssignmentState { + + /** + * @return The current active tasks keyed by subtopology Id. + */ + Map> activeTasks(); + + /** + * @return The current standby tasks keyed by subtopology Id. + */ + Map> standbyTasks(); + + /** + * @return The current warm-up tasks keyed by subtopology Id. + */ + Map> warmupTasks(); + + /** + * @return The last received cumulative task offsets of assigned tasks or dormant tasks. + */ + Map taskOffsets(); + + /** + * @return The last received cumulative task end offsets of assigned tasks or dormant tasks. + */ + Map taskEndOffsets(); + +} diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java index 5f3d1859931c6..96eee5f6a39fd 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java @@ -19,47 +19,38 @@ import org.apache.kafka.common.annotation.InterfaceAudience; import org.apache.kafka.common.annotation.InterfaceStability; -import java.util.Collections; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** - * The assignment specification for a streams group member. + * Interface representing the static subscription metadata for a streams group member. * - * @param instanceId The instance ID if provided. - * @param rackId The rack ID if provided. - * @param activeTasks Current target active tasks - * @param standbyTasks Current target standby tasks - * @param warmupTasks Current target warm-up tasks - * @param processId The process ID. - * @param clientTags The client tags for a rack-aware assignment. - * @param taskOffsets The last received cumulative task offsets of assigned tasks or dormant tasks. + *

The subscription contains the per-member metadata that does not change during the assignment + * computation. The member's current task assignment state is exposed separately through + * {@link MemberAssignmentState}. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public record MemberSubscription(Optional instanceId, - Optional rackId, - Map> activeTasks, - Map> standbyTasks, - Map> warmupTasks, - String processId, - Map clientTags, - Map taskOffsets, - Map taskEndOffsets -) { +public interface MemberSubscription { - public MemberSubscription { - Objects.requireNonNull(instanceId); - Objects.requireNonNull(rackId); - activeTasks = Collections.unmodifiableMap(Objects.requireNonNull(activeTasks)); - standbyTasks = Collections.unmodifiableMap(Objects.requireNonNull(standbyTasks)); - warmupTasks = Collections.unmodifiableMap(Objects.requireNonNull(warmupTasks)); - Objects.requireNonNull(processId); - clientTags = Collections.unmodifiableMap(Objects.requireNonNull(clientTags)); - taskOffsets = Collections.unmodifiableMap(Objects.requireNonNull(taskOffsets)); - taskEndOffsets = Collections.unmodifiableMap(Objects.requireNonNull(taskEndOffsets)); - } + /** + * @return The instance ID if provided. + */ + Optional instanceId(); + + /** + * @return The rack ID if provided. + */ + Optional rackId(); + + /** + * @return The process ID. + */ + String processId(); + + /** + * @return The client tags for a rack-aware assignment. + */ + Map clientTags(); } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java index 2878b2b752e74..4019e403488dd 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java @@ -21,10 +21,11 @@ import org.apache.kafka.coordinator.common.runtime.CoordinatorRecord; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import java.util.ArrayList; @@ -116,12 +117,12 @@ public TargetAssignmentBuilder( this.assignmentConfigs = Objects.requireNonNull(assignmentConfigs); } - static MemberSubscription createMemberSubscription( + static MemberSubscriptionAndAssignmentImpl createMemberSubscriptionAndAssignment( StreamsGroupMember member, TasksTuple targetAssignment, MemberTaskOffsets taskOffsets ) { - return new MemberSubscription( + return new MemberSubscriptionAndAssignmentImpl( member.instanceId(), member.rackId(), targetAssignment.activeTasks(), @@ -217,10 +218,10 @@ public TargetAssignmentBuilder withTopology( * @throws TaskAssignorException if the target assignment cannot be computed. */ public TargetAssignmentResult build() throws TaskAssignorException { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. - members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberSubscription( + members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberSubscriptionAndAssignment( member, targetAssignment.getOrDefault(memberId, org.apache.kafka.coordinator.group.streams.TasksTuple.EMPTY), taskOffsets.getOrDefault(memberId, MemberTaskOffsets.EMPTY) @@ -241,7 +242,7 @@ public TargetAssignmentResult build() throws TaskAssignorException { ); } else { newGroupAssignment = new GroupAssignment( - memberSpecs.keySet().stream().collect(Collectors.toMap(x -> x, x -> MemberAssignment.empty()))); + memberSpecs.keySet().stream().collect(Collectors.toMap(x -> x, x -> (MemberAssignment) MemberAssignmentImpl.empty()))); } // Compute delta from previous to new target assignment and create the @@ -294,7 +295,8 @@ private TasksTuple newMemberAssignment( return new TasksTuple( newMemberAssignment.activeTasks(), newMemberAssignment.standbyTasks(), - newMemberAssignment.warmupTasks() + // Warm-up tasks are not assigned by the assignor; they are decided during reconciliation. + Map.of() ); } else { return TasksTuple.EMPTY; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java index 0f2cea510d96f..231ed02dd0c21 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java @@ -17,23 +17,51 @@ package org.apache.kafka.coordinator.group.streams.assignor; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import java.util.Collection; import java.util.Map; import java.util.Objects; /** * The assignment specification for a streams group. * - * @param members The member metadata keyed by member ID. - * @param assignmentConfigs Any configurations passed to the assignor. + * @param members The member metadata keyed by member Id. Each value provides both the + * {@link MemberSubscription} and the {@link MemberAssignmentState} for the member. + * @param configs Any configurations passed to the assignor. */ -public record GroupSpecImpl(Map members, - Map assignmentConfigs) implements GroupSpec { +public record GroupSpecImpl( + Map members, + Map configs +) implements GroupSpec { public GroupSpecImpl { Objects.requireNonNull(members); - Objects.requireNonNull(assignmentConfigs); + Objects.requireNonNull(configs); + } + + @Override + public Collection memberIds() { + return members.keySet(); + } + + @Override + public MemberSubscription memberSubscription(String memberId) { + MemberSubscriptionAndAssignmentImpl member = members.get(memberId); + if (member == null) { + throw new IllegalArgumentException("Member Id " + memberId + " not found."); + } + return member; + } + + @Override + public MemberAssignmentState memberAssignmentState(String memberId) { + MemberSubscriptionAndAssignmentImpl member = members.get(memberId); + if (member == null) { + throw new IllegalArgumentException("Member Id " + memberId + " not found."); + } + return member; } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignmentImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignmentImpl.java new file mode 100644 index 0000000000000..75abe38cb469b --- /dev/null +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberAssignmentImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams.assignor; + +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * The task assignment for a streams group member. + * + * @param activeTasks The active tasks assigned to this member keyed by subtopology Id. + * @param standbyTasks The standby tasks assigned to this member keyed by subtopology Id. + * The maps are not made immutable, since the server-side assignors rely on + * being able to mutate them while building new assignments. + */ +public record MemberAssignmentImpl(Map> activeTasks, + Map> standbyTasks) implements MemberAssignment { + + public MemberAssignmentImpl { + Objects.requireNonNull(activeTasks); + Objects.requireNonNull(standbyTasks); + } + + public static MemberAssignmentImpl empty() { + return new MemberAssignmentImpl(new HashMap<>(), new HashMap<>()); + } + +} diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java new file mode 100644 index 0000000000000..f2ea536d15d4c --- /dev/null +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams.assignor; + +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * Implementation of both the {@link MemberSubscription} and the {@link MemberAssignmentState} + * interfaces for a streams group member. + * + * @param instanceId The instance ID if provided. + * @param rackId The rack ID if provided. + * @param activeTasks Current active tasks. + * @param standbyTasks Current standby tasks. + * @param warmupTasks Current warm-up tasks. + * @param processId The process ID. + * @param clientTags The client tags for a rack-aware assignment. + * @param taskOffsets The last received cumulative task offsets of assigned tasks or dormant tasks. + * @param taskEndOffsets The last received cumulative task end offsets of assigned tasks or dormant tasks. + */ +public record MemberSubscriptionAndAssignmentImpl(Optional instanceId, + Optional rackId, + Map> activeTasks, + Map> standbyTasks, + Map> warmupTasks, + String processId, + Map clientTags, + Map taskOffsets, + Map taskEndOffsets +) implements MemberSubscription, MemberAssignmentState { + + public MemberSubscriptionAndAssignmentImpl { + Objects.requireNonNull(instanceId); + Objects.requireNonNull(rackId); + activeTasks = Collections.unmodifiableMap(Objects.requireNonNull(activeTasks)); + standbyTasks = Collections.unmodifiableMap(Objects.requireNonNull(standbyTasks)); + warmupTasks = Collections.unmodifiableMap(Objects.requireNonNull(warmupTasks)); + Objects.requireNonNull(processId); + clientTags = Collections.unmodifiableMap(Objects.requireNonNull(clientTags)); + taskOffsets = Collections.unmodifiableMap(Objects.requireNonNull(taskOffsets)); + taskEndOffsets = Collections.unmodifiableMap(Objects.requireNonNull(taskEndOffsets)); + } + +} diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java index c10fd943a4e1c..e933c172392d5 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignor.java @@ -19,7 +19,6 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; @@ -64,13 +63,10 @@ public GroupAssignment assign( } // Copy existing assignment and fill temporary data structures - for (Map.Entry memberEntry : groupSpec.members().entrySet()) { - final String memberId = memberEntry.getKey(); - final MemberSubscription memberSpec = memberEntry.getValue(); + for (final String memberId : groupSpec.memberIds()) { + Map> activeTasks = new HashMap<>(groupSpec.memberAssignmentState(memberId).activeTasks()); - Map> activeTasks = new HashMap<>(memberSpec.activeTasks()); - - newTargetAssignment.put(memberId, new MemberAssignment(activeTasks, new HashMap<>(), new HashMap<>())); + newTargetAssignment.put(memberId, new MemberAssignmentImpl(activeTasks, new HashMap<>())); for (Map.Entry> entry : activeTasks.entrySet()) { final String subtopologyId = entry.getKey(); final Set taskIds = entry.getValue(); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index 8c0075d85b665..d62f7fd6de474 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -20,6 +20,7 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; @@ -30,6 +31,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -75,7 +77,7 @@ private GroupAssignment doAssign(final GroupSpec groupSpec, final TopologyDescri assignStandby(statefulTasks); } - return buildGroupAssignment(groupSpec.members().keySet()); + return buildGroupAssignment(groupSpec.memberIds()); } private LinkedList taskIds(final TopologyDescriber topologyDescriber, final boolean isActive) { @@ -94,8 +96,8 @@ private LinkedList taskIds(final TopologyDescriber topologyDescriber, fi private void initialize(final GroupSpec groupSpec, final TopologyDescriber topologyDescriber) { localState = new LocalState(); localState.numStandbyReplicas = - groupSpec.assignmentConfigs().isEmpty() ? 0 - : Integer.parseInt(groupSpec.assignmentConfigs().get("num.standby.replicas")); + groupSpec.configs().isEmpty() ? 0 + : Integer.parseInt(groupSpec.configs().get("num.standby.replicas")); // Helpers for computing active tasks per member, and tasks per member localState.totalActiveTasks = 0; @@ -107,25 +109,25 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol if (topologyDescriber.isStateful(subtopology)) localState.totalTasks += numberOfPartitions * localState.numStandbyReplicas; } - localState.totalMembersWithActiveTaskCapacity = groupSpec.members().size(); - localState.totalMembersWithTaskCapacity = groupSpec.members().size(); + localState.totalMembersWithActiveTaskCapacity = groupSpec.memberIds().size(); + localState.totalMembersWithTaskCapacity = groupSpec.memberIds().size(); localState.activeTasksPerMember = computeTasksPerMember(localState.totalActiveTasks, localState.totalMembersWithActiveTaskCapacity); localState.totalTasksPerMember = computeTasksPerMember(localState.totalTasks, localState.totalMembersWithTaskCapacity); localState.processIdToState = new HashMap<>(localState.totalMembersWithActiveTaskCapacity); localState.activeTaskToPrevMember = new HashMap<>(localState.totalActiveTasks); localState.standbyTaskToPrevMember = new HashMap<>(localState.numStandbyReplicas > 0 ? (localState.totalTasks - localState.totalActiveTasks) / localState.numStandbyReplicas : 0); - for (final Map.Entry memberEntry : groupSpec.members().entrySet()) { - final String memberId = memberEntry.getKey(); - final String processId = memberEntry.getValue().processId(); + for (final String memberId : groupSpec.memberIds()) { + final MemberSubscription memberSubscription = groupSpec.memberSubscription(memberId); + final MemberAssignmentState memberAssignmentState = groupSpec.memberAssignmentState(memberId); + final String processId = memberSubscription.processId(); final Member member = new Member(processId, memberId); - final MemberSubscription memberSpec = memberEntry.getValue(); localState.processIdToState.putIfAbsent(processId, new ProcessState(processId)); localState.processIdToState.get(processId).addMember(memberId); // prev active tasks - for (final Map.Entry> entry : memberSpec.activeTasks().entrySet()) { + for (final Map.Entry> entry : memberAssignmentState.activeTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { localState.activeTaskToPrevMember.put(new TaskId(entry.getKey(), partitionNo), member); @@ -133,7 +135,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol } // prev standby tasks - for (final Map.Entry> entry : memberSpec.standbyTasks().entrySet()) { + for (final Map.Entry> entry : memberAssignmentState.standbyTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { final TaskId taskId = new TaskId(entry.getKey(), partitionNo); @@ -144,7 +146,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol } } - private GroupAssignment buildGroupAssignment(final Set members) { + private GroupAssignment buildGroupAssignment(final Collection members) { final Map memberAssignments = new HashMap<>(); final Map> activeTasksAssignments = localState.processIdToState.entrySet().stream() @@ -171,7 +173,7 @@ private GroupAssignment buildGroupAssignment(final Set members) { if (standbyTasksAssignments.containsKey(memberId)) { standByTasks.putAll(toCompactedTaskIds(standbyTasksAssignments.get(memberId))); } - memberAssignments.put(memberId, new MemberAssignment(activeTasks, standByTasks, new HashMap<>())); + memberAssignments.put(memberId, new MemberAssignmentImpl(activeTasks, standByTasks)); } return new GroupAssignment(memberAssignments); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index c704dfbad9e20..7ab51cfa77a15 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -19348,7 +19348,8 @@ public void testStreamsGroupMemberJoiningWithStaleTopology() { .build(); assignor.prepareGroupAssignment(new org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment(Map.of( - memberId, org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment.empty() + memberId, (org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment) + org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl.empty() ))); // Member joins the streams group. diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java index e3ffb14f9ac86..af8ba87fe0bdf 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MockTaskAssignor.java @@ -22,6 +22,7 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; +import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; import java.util.Map; import java.util.Map.Entry; @@ -48,8 +49,8 @@ public void prepareGroupAssignment(Map memberAssignments) { Entry::getKey, entry -> { TasksTuple tasksTuple = entry.getValue(); - return new MemberAssignment( - tasksTuple.activeTasks(), tasksTuple.standbyTasks(), tasksTuple.warmupTasks()); + return (MemberAssignment) new MemberAssignmentImpl( + tasksTuple.activeTasks(), tasksTuple.standbyTasks()); }))); } @@ -70,7 +71,7 @@ public String toString() { @Override public GroupAssignment assign(final GroupSpec groupSpec, final TopologyDescriber topologyDescriber) throws TaskAssignorException { - assignmentConfigs = groupSpec.assignmentConfigs(); + assignmentConfigs = groupSpec.configs(); return preparedGroupAssignment; } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index 333cc25119558..6d8c2bdb5f366 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -24,12 +24,13 @@ import org.apache.kafka.coordinator.common.runtime.MetadataImageBuilder; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; @@ -50,7 +51,7 @@ import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord; -import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createMemberSubscription; +import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createMemberSubscriptionAndAssignment; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -105,13 +106,13 @@ public void testCreateMemberSubscription(TaskRole taskRole) { mkTasks(barSubtopologyId, 1, 2, 3) ); - MemberSubscription memberSubscription = createMemberSubscription( + MemberSubscriptionAndAssignmentImpl memberSubscription = createMemberSubscriptionAndAssignment( member, assignment, MemberTaskOffsets.EMPTY ); - assertEquals(new MemberSubscription( + assertEquals(new MemberSubscriptionAndAssignmentImpl( Optional.of("instanceId"), Optional.of("rackId"), assignment.activeTasks(), @@ -138,7 +139,7 @@ public void testCreateMemberSubscriptionPopulatesTaskOffsets() { Map taskOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 10L); Map taskEndOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 20L); - MemberSubscription memberSubscription = createMemberSubscription( + MemberSubscriptionAndAssignmentImpl memberSubscription = createMemberSubscriptionAndAssignment( member, TasksTuple.EMPTY, new MemberTaskOffsets(taskOffsets, taskEndOffsets) @@ -167,7 +168,9 @@ public void testEmpty() { @ParameterizedTest - @EnumSource(TaskRole.class) + // Warm-up tasks are not produced by the assignor (only active and standby), so they cannot appear + // in the resulting target assignment. See MemberAssignment. + @EnumSource(value = TaskRole.class, names = {"ACTIVE", "STANDBY"}) public void testAssignmentHasNotChanged(TaskRole taskRole) { TargetAssignmentBuilderTestContext context = new TargetAssignmentBuilderTestContext( "my-group", @@ -221,7 +224,9 @@ public void testAssignmentHasNotChanged(TaskRole taskRole) { @ParameterizedTest - @EnumSource(TaskRole.class) + // Warm-up tasks are not produced by the assignor (only active and standby), so they cannot appear + // in the resulting target assignment. See MemberAssignment. + @EnumSource(value = TaskRole.class, names = {"ACTIVE", "STANDBY"}) public void testAssignmentSwapped(TaskRole taskRole) { TargetAssignmentBuilderTestContext context = new TargetAssignmentBuilderTestContext( "my-group", @@ -288,7 +293,9 @@ public void testAssignmentSwapped(TaskRole taskRole) { @ParameterizedTest - @EnumSource(TaskRole.class) + // Warm-up tasks are not produced by the assignor (only active and standby), so they cannot appear + // in the resulting target assignment. See MemberAssignment. + @EnumSource(value = TaskRole.class, names = {"ACTIVE", "STANDBY"}) public void testPartialAssignmentUpdate(TaskRole taskRole) { TargetAssignmentBuilderTestContext context = new TargetAssignmentBuilderTestContext( "my-group", @@ -424,14 +431,14 @@ public void prepareMemberAssignment( String memberId, TasksTuple assignment ) { - memberAssignments.put(memberId, new MemberAssignment(assignment.activeTasks(), assignment.standbyTasks(), assignment.warmupTasks())); + memberAssignments.put(memberId, new MemberAssignmentImpl(assignment.activeTasks(), assignment.standbyTasks())); } public org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.TargetAssignmentResult build() { // Prepare expected member specs. - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); members.forEach((memberId, member) -> - memberSpecs.put(memberId, createMemberSubscription( + memberSpecs.put(memberId, createMemberSubscriptionAndAssignment( member, targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY), MemberTaskOffsets.EMPTY diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java index c8da62c74e650..2ad6efb9d2f61 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java @@ -16,8 +16,6 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,18 +24,21 @@ import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class GroupSpecImplTest { - private Map members; + private Map members; + private MemberSubscriptionAndAssignmentImpl member; private GroupSpecImpl groupSpec; @BeforeEach void setUp() { members = new HashMap<>(); - members.put("test-member", new MemberSubscription( + member = new MemberSubscriptionAndAssignmentImpl( Optional.of("test-instance"), Optional.of("test-rack"), Map.of(), @@ -47,7 +48,8 @@ void setUp() { Map.of(), Map.of(), Map.of() - )); + ); + members.put("test-member", member); groupSpec = new GroupSpecImpl( members, @@ -56,8 +58,29 @@ void setUp() { } @Test - void testMembers() { - assertEquals(members, groupSpec.members()); + void testMemberIds() { + assertEquals(members.keySet(), groupSpec.memberIds()); + } + + @Test + void testMemberSubscription() { + assertEquals(member, groupSpec.memberSubscription("test-member")); + } + + @Test + void testMemberAssignmentState() { + assertEquals(member, groupSpec.memberAssignmentState("test-member")); + } + + @Test + void testMemberNotFound() { + assertThrows(IllegalArgumentException.class, () -> groupSpec.memberSubscription("unknown")); + assertThrows(IllegalArgumentException.class, () -> groupSpec.memberAssignmentState("unknown")); + } + + @Test + void testConfigs() { + assertTrue(groupSpec.configs().isEmpty()); } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java index 23e9aac07850b..c13ebe1a83fdb 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java @@ -18,7 +18,6 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; @@ -66,7 +65,7 @@ public void testZeroMembers() { @Test public void testDoubleAssignment() { - final MemberSubscription memberSpec1 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -78,7 +77,7 @@ public void testDoubleAssignment() { Map.of() ); - final MemberSubscription memberSpec2 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -119,7 +118,7 @@ public void testBasicScenario() { @Test public void testSingleMember() { - final MemberSubscription memberSpec = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -151,7 +150,7 @@ public void testSingleMember() { @Test public void testTwoMembersTwoSubtopologies() { - final MemberSubscription memberSpec1 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -163,7 +162,7 @@ public void testTwoMembersTwoSubtopologies() { Map.of() ); - final MemberSubscription memberSpec2 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -204,7 +203,7 @@ public void testTwoMembersTwoSubtopologies() { @Test public void testTwoMembersTwoSubtopologiesStickiness() { - final MemberSubscription memberSpec1 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( @@ -219,7 +218,7 @@ public void testTwoMembersTwoSubtopologiesStickiness() { Map.of() ); - final MemberSubscription memberSpec2 = new MemberSubscription( + final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java index 54725e9a243b2..2f03df5748024 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java @@ -19,7 +19,6 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import org.junit.jupiter.api.Test; @@ -58,9 +57,9 @@ public void testToStringReturnsName() { @Test public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCount() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); final GroupAssignment result = assignor.assign( new GroupSpecImpl( @@ -83,13 +82,13 @@ public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCoun @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { - final MemberSubscription memberSpec11 = createMemberSubscription("process1"); - final MemberSubscription memberSpec12 = createMemberSubscription("process1"); - final MemberSubscription memberSpec21 = createMemberSubscription("process2"); - final MemberSubscription memberSpec22 = createMemberSubscription("process2"); - final MemberSubscription memberSpec31 = createMemberSubscription("process3"); - final MemberSubscription memberSpec32 = createMemberSubscription("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -111,13 +110,13 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { final Map> tasks = mkMap(mkEntry("test-subtopology1", Sets.newSet(0, 1, 2)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2))); - final MemberSubscription memberSpec11 = createMemberSubscription("process1"); - final MemberSubscription memberSpec12 = createMemberSubscription("process1"); - final MemberSubscription memberSpec21 = createMemberSubscription("process2"); - final MemberSubscription memberSpec22 = createMemberSubscription("process2"); - final MemberSubscription memberSpec31 = createMemberSubscription("process3"); - final MemberSubscription memberSpec32 = createMemberSubscription("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -140,9 +139,9 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { @Test public void shouldNotMigrateActiveTaskToOtherProcess() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -160,7 +159,7 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { // flip the previous active tasks assignment around. memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); members = mkMap(mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); result = assignor.assign( @@ -180,10 +179,10 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { @Test public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -208,10 +207,10 @@ public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() @Test public void shouldAssignBasedOnCapacity() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec21 = createMemberSubscription("process2"); - final MemberSubscription memberSpec22 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -235,9 +234,9 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { final Map> activeTasks = mkMap( mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3, 4, 5)), mkEntry("test-subtopology2", Sets.newSet(0))); - final MemberSubscription memberSpec1 = createMemberSubscription("process1", activeTasks, Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", activeTasks, Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -262,12 +261,12 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { @Test public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { - MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - MemberSubscription memberSpec4 = createMemberSubscription("process4"); - MemberSubscription memberSpec5 = createMemberSubscription("process5"); - Map members = mkMap( + MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); + MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); @@ -332,10 +331,10 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { @Test public void shouldAssignTasksToClientWithPreviousStandbyTasks() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscription memberSpec3 = createMemberSubscription("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -359,9 +358,9 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() { @Test public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurrentActiveTasks() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -385,15 +384,15 @@ public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurren @Test public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscription memberSpec21 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscription memberSpec22 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2", Map.of(), Map.of()); - Map members = mkMap( + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); @@ -419,11 +418,11 @@ public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { @Test public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTaskIsAssignedTo() { final Map> tasks = mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))); - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - final MemberSubscription memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); @@ -456,10 +455,10 @@ public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTask @Test public void shouldAssignMultipleReplicasOfStandbyTask() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -476,8 +475,8 @@ public void shouldAssignMultipleReplicasOfStandbyTask() { @Test public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHavingTheTaskAssigned() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + Map members = mkMap( mkEntry("member1", memberSpec1)); final GroupAssignment result = assignor.assign( @@ -491,10 +490,10 @@ public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHaving @Test public void shouldAssignActiveAndStandbyTasks() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -510,12 +509,12 @@ public void shouldAssignActiveAndStandbyTasks() { @Test public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { - final MemberSubscription memberSpec11 = createMemberSubscription("process1"); - final MemberSubscription memberSpec12 = createMemberSubscription("process1"); - final MemberSubscription memberSpec13 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec13 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member1_3", memberSpec13), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -531,13 +530,13 @@ public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { @Test public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final MemberSubscription memberSpec4 = createMemberSubscription("process4"); - final MemberSubscription memberSpec5 = createMemberSubscription("process5"); - final MemberSubscription memberSpec6 = createMemberSubscription("process6"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); + final MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); + final MemberSubscriptionAndAssignmentImpl memberSpec6 = createMemberSubscription("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -552,13 +551,13 @@ public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { @Test public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final MemberSubscription memberSpec4 = createMemberSubscription("process4"); - final MemberSubscription memberSpec5 = createMemberSubscription("process5"); - final MemberSubscription memberSpec6 = createMemberSubscription("process6"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); + final MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); + final MemberSubscriptionAndAssignmentImpl memberSpec6 = createMemberSubscription("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -575,10 +574,10 @@ public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { @Test public void shouldAssignMoreTasksToClientWithMoreCapacity() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec21 = createMemberSubscription("process2"); - final MemberSubscription memberSpec22 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -592,11 +591,11 @@ public void shouldAssignMoreTasksToClientWithMoreCapacity() { @Test public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame() { - final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final MemberSubscription memberSpec4 = createMemberSubscription("process4"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); GroupAssignment result = assignor.assign( @@ -612,10 +611,10 @@ public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame( @Test public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { - final MemberSubscription memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -630,10 +629,10 @@ public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { @Test public void shouldRebalanceTasksToClientsBasedOnCapacity() { - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); - final MemberSubscription memberSpec31 = createMemberSubscription("process3"); - final MemberSubscription memberSpec32 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); + Map members = mkMap( mkEntry("member2", memberSpec2), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); GroupAssignment result = assignor.assign( @@ -649,10 +648,10 @@ public void shouldRebalanceTasksToClientsBasedOnCapacity() { public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClientAdded() { final Set p1PrevTasks = Sets.newSet(0, 2); final Set p2PrevTasks = Sets.newSet(1, 3); - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -671,9 +670,9 @@ public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClien @Test public void shouldNotMoveAnyTasksWhenNewTasksAdded() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -691,10 +690,10 @@ public void shouldNotMoveAnyTasksWhenNewTasksAdded() { @Test public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -715,19 +714,19 @@ public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients @Test public void shouldAssignTasksNotPreviouslyActiveToNewClient() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscription memberSpec3 = createMemberSubscription("process3", + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(2)), mkEntry("test-subtopology1", Sets.newSet(2)))); - final MemberSubscription newMemberSpec = createMemberSubscription("process4", + final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("process4", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2, 3)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -747,19 +746,19 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() { @Test public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscription bounce1 = createMemberSubscription("bounce1", + final MemberSubscriptionAndAssignmentImpl bounce1 = createMemberSubscription("bounce1", Map.of(), mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscription bounce2 = createMemberSubscription("bounce2", + final MemberSubscriptionAndAssignmentImpl bounce2 = createMemberSubscription("bounce2", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(2, 3)), mkEntry("test-subtopology1", Sets.newSet(0)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("bounce_member1", bounce1), mkEntry("bounce_member2", bounce2)); GroupAssignment result = assignor.assign( @@ -779,9 +778,9 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { @Test public void shouldAssignTasksToNewClient() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -794,10 +793,10 @@ public void shouldAssignTasksToNewClient() { @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingClients() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); - final MemberSubscription newMemberSpec = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); + final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -820,10 +819,10 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingAndBouncedClients() { - final MemberSubscription memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); - final MemberSubscription newMemberSpec = createMemberSubscription("newProcess"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); + final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("newProcess"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -850,7 +849,7 @@ public void shouldHandleLargeNumberOfTasksWithStandbyAssignment() { final int numClients = 5; final int numStandbyReplicas = 2; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { members.put("member" + i, createMemberSubscription("process" + i)); } @@ -921,7 +920,7 @@ public void shouldHandleOddNumberOfClientsWithStandbyTasks() { final int numClients = 7; final int numStandbyReplicas = 1; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { members.put("member" + i, createMemberSubscription("process" + i)); } @@ -972,7 +971,7 @@ public void shouldHandleHighStandbyReplicaCount() { final int numClients = 3; final int numStandbyReplicas = 5; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { members.put("member" + i, createMemberSubscription("process" + i)); } @@ -1016,7 +1015,7 @@ public void shouldHandleLargeNumberOfSubtopologiesWithStandbyTasks() { subtopologies.add("subtopology-" + i); } - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { members.put("member" + i, createMemberSubscription("process" + i)); } @@ -1049,7 +1048,7 @@ public void shouldHandleEdgeCaseWithSingleClientAndMultipleStandbyReplicas() { final int numTasks = 10; final int numStandbyReplicas = 3; - Map members = mkMap( + Map members = mkMap( mkEntry("member1", createMemberSubscription("process1")) ); @@ -1073,7 +1072,7 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { final int numClients = 2; final int numStandbyReplicas = 5; // More than available clients - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { members.put("member" + i, createMemberSubscription("process" + i)); } @@ -1106,18 +1105,18 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssignments() { // Initial setup: Node 1 has active tasks 0,1 and standby tasks 2,3 // Node 2 has active tasks 2,3 and standby tasks 0,1 - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1)))); // Node 3 joins as new client - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); final GroupAssignment result = assignor.assign( @@ -1153,11 +1152,11 @@ public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssig @Test public void shouldRangeAssignTasksWhenScalingUp() { // Two clients, the second one is new - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", Map.of("test-subtopology1", Set.of(0, 1), "test-subtopology2", Set.of(0, 1)), Map.of()); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1201,9 +1200,9 @@ public void shouldRangeAssignTasksWhenScalingUp() { @Test public void shouldRangeAssignTasksWhenStartingEmpty() { // Two clients starting empty (no previous tasks) - final MemberSubscription memberSpec1 = createMemberSubscription("process1"); - final MemberSubscription memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1262,15 +1261,15 @@ public void shouldAssignStandbyTaskToPreviousOwnerBasedOnBelowQuotaCondition() { // Process1: active=[0], standby=[1] (previously had both active and standby tasks) // Process2: active=[1] (had the active task that process1 had as standby) // Process3: no previous tasks - final MemberSubscription memberSpec1 = createMemberSubscription("process1", + final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0))), mkMap(mkEntry("test-subtopology", Sets.newSet(1)))); - final MemberSubscription memberSpec2 = createMemberSubscription("process2", + final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(1))), Map.of()); - final MemberSubscription memberSpec3 = createMemberSubscription("process3"); + final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -1434,8 +1433,8 @@ private Map> mergeAllStandbyTasks(GroupAssignment result) { return mergeAllStandbyTasks(result, result.members().keySet().toArray(memberIds)); } - private MemberSubscription createMemberSubscription(final String processId) { - return new MemberSubscription( + private MemberSubscriptionAndAssignmentImpl createMemberSubscription(final String processId) { + return new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -1447,9 +1446,9 @@ private MemberSubscription createMemberSubscription(final String processId) { Map.of()); } - private MemberSubscription createMemberSubscription(final String processId, final Map> prevActiveTasks, + private MemberSubscriptionAndAssignmentImpl createMemberSubscription(final String processId, final Map> prevActiveTasks, final Map> prevStandbyTasks) { - return new MemberSubscription( + return new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), prevActiveTasks, diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java index ffc05b7ddd847..84eac5d064342 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java @@ -17,9 +17,9 @@ package org.apache.kafka.jmh.assignor; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredInternalTopic; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; @@ -46,14 +46,14 @@ public static GroupSpec createGroupSpec( Map members, Map assignmentConfigs ) { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. for (Map.Entry memberEntry : members.entrySet()) { String memberId = memberEntry.getKey(); StreamsGroupMember member = memberEntry.getValue(); - memberSpecs.put(memberId, new MemberSubscription( + memberSpecs.put(memberId, new MemberSubscriptionAndAssignmentImpl( member.instanceId(), member.rackId(), Map.of(), diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java index f950c3c8d39be..9c7dc9160cd70 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java @@ -20,12 +20,13 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; import org.apache.kafka.coordinator.group.streams.TopologyMetadata; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.assignor.StickyTaskAssignor; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; @@ -132,28 +133,29 @@ private void simulateIncrementalRebalance() { GroupAssignment initialAssignment = new StickyTaskAssignor().assign(groupSpec, topologyDescriber); Map members = initialAssignment.members(); - Map updatedMemberSpec = new HashMap<>(); + Map updatedMemberSpec = new HashMap<>(); - for (Map.Entry member : groupSpec.members().entrySet()) { + for (String memberId : groupSpec.memberIds()) { MemberAssignment memberAssignment = members.getOrDefault( - member.getKey(), - new MemberAssignment(Map.of(), Map.of(), Map.of()) + memberId, + new MemberAssignmentImpl(Map.of(), Map.of()) ); - updatedMemberSpec.put(member.getKey(), new MemberSubscription( + updatedMemberSpec.put(memberId, new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), memberAssignment.activeTasks(), memberAssignment.standbyTasks(), - memberAssignment.warmupTasks(), - member.getValue().processId(), + // Warm-up tasks are not assigned by the assignor; they are decided during reconciliation. + Map.of(), + groupSpec.memberSubscription(memberId).processId(), Map.of(), Map.of(), Map.of() )); } - updatedMemberSpec.put("newMember", new MemberSubscription( + updatedMemberSpec.put("newMember", new MemberSubscriptionAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), From d88e439cd502ec9ec7d6eb17c583eb12d37ca35c Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Wed, 8 Jul 2026 16:22:23 -0700 Subject: [PATCH 03/15] clean up the extra code --- docs/getting-started/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/upgrade.md b/docs/getting-started/upgrade.md index 42a33fd782f79..f2594cdd3e89e 100644 --- a/docs/getting-started/upgrade.md +++ b/docs/getting-started/upgrade.md @@ -99,7 +99,7 @@ Note: Apache Kafka 4.2 only supports KRaft mode - ZooKeeper mode has been remove ### Notable changes in 4.2.0 -m * The `--max-partition-memory-bytes` option in `kafka-console-producer` is deprecated and will be removed in Kafka 5.0. Please use `--batch-size` instead. + * The `--max-partition-memory-bytes` option in `kafka-console-producer` is deprecated and will be removed in Kafka 5.0. Please use `--batch-size` instead. * Queues for Kafka ([KIP-932](https://cwiki.apache.org/confluence/x/4hA0Dw)) is production-ready in Apache Kafka 4.2. This feature introduces a new kind of group called share groups, as an alternative to consumer groups. Consumers in a share group cooperatively consume records from topics, without assigning each partition to just one consumer. Share groups also introduce per-record acknowledgement and counting of delivery attempts. Use share groups in cases where records are processed one at a time, rather than as part of an ordered stream. * The Streams Rebalance Protocol ([KIP-1071](https://cwiki.apache.org/confluence/display/KAFKA/KIP-1071%3A+Streams+Rebalance+Protocol)) is now production-ready for its core feature set. This broker-driven rebalancing system designed specifically for Kafka Streams applications provides faster, more stable rebalances and better observability. For more information about the supported feature set, usage, and migration, please refer to the [Streams developer guide](/{version}/streams/developer-guide/streams-rebalance-protocol). Due to a critical broker-side bug in the offline migration code ([KAFKA-20254](https://issues.apache.org/jira/browse/KAFKA-20254)), we recommend against doing migrations from classic to streams groups in 4.2.0. Newly created streams groups are not impacted. The fix is available in 4.2.1. * The `org.apache.kafka.common.header.internals.RecordHeader` class has been updated to be read thread-safe. See [KIP-1205](https://cwiki.apache.org/confluence/x/nYmhFg) for details. In other words, each individual `Header` object within a `ConsumerRecord`'s `headers` can now be safely read from multiple threads concurrently. From d7f6a4ae56a69c1ea2d3301995031cc6e1302dc3 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Thu, 9 Jul 2026 11:44:24 -0700 Subject: [PATCH 04/15] rename the api --- .../coordinator/group/api/assignor/streams/GroupSpec.java | 6 +++--- ...emberSubscription.java => MemberAssignmentMetadata.java} | 6 +++--- .../coordinator/group/streams/assignor/GroupSpecImpl.java | 6 +++--- .../assignor/MemberSubscriptionAndAssignmentImpl.java | 6 +++--- .../group/streams/assignor/StickyTaskAssignor.java | 6 +++--- .../group/streams/assignor/GroupSpecImplTest.java | 6 +++--- .../kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) rename group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/{MemberSubscription.java => MemberAssignmentMetadata.java} (87%) diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java index a1d64f92f2e15..e9f90c888d342 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/GroupSpec.java @@ -35,12 +35,12 @@ public interface GroupSpec { Collection memberIds(); /** - * Gets the static subscription metadata for a given member. + * Gets the static metadata for a given member. * * @param memberId The member Id. - * @return The static member subscription. + * @return The static member metadata. */ - MemberSubscription memberSubscription(String memberId); + MemberAssignmentMetadata memberMetadata(String memberId); /** * Gets the current assignment state for a given member. diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentMetadata.java similarity index 87% rename from group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java rename to group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentMetadata.java index 96eee5f6a39fd..c7d25b9dc7c07 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberSubscription.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentMetadata.java @@ -23,15 +23,15 @@ import java.util.Optional; /** - * Interface representing the static subscription metadata for a streams group member. + * Interface representing the static metadata for a streams group member. * - *

The subscription contains the per-member metadata that does not change during the assignment + *

The metadata contains the per-member information that does not change during the assignment * computation. The member's current task assignment state is exposed separately through * {@link MemberAssignmentState}. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public interface MemberSubscription { +public interface MemberAssignmentMetadata { /** * @return The instance ID if provided. diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java index 231ed02dd0c21..90fe8f5c0a8a8 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java @@ -17,8 +17,8 @@ package org.apache.kafka.coordinator.group.streams.assignor; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentMetadata; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import java.util.Collection; import java.util.Map; @@ -28,7 +28,7 @@ * The assignment specification for a streams group. * * @param members The member metadata keyed by member Id. Each value provides both the - * {@link MemberSubscription} and the {@link MemberAssignmentState} for the member. + * {@link MemberAssignmentMetadata} and the {@link MemberAssignmentState} for the member. * @param configs Any configurations passed to the assignor. */ public record GroupSpecImpl( @@ -47,7 +47,7 @@ public Collection memberIds() { } @Override - public MemberSubscription memberSubscription(String memberId) { + public MemberAssignmentMetadata memberMetadata(String memberId) { MemberSubscriptionAndAssignmentImpl member = members.get(memberId); if (member == null) { throw new IllegalArgumentException("Member Id " + memberId + " not found."); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java index f2ea536d15d4c..6c5138f31c939 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java @@ -16,8 +16,8 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentMetadata; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import java.util.Collections; @@ -27,7 +27,7 @@ import java.util.Set; /** - * Implementation of both the {@link MemberSubscription} and the {@link MemberAssignmentState} + * Implementation of both the {@link MemberAssignmentMetadata} and the {@link MemberAssignmentState} * interfaces for a streams group member. * * @param instanceId The instance ID if provided. @@ -49,7 +49,7 @@ public record MemberSubscriptionAndAssignmentImpl(Optional instanceId, Map clientTags, Map taskOffsets, Map taskEndOffsets -) implements MemberSubscription, MemberAssignmentState { +) implements MemberAssignmentMetadata, MemberAssignmentState { public MemberSubscriptionAndAssignmentImpl { Objects.requireNonNull(instanceId); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index d62f7fd6de474..75b43199a2c29 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -20,8 +20,8 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; +import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentMetadata; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; -import org.apache.kafka.coordinator.group.api.assignor.streams.MemberSubscription; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; @@ -118,9 +118,9 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol localState.activeTaskToPrevMember = new HashMap<>(localState.totalActiveTasks); localState.standbyTaskToPrevMember = new HashMap<>(localState.numStandbyReplicas > 0 ? (localState.totalTasks - localState.totalActiveTasks) / localState.numStandbyReplicas : 0); for (final String memberId : groupSpec.memberIds()) { - final MemberSubscription memberSubscription = groupSpec.memberSubscription(memberId); + final MemberAssignmentMetadata memberMetadata = groupSpec.memberMetadata(memberId); final MemberAssignmentState memberAssignmentState = groupSpec.memberAssignmentState(memberId); - final String processId = memberSubscription.processId(); + final String processId = memberMetadata.processId(); final Member member = new Member(processId, memberId); localState.processIdToState.putIfAbsent(processId, new ProcessState(processId)); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java index 2ad6efb9d2f61..b0082bb526b91 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java @@ -63,8 +63,8 @@ void testMemberIds() { } @Test - void testMemberSubscription() { - assertEquals(member, groupSpec.memberSubscription("test-member")); + void testMemberMetadata() { + assertEquals(member, groupSpec.memberMetadata("test-member")); } @Test @@ -74,7 +74,7 @@ void testMemberAssignmentState() { @Test void testMemberNotFound() { - assertThrows(IllegalArgumentException.class, () -> groupSpec.memberSubscription("unknown")); + assertThrows(IllegalArgumentException.class, () -> groupSpec.memberMetadata("unknown")); assertThrows(IllegalArgumentException.class, () -> groupSpec.memberAssignmentState("unknown")); } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java index 9c7dc9160cd70..d810fe2347686 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java @@ -148,7 +148,7 @@ private void simulateIncrementalRebalance() { memberAssignment.standbyTasks(), // Warm-up tasks are not assigned by the assignor; they are decided during reconciliation. Map.of(), - groupSpec.memberSubscription(memberId).processId(), + groupSpec.memberMetadata(memberId).processId(), Map.of(), Map.of(), Map.of() From 316278db04df513fb18be44679a36b36429b366b Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Thu, 16 Jul 2026 16:36:54 -0400 Subject: [PATCH 05/15] change taskid to interface --- .../group/api/assignor/streams/TaskId.java | 27 ++++++------- .../group/streams/MemberTaskOffsets.java | 3 +- .../streams/assignor/StickyTaskAssignor.java | 6 +-- .../group/streams/assignor/TaskIdImpl.java | 40 +++++++++++++++++++ .../group/GroupMetadataManagerTest.java | 8 ++-- .../group/streams/MemberTaskOffsetsTest.java | 26 ++++++------ .../group/streams/StreamsGroupTest.java | 14 +++---- .../streams/TargetAssignmentBuilderTest.java | 5 ++- .../streams/assignor/ProcessStateTest.java | 16 ++++---- 9 files changed, 92 insertions(+), 53 deletions(-) create mode 100644 group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java index 22497e378ec62..d8b52c7c43b73 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java @@ -20,32 +20,29 @@ import org.apache.kafka.common.annotation.InterfaceStability; import java.util.Comparator; -import java.util.Objects; /** - * The identifier for a task - * - * @param subtopologyId The unique identifier of the subtopology. - * @param partition The partition of the input topics this task is processing. + * The identifier for a task. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public record TaskId(String subtopologyId, int partition) implements Comparable { +public interface TaskId extends Comparable { - public TaskId { - Objects.requireNonNull(subtopologyId); - } + /** + * @return The unique identifier of the subtopology. + */ + String subtopologyId(); + + /** + * @return The partition of the input topics this task is processing. + */ + int partition(); @Override - public int compareTo(final TaskId other) { + default int compareTo(final TaskId other) { return Comparator.comparing(TaskId::subtopologyId) .thenComparingInt(TaskId::partition) .compare(this, other); } - @Override - public String toString() { - return subtopologyId + '_' + partition; - } - } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java index 552d47eaf6ae8..f3935db74a3f5 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; +import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import java.util.HashMap; import java.util.List; @@ -60,7 +61,7 @@ public MemberTaskOffsets update( private static Map toTaskIdMap(final List taskOffsets) { final Map result = new HashMap<>(taskOffsets.size()); for (final StreamsGroupHeartbeatRequestData.TaskOffset taskOffset : taskOffsets) { - final TaskId taskId = new TaskId(taskOffset.subtopologyId(), taskOffset.partition()); + final TaskId taskId = new TaskIdImpl(taskOffset.subtopologyId(), taskOffset.partition()); // The reported values come straight from the client heartbeat and the protocol does not enforce uniqueness // of (subtopologyId, partition). Reject a duplicate with a clear client error rather than silently picking // one of the values. diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index 75b43199a2c29..93556b096b9e8 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -86,7 +86,7 @@ private LinkedList taskIds(final TopologyDescriber topologyDescriber, fi if (isActive || topologyDescriber.isStateful(subtopology)) { final int numberOfPartitions = topologyDescriber.maxNumInputPartitions(subtopology); for (int i = 0; i < numberOfPartitions; i++) { - ret.add(new TaskId(subtopology, i)); + ret.add(new TaskIdImpl(subtopology, i)); } } } @@ -130,7 +130,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol for (final Map.Entry> entry : memberAssignmentState.activeTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { - localState.activeTaskToPrevMember.put(new TaskId(entry.getKey(), partitionNo), member); + localState.activeTaskToPrevMember.put(new TaskIdImpl(entry.getKey(), partitionNo), member); } } @@ -138,7 +138,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol for (final Map.Entry> entry : memberAssignmentState.standbyTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { - final TaskId taskId = new TaskId(entry.getKey(), partitionNo); + final TaskId taskId = new TaskIdImpl(entry.getKey(), partitionNo); localState.standbyTaskToPrevMember.putIfAbsent(taskId, new ArrayList<>(localState.numStandbyReplicas)); localState.standbyTaskToPrevMember.get(taskId).add(member); } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java new file mode 100644 index 0000000000000..e5e5d5c833763 --- /dev/null +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.coordinator.group.streams.assignor; + +import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; + +import java.util.Objects; + +/** + * Implementation of the {@link TaskId} interface. + * + * @param subtopologyId The unique identifier of the subtopology. + * @param partition The partition of the input topics this task is processing. + */ +public record TaskIdImpl(String subtopologyId, int partition) implements TaskId { + + public TaskIdImpl { + Objects.requireNonNull(subtopologyId); + } + + @Override + public String toString() { + return subtopologyId + '_' + partition; + } + +} diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index b16cc699552e3..4f5c27349bdb1 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -18991,8 +18991,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 10L), - Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 20L) + Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 10L), + Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 20L) ), group.taskOffsets(memberId) ); @@ -19017,8 +19017,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) assertEquals(List.of(), result.records()); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 12L), - Map.of(new org.apache.kafka.coordinator.group.api.assignor.streams.TaskId(subtopology1, 0), 20L) + Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 12L), + Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 20L) ), group.taskOffsets(memberId) ); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java index 1a889d20c7375..182c768b66f28 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; +import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.junit.jupiter.api.Test; @@ -47,11 +47,11 @@ public void shouldConvertHeartbeatRequestTaskOffsets() { ); assertEquals( - Map.of(new TaskId("sub-1", 0), 10L, new TaskId("sub-1", 1), 20L), + Map.of(new TaskIdImpl("sub-1", 0), 10L, new TaskIdImpl("sub-1", 1), 20L), result.taskOffsets() ); assertEquals( - Map.of(new TaskId("sub-1", 0), 15L, new TaskId("sub-1", 1), 25L), + Map.of(new TaskIdImpl("sub-1", 0), 15L, new TaskIdImpl("sub-1", 1), 25L), result.taskEndOffsets() ); } @@ -76,8 +76,8 @@ public void shouldRejectDuplicateTaskEntries() { @Test public void shouldRetainBothMapsWhenBothListsAreNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 10L), - Map.of(new TaskId("sub-1", 0), 15L) + Map.of(new TaskIdImpl("sub-1", 0), 10L), + Map.of(new TaskIdImpl("sub-1", 0), 15L) ); assertEquals(previous, previous.update(null, null)); @@ -87,28 +87,28 @@ public void shouldRetainBothMapsWhenBothListsAreNull() { @Test public void shouldUpdateTaskOffsetsAndRetainTaskEndOffsetsWhenEndOffsetsNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 10L), - Map.of(new TaskId("sub-1", 0), 15L) + Map.of(new TaskIdImpl("sub-1", 0), 10L), + Map.of(new TaskIdImpl("sub-1", 0), 15L) ); MemberTaskOffsets result = previous.update(List.of(taskOffset("sub-1", 0, 12L)), null); - assertEquals(Map.of(new TaskId("sub-1", 0), 12L), result.taskOffsets()); + assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 12L), result.taskOffsets()); // The end-offsets were not reported, so the previously reported values are retained. - assertEquals(Map.of(new TaskId("sub-1", 0), 15L), result.taskEndOffsets()); + assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 15L), result.taskEndOffsets()); } @Test public void shouldUpdateTaskEndOffsetsAndRetainTaskOffsetsWhenOffsetsNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 10L), - Map.of(new TaskId("sub-1", 0), 15L) + Map.of(new TaskIdImpl("sub-1", 0), 10L), + Map.of(new TaskIdImpl("sub-1", 0), 15L) ); MemberTaskOffsets result = previous.update(null, List.of(taskOffset("sub-1", 0, 18L))); // The offsets were not reported, so the previously reported values are retained. - assertEquals(Map.of(new TaskId("sub-1", 0), 10L), result.taskOffsets()); - assertEquals(Map.of(new TaskId("sub-1", 0), 18L), result.taskEndOffsets()); + assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 10L), result.taskOffsets()); + assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 18L), result.taskEndOffsets()); } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java index c947d2aed8c2a..2ea17c40d4046 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java @@ -38,7 +38,6 @@ import org.apache.kafka.coordinator.group.OffsetAndMetadata; import org.apache.kafka.coordinator.group.OffsetExpirationCondition; import org.apache.kafka.coordinator.group.OffsetExpirationConditionImpl; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.generated.StreamsGroupCurrentMemberAssignmentKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataKey; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; @@ -49,6 +48,7 @@ import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; +import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.timeline.SnapshotRegistry; @@ -120,8 +120,8 @@ public void testUpdateAndRetrieveTaskOffsets() { assertEquals(Map.of(), streamsGroup.taskOffsets()); MemberTaskOffsets offsets = new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 10L), - Map.of(new TaskId("sub-1", 0), 20L) + Map.of(new TaskIdImpl("sub-1", 0), 10L), + Map.of(new TaskIdImpl("sub-1", 0), 20L) ); streamsGroup.updateTaskOffsets("member-id", offsets); @@ -130,8 +130,8 @@ public void testUpdateAndRetrieveTaskOffsets() { // A new report replaces the previous one. MemberTaskOffsets newerOffsets = new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 15L), - Map.of(new TaskId("sub-1", 0), 25L) + Map.of(new TaskIdImpl("sub-1", 0), 15L), + Map.of(new TaskIdImpl("sub-1", 0), 25L) ); streamsGroup.updateTaskOffsets("member-id", newerOffsets); assertEquals(newerOffsets, streamsGroup.taskOffsets("member-id")); @@ -142,8 +142,8 @@ public void testRemoveMemberClearsTaskOffsets() { StreamsGroup streamsGroup = createStreamsGroup("foo"); streamsGroup.updateMember(new StreamsGroupMember.Builder("member-id").build()); streamsGroup.updateTaskOffsets("member-id", new MemberTaskOffsets( - Map.of(new TaskId("sub-1", 0), 10L), - Map.of(new TaskId("sub-1", 0), 20L) + Map.of(new TaskIdImpl("sub-1", 0), 10L), + Map.of(new TaskIdImpl("sub-1", 0), 20L) )); streamsGroup.removeMember("member-id"); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index 6d8c2bdb5f366..d44ded39467b4 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -31,6 +31,7 @@ import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; @@ -136,8 +137,8 @@ public void testCreateMemberSubscriptionPopulatesTaskOffsets() { .setClientTags(Map.of()) .build(); - Map taskOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 10L); - Map taskEndOffsets = Map.of(new TaskId(fooSubtopologyId, 0), 20L); + Map taskOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 10L); + Map taskEndOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 20L); MemberSubscriptionAndAssignmentImpl memberSubscription = createMemberSubscriptionAndAssignment( member, diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java index 953970bb8ad0b..36abd22c6da2b 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java @@ -33,7 +33,7 @@ public class ProcessStateTest { void shouldAddActiveTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskId("id1", 0); + TaskId taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, true); @@ -45,7 +45,7 @@ void shouldAddActiveTaskToMember() { void shouldAddStandbyTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskId("id1", 0); + TaskId taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, false); @@ -74,7 +74,7 @@ void shouldReturnTrueWhenCapacityIsAvailable() { void shouldReturnFalseWhenCapacityIsFull() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - processState.addTask("member1", new TaskId("id1", 0), true); + processState.addTask("member1", new TaskIdImpl("id1", 0), true); assertFalse(processState.hasCapacity()); } @@ -85,7 +85,7 @@ void shouldCompareBasedOnLoadAndCapacity() { ProcessState processState2 = new ProcessState("process2"); processState1.addMember("member1"); processState2.addMember("member2"); - processState1.addTask("member1", new TaskId("id1", 0), true); + processState1.addTask("member1", new TaskIdImpl("id1", 0), true); assertTrue(processState1.compareTo(processState2) > 0); } @@ -94,7 +94,7 @@ void shouldCompareBasedOnLoadAndCapacity() { void shouldReturnTrueIfTaskIsAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskId("id1", 0); + TaskId taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, true); assertTrue(processState.hasTask(taskId)); @@ -104,7 +104,7 @@ void shouldReturnTrueIfTaskIsAssigned() { void shouldReturnFalseIfTaskIsNotAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskId("id1", 0); + TaskId taskId = new TaskIdImpl("id1", 0); assertFalse(processState.hasTask(taskId)); } @@ -113,8 +113,8 @@ void shouldReturnFalseIfTaskIsNotAssigned() { void shouldReturnAllAssignedTasks() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId activeTaskId = new TaskId("id1", 0); - TaskId standbyTaskId = new TaskId("id1", 1); + TaskId activeTaskId = new TaskIdImpl("id1", 0); + TaskId standbyTaskId = new TaskIdImpl("id1", 1); processState.addTask("member1", activeTaskId, true); processState.addTask("member1", standbyTaskId, false); From 664ce5e42a0ccf457dd47b39f200912973d6ed3f Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Thu, 16 Jul 2026 16:46:23 -0400 Subject: [PATCH 06/15] adding comment about keep taskoffset --- .../group/api/assignor/streams/MemberAssignmentState.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java index d3199eff129dd..8f3f38ddeea28 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java @@ -29,6 +29,13 @@ * to compute a new, sticky target assignment. Note that the assignor does not assign warm-up * tasks itself (they are decided during reconciliation), but the currently-assigned warm-up * tasks are exposed here so that the assignor can take them into account. + * + *

The task-set accessors ({@link #activeTasks()}, {@link #standbyTasks()}, + * {@link #warmupTasks()}) are keyed by subtopology ID, mirroring the grouped {@code TaskIds} + * wire format ({@code subtopologyId, [partitions]}). In contrast, {@link #taskOffsets()} and + * {@link #taskEndOffsets()} are keyed by {@link TaskId}, mirroring the flat {@code TaskOffset} + * wire format ({@code subtopologyId, partition, offset}). The two shapes differ because the + * underlying wire representations differ. */ @InterfaceAudience.Public @InterfaceStability.Evolving From 034a7aa63486e916d1ca2bc4a8a0f5564422d0f3 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 00:12:52 -0400 Subject: [PATCH 07/15] nit --- .../coordinator/group/api/assignor/streams/TaskAssignor.java | 4 ++-- .../group/api/assignor/streams/TaskAssignorException.java | 4 +++- .../group/api/assignor/streams/TopologyDescriber.java | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java index 895cfcbbf3637..065862459a81c 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignor.java @@ -27,7 +27,7 @@ public interface TaskAssignor { /** - * Unique name for this assignor. + * Unique name for this assignor. Used in configuration to select this assignor. */ String name(); @@ -38,7 +38,7 @@ public interface TaskAssignor { * @param topologyDescriber The task metadata describer. * @return The new assignment for the group. * - * @throws TaskAssignorException For empty groups + * @throws TaskAssignorException If the assignment cannot be computed. */ GroupAssignment assign( GroupSpec groupSpec, diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java index 17a8a2974fdb3..6b8f279927134 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskAssignorException.java @@ -21,7 +21,9 @@ import org.apache.kafka.common.errors.ApiException; /** - * Exception thrown by {@link TaskAssignor#assign(GroupSpec, TopologyDescriber)}}. The exception is only used internally. + * Exception thrown by {@link TaskAssignor#assign(GroupSpec, TopologyDescriber)} when the group's tasks + * cannot be assigned. Custom {@link TaskAssignor} implementations should throw this exception to signal + * an assignment failure. */ @InterfaceAudience.Public @InterfaceStability.Evolving diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java index 5865779d6a2de..12f9af20929f5 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TopologyDescriber.java @@ -30,9 +30,9 @@ public interface TopologyDescriber { /** - * Map of topic names to topic metadata. + * The IDs of all subtopologies in the group's topology. * - * @return The list of subtopologies IDs. + * @return The list of subtopology IDs. */ List subtopologies(); From 5fedce1651db47553699cb175ea90e18b59867ed Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 00:28:08 -0400 Subject: [PATCH 08/15] mover compareto to taskidimpl --- .../coordinator/group/api/assignor/streams/TaskId.java | 9 --------- .../coordinator/group/streams/assignor/TaskIdImpl.java | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java index d8b52c7c43b73..054d47196eb69 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java @@ -19,8 +19,6 @@ import org.apache.kafka.common.annotation.InterfaceAudience; import org.apache.kafka.common.annotation.InterfaceStability; -import java.util.Comparator; - /** * The identifier for a task. */ @@ -38,11 +36,4 @@ public interface TaskId extends Comparable { */ int partition(); - @Override - default int compareTo(final TaskId other) { - return Comparator.comparing(TaskId::subtopologyId) - .thenComparingInt(TaskId::partition) - .compare(this, other); - } - } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java index e5e5d5c833763..7bbf22700d962 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java @@ -18,6 +18,7 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; +import java.util.Comparator; import java.util.Objects; /** @@ -32,6 +33,13 @@ public record TaskIdImpl(String subtopologyId, int partition) implements TaskId Objects.requireNonNull(subtopologyId); } + @Override + public int compareTo(final TaskId other) { + return Comparator.comparing(TaskId::subtopologyId) + .thenComparingInt(TaskId::partition) + .compare(this, other); + } + @Override public String toString() { return subtopologyId + '_' + partition; From 9c6ac46d14d04f483844d888dae03e7ed3abbf5d Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 13:55:24 -0400 Subject: [PATCH 09/15] rename some param --- .../streams/TargetAssignmentBuilder.java | 10 +- .../group/streams/assignor/GroupSpecImpl.java | 6 +- ...a => MemberMetadataAndAssignmentImpl.java} | 4 +- .../streams/TargetAssignmentBuilderTest.java | 24 +- .../streams/assignor/GroupSpecImplTest.java | 6 +- .../streams/assignor/MockAssignorTest.java | 14 +- .../assignor/StickyTaskAssignorTest.java | 338 +++++++++--------- .../StreamsAssignorBenchmarkUtils.java | 6 +- .../StreamsStickyAssignorBenchmark.java | 8 +- 9 files changed, 208 insertions(+), 208 deletions(-) rename group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/{MemberSubscriptionAndAssignmentImpl.java => MemberMetadataAndAssignmentImpl.java} (96%) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java index 4019e403488dd..8dca62d4ffb25 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java @@ -25,7 +25,7 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberMetadataAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import java.util.ArrayList; @@ -117,12 +117,12 @@ public TargetAssignmentBuilder( this.assignmentConfigs = Objects.requireNonNull(assignmentConfigs); } - static MemberSubscriptionAndAssignmentImpl createMemberSubscriptionAndAssignment( + static MemberMetadataAndAssignmentImpl createMemberMetadataAndAssignment( StreamsGroupMember member, TasksTuple targetAssignment, MemberTaskOffsets taskOffsets ) { - return new MemberSubscriptionAndAssignmentImpl( + return new MemberMetadataAndAssignmentImpl( member.instanceId(), member.rackId(), targetAssignment.activeTasks(), @@ -218,10 +218,10 @@ public TargetAssignmentBuilder withTopology( * @throws TaskAssignorException if the target assignment cannot be computed. */ public TargetAssignmentResult build() throws TaskAssignorException { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. - members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberSubscriptionAndAssignment( + members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberMetadataAndAssignment( member, targetAssignment.getOrDefault(memberId, org.apache.kafka.coordinator.group.streams.TasksTuple.EMPTY), taskOffsets.getOrDefault(memberId, MemberTaskOffsets.EMPTY) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java index 90fe8f5c0a8a8..9f27b4de41392 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java @@ -32,7 +32,7 @@ * @param configs Any configurations passed to the assignor. */ public record GroupSpecImpl( - Map members, + Map members, Map configs ) implements GroupSpec { @@ -48,7 +48,7 @@ public Collection memberIds() { @Override public MemberAssignmentMetadata memberMetadata(String memberId) { - MemberSubscriptionAndAssignmentImpl member = members.get(memberId); + MemberMetadataAndAssignmentImpl member = members.get(memberId); if (member == null) { throw new IllegalArgumentException("Member Id " + memberId + " not found."); } @@ -57,7 +57,7 @@ public MemberAssignmentMetadata memberMetadata(String memberId) { @Override public MemberAssignmentState memberAssignmentState(String memberId) { - MemberSubscriptionAndAssignmentImpl member = members.get(memberId); + MemberMetadataAndAssignmentImpl member = members.get(memberId); if (member == null) { throw new IllegalArgumentException("Member Id " + memberId + " not found."); } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java similarity index 96% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java rename to group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java index 6c5138f31c939..196a205c71419 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberSubscriptionAndAssignmentImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java @@ -40,7 +40,7 @@ * @param taskOffsets The last received cumulative task offsets of assigned tasks or dormant tasks. * @param taskEndOffsets The last received cumulative task end offsets of assigned tasks or dormant tasks. */ -public record MemberSubscriptionAndAssignmentImpl(Optional instanceId, +public record MemberMetadataAndAssignmentImpl(Optional instanceId, Optional rackId, Map> activeTasks, Map> standbyTasks, @@ -51,7 +51,7 @@ public record MemberSubscriptionAndAssignmentImpl(Optional instanceId, Map taskEndOffsets ) implements MemberAssignmentMetadata, MemberAssignmentState { - public MemberSubscriptionAndAssignmentImpl { + public MemberMetadataAndAssignmentImpl { Objects.requireNonNull(instanceId); Objects.requireNonNull(rackId); activeTasks = Collections.unmodifiableMap(Objects.requireNonNull(activeTasks)); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index d44ded39467b4..bad35e8eff0a7 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -30,7 +30,7 @@ import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberMetadataAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; @@ -52,7 +52,7 @@ import static org.apache.kafka.coordinator.group.Assertions.assertUnorderedRecordsEquals; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentMetadataRecord; import static org.apache.kafka.coordinator.group.streams.StreamsCoordinatorRecordHelpers.newStreamsGroupTargetAssignmentRecord; -import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createMemberSubscriptionAndAssignment; +import static org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.createMemberMetadataAndAssignment; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasks; import static org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.mkTasksTuple; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -90,7 +90,7 @@ public void testBuildEmptyAssignmentWhenTopologyNotReady() { @ParameterizedTest @EnumSource(TaskRole.class) - public void testCreateMemberSubscription(TaskRole taskRole) { + public void testCreateMemberMetadataAndAssignment(TaskRole taskRole) { String fooSubtopologyId = Uuid.randomUuid().toString(); String barSubtopologyId = Uuid.randomUuid().toString(); @@ -107,13 +107,13 @@ public void testCreateMemberSubscription(TaskRole taskRole) { mkTasks(barSubtopologyId, 1, 2, 3) ); - MemberSubscriptionAndAssignmentImpl memberSubscription = createMemberSubscriptionAndAssignment( + MemberMetadataAndAssignmentImpl memberMetadata = createMemberMetadataAndAssignment( member, assignment, MemberTaskOffsets.EMPTY ); - assertEquals(new MemberSubscriptionAndAssignmentImpl( + assertEquals(new MemberMetadataAndAssignmentImpl( Optional.of("instanceId"), Optional.of("rackId"), assignment.activeTasks(), @@ -123,11 +123,11 @@ public void testCreateMemberSubscription(TaskRole taskRole) { clientTags, Map.of(), Map.of() - ), memberSubscription); + ), memberMetadata); } @Test - public void testCreateMemberSubscriptionPopulatesTaskOffsets() { + public void testCreateMemberMetadataAndAssignmentPopulatesTaskOffsets() { String fooSubtopologyId = Uuid.randomUuid().toString(); StreamsGroupMember member = new StreamsGroupMember.Builder("member-id") @@ -140,14 +140,14 @@ public void testCreateMemberSubscriptionPopulatesTaskOffsets() { Map taskOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 10L); Map taskEndOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 20L); - MemberSubscriptionAndAssignmentImpl memberSubscription = createMemberSubscriptionAndAssignment( + MemberMetadataAndAssignmentImpl memberMetadata = createMemberMetadataAndAssignment( member, TasksTuple.EMPTY, new MemberTaskOffsets(taskOffsets, taskEndOffsets) ); - assertEquals(taskOffsets, memberSubscription.taskOffsets()); - assertEquals(taskEndOffsets, memberSubscription.taskEndOffsets()); + assertEquals(taskOffsets, memberMetadata.taskOffsets()); + assertEquals(taskEndOffsets, memberMetadata.taskEndOffsets()); } @Test @@ -437,9 +437,9 @@ public void prepareMemberAssignment( public org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.TargetAssignmentResult build() { // Prepare expected member specs. - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); members.forEach((memberId, member) -> - memberSpecs.put(memberId, createMemberSubscriptionAndAssignment( + memberSpecs.put(memberId, createMemberMetadataAndAssignment( member, targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY), MemberTaskOffsets.EMPTY diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java index b0082bb526b91..1e6e951202418 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImplTest.java @@ -30,15 +30,15 @@ public class GroupSpecImplTest { - private Map members; - private MemberSubscriptionAndAssignmentImpl member; + private Map members; + private MemberMetadataAndAssignmentImpl member; private GroupSpecImpl groupSpec; @BeforeEach void setUp() { members = new HashMap<>(); - member = new MemberSubscriptionAndAssignmentImpl( + member = new MemberMetadataAndAssignmentImpl( Optional.of("test-instance"), Optional.of("test-rack"), Map.of(), diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java index c13ebe1a83fdb..3409dc9645d7a 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java @@ -65,7 +65,7 @@ public void testZeroMembers() { @Test public void testDoubleAssignment() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -77,7 +77,7 @@ public void testDoubleAssignment() { Map.of() ); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -118,7 +118,7 @@ public void testBasicScenario() { @Test public void testSingleMember() { - final MemberSubscriptionAndAssignmentImpl memberSpec = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -150,7 +150,7 @@ public void testSingleMember() { @Test public void testTwoMembersTwoSubtopologies() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -162,7 +162,7 @@ public void testTwoMembersTwoSubtopologies() { Map.of() ); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -203,7 +203,7 @@ public void testTwoMembersTwoSubtopologies() { @Test public void testTwoMembersTwoSubtopologiesStickiness() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( @@ -218,7 +218,7 @@ public void testTwoMembersTwoSubtopologiesStickiness() { Map.of() ); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = new MemberSubscriptionAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java index 2f03df5748024..cdd8addc41f9c 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java @@ -57,9 +57,9 @@ public void testToStringReturnsName() { @Test public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCount() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); final GroupAssignment result = assignor.assign( new GroupSpecImpl( @@ -82,13 +82,13 @@ public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCoun @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); - final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -110,13 +110,13 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { final Map> tasks = mkMap(mkEntry("test-subtopology1", Sets.newSet(0, 1, 2)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2))); - final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); - final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), + final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); + final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); @@ -139,9 +139,9 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { @Test public void shouldNotMigrateActiveTaskToOtherProcess() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -158,8 +158,8 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { testMember1.activeTasks().get("test-subtopology").size() + testMember2.activeTasks().get("test-subtopology").size()); // flip the previous active tasks assignment around. - memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); members = mkMap(mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); result = assignor.assign( @@ -179,10 +179,10 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { @Test public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -207,10 +207,10 @@ public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() @Test public void shouldAssignBasedOnCapacity() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -234,9 +234,9 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { final Map> activeTasks = mkMap( mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3, 4, 5)), mkEntry("test-subtopology2", Sets.newSet(0))); - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", activeTasks, Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", activeTasks, Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -261,12 +261,12 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { @Test public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { - MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); - MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); - Map members = mkMap( + MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); + MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); @@ -295,11 +295,11 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { assertNull(testMember5.activeTasks().get("test-subtopology")); // change up the assignment and make sure it is still sticky - memberSpec1 = createMemberSubscription("process1"); - memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - memberSpec3 = createMemberSubscription("process3"); - memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - memberSpec5 = createMemberSubscription("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + memberSpec1 = createMemberMetadata("process1"); + memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + memberSpec3 = createMemberMetadata("process3"); + memberSpec4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + memberSpec5 = createMemberMetadata("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); @@ -331,10 +331,10 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { @Test public void shouldAssignTasksToClientWithPreviousStandbyTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -358,9 +358,9 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() { @Test public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurrentActiveTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -384,15 +384,15 @@ public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurren @Test public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2", Map.of(), Map.of()); - Map members = mkMap( + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); @@ -418,11 +418,11 @@ public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { @Test public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTaskIsAssignedTo() { final Map> tasks = mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))); - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); @@ -455,10 +455,10 @@ public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTask @Test public void shouldAssignMultipleReplicasOfStandbyTask() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -475,8 +475,8 @@ public void shouldAssignMultipleReplicasOfStandbyTask() { @Test public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHavingTheTaskAssigned() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + Map members = mkMap( mkEntry("member1", memberSpec1)); final GroupAssignment result = assignor.assign( @@ -490,10 +490,10 @@ public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHaving @Test public void shouldAssignActiveAndStandbyTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -509,12 +509,12 @@ public void shouldAssignActiveAndStandbyTasks() { @Test public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { - final MemberSubscriptionAndAssignmentImpl memberSpec11 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec12 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec13 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec13 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + Map members = mkMap( mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member1_3", memberSpec13), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -530,13 +530,13 @@ public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { @Test public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); - final MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); - final MemberSubscriptionAndAssignmentImpl memberSpec6 = createMemberSubscription("process6"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); + final MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); + final MemberMetadataAndAssignmentImpl memberSpec6 = createMemberMetadata("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -551,13 +551,13 @@ public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { @Test public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); - final MemberSubscriptionAndAssignmentImpl memberSpec5 = createMemberSubscription("process5"); - final MemberSubscriptionAndAssignmentImpl memberSpec6 = createMemberSubscription("process6"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); + final MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); + final MemberMetadataAndAssignmentImpl memberSpec6 = createMemberMetadata("process6"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); @@ -574,10 +574,10 @@ public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { @Test public void shouldAssignMoreTasksToClientWithMoreCapacity() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec21 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec22 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); GroupAssignment result = assignor.assign( @@ -591,11 +591,11 @@ public void shouldAssignMoreTasksToClientWithMoreCapacity() { @Test public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame() { - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final MemberSubscriptionAndAssignmentImpl memberSpec4 = createMemberSubscription("process4"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); GroupAssignment result = assignor.assign( @@ -611,10 +611,10 @@ public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame( @Test public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -629,10 +629,10 @@ public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { @Test public void shouldRebalanceTasksToClientsBasedOnCapacity() { - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec31 = createMemberSubscription("process3"); - final MemberSubscriptionAndAssignmentImpl memberSpec32 = createMemberSubscription("process3"); - Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); + Map members = mkMap( mkEntry("member2", memberSpec2), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); GroupAssignment result = assignor.assign( @@ -648,10 +648,10 @@ public void shouldRebalanceTasksToClientsBasedOnCapacity() { public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClientAdded() { final Set p1PrevTasks = Sets.newSet(0, 2); final Set p2PrevTasks = Sets.newSet(1, 3); - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -670,9 +670,9 @@ public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClien @Test public void shouldNotMoveAnyTasksWhenNewTasksAdded() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -690,10 +690,10 @@ public void shouldNotMoveAnyTasksWhenNewTasksAdded() { @Test public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); GroupAssignment result = assignor.assign( @@ -714,19 +714,19 @@ public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients @Test public void shouldAssignTasksNotPreviouslyActiveToNewClient() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3", + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(2)), mkEntry("test-subtopology1", Sets.newSet(2)))); - final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("process4", + final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("process4", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2, 3)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -746,19 +746,19 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() { @Test public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscriptionAndAssignmentImpl bounce1 = createMemberSubscription("bounce1", + final MemberMetadataAndAssignmentImpl bounce1 = createMemberMetadata("bounce1", Map.of(), mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberSubscriptionAndAssignmentImpl bounce2 = createMemberSubscription("bounce2", + final MemberMetadataAndAssignmentImpl bounce2 = createMemberMetadata("bounce2", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(2, 3)), mkEntry("test-subtopology1", Sets.newSet(0)))); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("bounce_member1", bounce1), mkEntry("bounce_member2", bounce2)); GroupAssignment result = assignor.assign( @@ -778,9 +778,9 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { @Test public void shouldAssignTasksToNewClient() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); GroupAssignment result = assignor.assign( @@ -793,10 +793,10 @@ public void shouldAssignTasksToNewClient() { @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingClients() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); - final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("process3"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); + final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("process3"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -819,10 +819,10 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingAndBouncedClients() { - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); - final MemberSubscriptionAndAssignmentImpl newMemberSpec = createMemberSubscription("newProcess"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); + final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("newProcess"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( @@ -849,9 +849,9 @@ public void shouldHandleLargeNumberOfTasksWithStandbyAssignment() { final int numClients = 5; final int numStandbyReplicas = 2; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createMemberSubscription("process" + i)); + members.put("member" + i, createMemberMetadata("process" + i)); } GroupAssignment result = assignor.assign( @@ -920,9 +920,9 @@ public void shouldHandleOddNumberOfClientsWithStandbyTasks() { final int numClients = 7; final int numStandbyReplicas = 1; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createMemberSubscription("process" + i)); + members.put("member" + i, createMemberMetadata("process" + i)); } GroupAssignment result = assignor.assign( @@ -971,9 +971,9 @@ public void shouldHandleHighStandbyReplicaCount() { final int numClients = 3; final int numStandbyReplicas = 5; - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createMemberSubscription("process" + i)); + members.put("member" + i, createMemberMetadata("process" + i)); } GroupAssignment result = assignor.assign( @@ -1015,9 +1015,9 @@ public void shouldHandleLargeNumberOfSubtopologiesWithStandbyTasks() { subtopologies.add("subtopology-" + i); } - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createMemberSubscription("process" + i)); + members.put("member" + i, createMemberMetadata("process" + i)); } GroupAssignment result = assignor.assign( @@ -1048,8 +1048,8 @@ public void shouldHandleEdgeCaseWithSingleClientAndMultipleStandbyReplicas() { final int numTasks = 10; final int numStandbyReplicas = 3; - Map members = mkMap( - mkEntry("member1", createMemberSubscription("process1")) + Map members = mkMap( + mkEntry("member1", createMemberMetadata("process1")) ); GroupAssignment result = assignor.assign( @@ -1072,9 +1072,9 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { final int numClients = 2; final int numStandbyReplicas = 5; // More than available clients - Map members = new HashMap<>(); + Map members = new HashMap<>(); for (int i = 0; i < numClients; i++) { - members.put("member" + i, createMemberSubscription("process" + i)); + members.put("member" + i, createMemberMetadata("process" + i)); } GroupAssignment result = assignor.assign( @@ -1105,18 +1105,18 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssignments() { // Initial setup: Node 1 has active tasks 0,1 and standby tasks 2,3 // Node 2 has active tasks 2,3 and standby tasks 0,1 - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1)))); // Node 3 joins as new client - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); final GroupAssignment result = assignor.assign( @@ -1152,11 +1152,11 @@ public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssig @Test public void shouldRangeAssignTasksWhenScalingUp() { // Two clients, the second one is new - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", Map.of("test-subtopology1", Set.of(0, 1), "test-subtopology2", Set.of(0, 1)), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1200,9 +1200,9 @@ public void shouldRangeAssignTasksWhenScalingUp() { @Test public void shouldRangeAssignTasksWhenStartingEmpty() { // Two clients starting empty (no previous tasks) - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1"); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2"); - final Map members = mkMap( + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled @@ -1261,15 +1261,15 @@ public void shouldAssignStandbyTaskToPreviousOwnerBasedOnBelowQuotaCondition() { // Process1: active=[0], standby=[1] (previously had both active and standby tasks) // Process2: active=[1] (had the active task that process1 had as standby) // Process3: no previous tasks - final MemberSubscriptionAndAssignmentImpl memberSpec1 = createMemberSubscription("process1", + final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0))), mkMap(mkEntry("test-subtopology", Sets.newSet(1)))); - final MemberSubscriptionAndAssignmentImpl memberSpec2 = createMemberSubscription("process2", + final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(1))), Map.of()); - final MemberSubscriptionAndAssignmentImpl memberSpec3 = createMemberSubscription("process3"); + final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); - final Map members = mkMap( + final Map members = mkMap( mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); @@ -1433,8 +1433,8 @@ private Map> mergeAllStandbyTasks(GroupAssignment result) { return mergeAllStandbyTasks(result, result.members().keySet().toArray(memberIds)); } - private MemberSubscriptionAndAssignmentImpl createMemberSubscription(final String processId) { - return new MemberSubscriptionAndAssignmentImpl( + private MemberMetadataAndAssignmentImpl createMemberMetadata(final String processId) { + return new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -1446,9 +1446,9 @@ private MemberSubscriptionAndAssignmentImpl createMemberSubscription(final Strin Map.of()); } - private MemberSubscriptionAndAssignmentImpl createMemberSubscription(final String processId, final Map> prevActiveTasks, + private MemberMetadataAndAssignmentImpl createMemberMetadata(final String processId, final Map> prevActiveTasks, final Map> prevStandbyTasks) { - return new MemberSubscriptionAndAssignmentImpl( + return new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), prevActiveTasks, diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java index 84eac5d064342..30bf7882c5230 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsAssignorBenchmarkUtils.java @@ -19,7 +19,7 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupSpec; import org.apache.kafka.coordinator.group.streams.StreamsGroupMember; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberMetadataAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredInternalTopic; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; @@ -46,14 +46,14 @@ public static GroupSpec createGroupSpec( Map members, Map assignmentConfigs ) { - Map memberSpecs = new HashMap<>(); + Map memberSpecs = new HashMap<>(); // Prepare the member spec for all members. for (Map.Entry memberEntry : members.entrySet()) { String memberId = memberEntry.getKey(); StreamsGroupMember member = memberEntry.getValue(); - memberSpecs.put(memberId, new MemberSubscriptionAndAssignmentImpl( + memberSpecs.put(memberId, new MemberMetadataAndAssignmentImpl( member.instanceId(), member.rackId(), Map.of(), diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java index d810fe2347686..8e390aad85c68 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/assignor/StreamsStickyAssignorBenchmark.java @@ -26,7 +26,7 @@ import org.apache.kafka.coordinator.group.streams.TopologyMetadata; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; -import org.apache.kafka.coordinator.group.streams.assignor.MemberSubscriptionAndAssignmentImpl; +import org.apache.kafka.coordinator.group.streams.assignor.MemberMetadataAndAssignmentImpl; import org.apache.kafka.coordinator.group.streams.assignor.StickyTaskAssignor; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; @@ -133,7 +133,7 @@ private void simulateIncrementalRebalance() { GroupAssignment initialAssignment = new StickyTaskAssignor().assign(groupSpec, topologyDescriber); Map members = initialAssignment.members(); - Map updatedMemberSpec = new HashMap<>(); + Map updatedMemberSpec = new HashMap<>(); for (String memberId : groupSpec.memberIds()) { MemberAssignment memberAssignment = members.getOrDefault( @@ -141,7 +141,7 @@ private void simulateIncrementalRebalance() { new MemberAssignmentImpl(Map.of(), Map.of()) ); - updatedMemberSpec.put(memberId, new MemberSubscriptionAndAssignmentImpl( + updatedMemberSpec.put(memberId, new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), memberAssignment.activeTasks(), @@ -155,7 +155,7 @@ private void simulateIncrementalRebalance() { )); } - updatedMemberSpec.put("newMember", new MemberSubscriptionAndAssignmentImpl( + updatedMemberSpec.put("newMember", new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), From 488e7cbb2b71078377e8c9f213d9e50a2f1c7c29 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 15:00:48 -0400 Subject: [PATCH 10/15] clean requirement logic --- .../group/streams/assignor/GroupSpecImpl.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java index 9f27b4de41392..1969293333cfd 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/GroupSpecImpl.java @@ -48,15 +48,15 @@ public Collection memberIds() { @Override public MemberAssignmentMetadata memberMetadata(String memberId) { - MemberMetadataAndAssignmentImpl member = members.get(memberId); - if (member == null) { - throw new IllegalArgumentException("Member Id " + memberId + " not found."); - } - return member; + return requireMember(memberId); } @Override public MemberAssignmentState memberAssignmentState(String memberId) { + return requireMember(memberId); + } + + private MemberMetadataAndAssignmentImpl requireMember(String memberId) { MemberMetadataAndAssignmentImpl member = members.get(memberId); if (member == null) { throw new IllegalArgumentException("Member Id " + memberId + " not found."); From 91f106e8d27fc27d509950531494f1da054dd8d8 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 15:51:22 -0400 Subject: [PATCH 11/15] revert taskid --- .../streams/MemberAssignmentState.java | 16 +++--- .../group/api/assignor/streams/TaskId.java | 39 ------------- .../group/streams/MemberTaskOffsets.java | 20 +++---- .../MemberMetadataAndAssignmentImpl.java | 5 +- .../group/streams/assignor/ProcessState.java | 25 ++++----- .../streams/assignor/StickyTaskAssignor.java | 55 +++++++++---------- .../group/streams/assignor/TaskIdImpl.java | 14 ++--- .../group/GroupMetadataManagerTest.java | 8 +-- .../group/streams/MemberTaskOffsetsTest.java | 25 ++++----- .../group/streams/StreamsGroupTest.java | 13 ++--- .../streams/TargetAssignmentBuilderTest.java | 6 +- .../streams/assignor/ProcessStateTest.java | 15 +++-- 12 files changed, 96 insertions(+), 145 deletions(-) delete mode 100644 group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java index 8f3f38ddeea28..e35719d41a91f 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java @@ -30,12 +30,10 @@ * tasks itself (they are decided during reconciliation), but the currently-assigned warm-up * tasks are exposed here so that the assignor can take them into account. * - *

The task-set accessors ({@link #activeTasks()}, {@link #standbyTasks()}, - * {@link #warmupTasks()}) are keyed by subtopology ID, mirroring the grouped {@code TaskIds} - * wire format ({@code subtopologyId, [partitions]}). In contrast, {@link #taskOffsets()} and - * {@link #taskEndOffsets()} are keyed by {@link TaskId}, mirroring the flat {@code TaskOffset} - * wire format ({@code subtopologyId, partition, offset}). The two shapes differ because the - * underlying wire representations differ. + *

All accessors are keyed by subtopology ID. The task-set accessors ({@link #activeTasks()}, + * {@link #standbyTasks()}, {@link #warmupTasks()}) map each subtopology ID to its set of + * partitions, while {@link #taskOffsets()} and {@link #taskEndOffsets()} map each subtopology ID + * to a map from partition to offset. */ @InterfaceAudience.Public @InterfaceStability.Evolving @@ -58,12 +56,14 @@ public interface MemberAssignmentState { /** * @return The last received cumulative task offsets of assigned tasks or dormant tasks. + * The outer map is keyed by subtopology ID and the inner map by partition. */ - Map taskOffsets(); + Map> taskOffsets(); /** * @return The last received cumulative task end offsets of assigned tasks or dormant tasks. + * The outer map is keyed by subtopology ID and the inner map by partition. */ - Map taskEndOffsets(); + Map> taskEndOffsets(); } diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java deleted file mode 100644 index 054d47196eb69..0000000000000 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/TaskId.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.coordinator.group.api.assignor.streams; - -import org.apache.kafka.common.annotation.InterfaceAudience; -import org.apache.kafka.common.annotation.InterfaceStability; - -/** - * The identifier for a task. - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public interface TaskId extends Comparable { - - /** - * @return The unique identifier of the subtopology. - */ - String subtopologyId(); - - /** - * @return The partition of the input topics this task is processing. - */ - int partition(); - -} diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java index f3935db74a3f5..82ebe9d096169 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsets.java @@ -18,8 +18,6 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; -import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import java.util.HashMap; import java.util.List; @@ -33,10 +31,10 @@ * constantly changing"); they are held in memory on the group coordinator and re-reported by the member on the * task-offset interval. * - * @param taskOffsets Cumulative changelog offsets per task. - * @param taskEndOffsets Cumulative changelog end-offsets per task. + * @param taskOffsets Cumulative changelog offsets, keyed by subtopology ID then partition. + * @param taskEndOffsets Cumulative changelog end-offsets, keyed by subtopology ID then partition. */ -public record MemberTaskOffsets(Map taskOffsets, Map taskEndOffsets) { +public record MemberTaskOffsets(Map> taskOffsets, Map> taskEndOffsets) { public static final MemberTaskOffsets EMPTY = new MemberTaskOffsets(Map.of(), Map.of()); @@ -53,19 +51,19 @@ public MemberTaskOffsets update( final List reportedTaskEndOffsets ) { return new MemberTaskOffsets( - reportedTaskOffsets == null ? taskOffsets : toTaskIdMap(reportedTaskOffsets), - reportedTaskEndOffsets == null ? taskEndOffsets : toTaskIdMap(reportedTaskEndOffsets) + reportedTaskOffsets == null ? taskOffsets : toNestedMap(reportedTaskOffsets), + reportedTaskEndOffsets == null ? taskEndOffsets : toNestedMap(reportedTaskEndOffsets) ); } - private static Map toTaskIdMap(final List taskOffsets) { - final Map result = new HashMap<>(taskOffsets.size()); + private static Map> toNestedMap(final List taskOffsets) { + final Map> result = new HashMap<>(); for (final StreamsGroupHeartbeatRequestData.TaskOffset taskOffset : taskOffsets) { - final TaskId taskId = new TaskIdImpl(taskOffset.subtopologyId(), taskOffset.partition()); + final Map byPartition = result.computeIfAbsent(taskOffset.subtopologyId(), k -> new HashMap<>()); // The reported values come straight from the client heartbeat and the protocol does not enforce uniqueness // of (subtopologyId, partition). Reject a duplicate with a clear client error rather than silently picking // one of the values. - if (result.putIfAbsent(taskId, taskOffset.offset()) != null) { + if (byPartition.putIfAbsent(taskOffset.partition(), taskOffset.offset()) != null) { throw new InvalidRequestException("Task offsets contain a duplicate entry for subtopology " + taskOffset.subtopologyId() + " and partition " + taskOffset.partition() + "."); } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java index 196a205c71419..d9d4961c95766 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/MemberMetadataAndAssignmentImpl.java @@ -18,7 +18,6 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentMetadata; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import java.util.Collections; import java.util.Map; @@ -47,8 +46,8 @@ public record MemberMetadataAndAssignmentImpl(Optional instanceId, Map> warmupTasks, String processId, Map clientTags, - Map taskOffsets, - Map taskEndOffsets + Map> taskOffsets, + Map> taskEndOffsets ) implements MemberAssignmentMetadata, MemberAssignmentState { public MemberMetadataAndAssignmentImpl { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java index 9a73b2401ab1a..22e7232efb3d1 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java @@ -17,7 +17,6 @@ package org.apache.kafka.coordinator.group.streams.assignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import java.util.AbstractMap; import java.util.HashMap; @@ -38,9 +37,9 @@ public class ProcessState { private double load; private int taskCount; private final Map memberToTaskCounts; - private final Map> assignedActiveTasks; - private final Map> assignedStandbyTasks; - private final Set assignedTasks; + private final Map> assignedActiveTasks; + private final Map> assignedStandbyTasks; + private final Set assignedTasks; private PriorityQueue> membersByLoad; ProcessState(final String processId) { @@ -70,23 +69,23 @@ public Map memberToTaskCounts() { return memberToTaskCounts; } - public Set assignedActiveTasks() { + public Set assignedActiveTasks() { return assignedActiveTasks.values().stream() .flatMap(Set::stream) .collect(Collectors.toSet()); } - public Map> assignedActiveTasksByMember() { + public Map> assignedActiveTasksByMember() { return assignedActiveTasks; } - public Set assignedStandbyTasks() { + public Set assignedStandbyTasks() { return assignedStandbyTasks.values().stream() .flatMap(Set::stream) .collect(Collectors.toSet()); } - public Map> assignedStandbyTasksByMember() { + public Map> assignedStandbyTasksByMember() { return assignedStandbyTasks; } @@ -98,7 +97,7 @@ public Map> assignedStandbyTasksByMember() { * @param isActive Whether the task is an active task (true) or a standby task (false). * @return the number of tasks that `memberId` has assigned after adding the new task. */ - public int addTask(final String memberId, final TaskId taskId, final boolean isActive) { + public int addTask(final String memberId, final TaskIdImpl taskId, final boolean isActive) { int newTaskCount = addTaskInternal(memberId, taskId, isActive); // We cannot efficiently add a task to a specific member and keep the memberByLoad ordered correctly. // So we just drop the heap here. @@ -109,7 +108,7 @@ public int addTask(final String memberId, final TaskId taskId, final boolean isA return newTaskCount; } - private int addTaskInternal(final String memberId, final TaskId taskId, final boolean isActive) { + private int addTaskInternal(final String memberId, final TaskIdImpl taskId, final boolean isActive) { taskCount += 1; assignedTasks.add(taskId); if (isActive) { @@ -133,7 +132,7 @@ private int addTaskInternal(final String memberId, final TaskId taskId, final bo * @return the number of tasks that `memberId` has assigned after adding the new task, or -1 if the * task was not assigned to any member. */ - public int addTaskToLeastLoadedMember(final TaskId taskId, final boolean isActive) { + public int addTaskToLeastLoadedMember(final TaskIdImpl taskId, final boolean isActive) { if (memberToTaskCounts.isEmpty()) { return -1; } @@ -191,11 +190,11 @@ public int compareTo(final ProcessState other) { return loadCompare; } - public boolean hasTask(final TaskId taskId) { + public boolean hasTask(final TaskIdImpl taskId) { return assignedTasks.contains(taskId); } - Set assignedTasks() { + Set assignedTasks() { return assignedTasks; } } \ No newline at end of file diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index 93556b096b9e8..8b53b0d285d80 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -24,7 +24,6 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignmentState; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignorException; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.api.assignor.streams.TopologyDescriber; import org.slf4j.Logger; @@ -69,19 +68,19 @@ public GroupAssignment assign(final GroupSpec groupSpec, final TopologyDescriber } private GroupAssignment doAssign(final GroupSpec groupSpec, final TopologyDescriber topologyDescriber) { - final LinkedList activeTasks = taskIds(topologyDescriber, true); + final LinkedList activeTasks = taskIds(topologyDescriber, true); assignActive(activeTasks); if (localState.numStandbyReplicas > 0) { - final LinkedList statefulTasks = taskIds(topologyDescriber, false); + final LinkedList statefulTasks = taskIds(topologyDescriber, false); assignStandby(statefulTasks); } return buildGroupAssignment(groupSpec.memberIds()); } - private LinkedList taskIds(final TopologyDescriber topologyDescriber, final boolean isActive) { - final LinkedList ret = new LinkedList<>(); + private LinkedList taskIds(final TopologyDescriber topologyDescriber, final boolean isActive) { + final LinkedList ret = new LinkedList<>(); for (final String subtopology : topologyDescriber.subtopologies()) { if (isActive || topologyDescriber.isStateful(subtopology)) { final int numberOfPartitions = topologyDescriber.maxNumInputPartitions(subtopology); @@ -138,7 +137,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol for (final Map.Entry> entry : memberAssignmentState.standbyTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { - final TaskId taskId = new TaskIdImpl(entry.getKey(), partitionNo); + final TaskIdImpl taskId = new TaskIdImpl(entry.getKey(), partitionNo); localState.standbyTaskToPrevMember.putIfAbsent(taskId, new ArrayList<>(localState.numStandbyReplicas)); localState.standbyTaskToPrevMember.get(taskId).add(member); } @@ -149,14 +148,14 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol private GroupAssignment buildGroupAssignment(final Collection members) { final Map memberAssignments = new HashMap<>(); - final Map> activeTasksAssignments = localState.processIdToState.entrySet().stream() + final Map> activeTasksAssignments = localState.processIdToState.entrySet().stream() .flatMap(entry -> entry.getValue().assignedActiveTasksByMember().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (set1, set2) -> { set1.addAll(set2); return set1; })); - final Map> standbyTasksAssignments = localState.processIdToState.entrySet().stream() + final Map> standbyTasksAssignments = localState.processIdToState.entrySet().stream() .flatMap(entry -> entry.getValue().assignedStandbyTasksByMember().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (set1, set2) -> { set1.addAll(set2); @@ -179,23 +178,23 @@ private GroupAssignment buildGroupAssignment(final Collection members) { return new GroupAssignment(memberAssignments); } - private Map> toCompactedTaskIds(final Set taskIds) { + private Map> toCompactedTaskIds(final Set taskIds) { final Map> ret = new HashMap<>(); - for (final TaskId taskId : taskIds) { + for (final TaskIdImpl taskId : taskIds) { ret.putIfAbsent(taskId.subtopologyId(), new HashSet<>()); ret.get(taskId.subtopologyId()).add(taskId.partition()); } return ret; } - private void assignActive(final LinkedList activeTasks) { + private void assignActive(final LinkedList activeTasks) { // Assuming our current assignment pairs same partitions (range-based), we want to sort by partition first - activeTasks.sort(Comparator.comparing(TaskId::partition).thenComparing(TaskId::subtopologyId)); + activeTasks.sort(Comparator.comparing(TaskIdImpl::partition).thenComparing(TaskIdImpl::subtopologyId)); // 1. re-assigning existing active tasks to clients that previously had the same active tasks - for (final Iterator it = activeTasks.iterator(); it.hasNext();) { - final TaskId task = it.next(); + for (final Iterator it = activeTasks.iterator(); it.hasNext();) { + final TaskIdImpl task = it.next(); final Member prevMember = localState.activeTaskToPrevMember.get(task); if (prevMember != null) { final ProcessState processState = localState.processIdToState.get(prevMember.processId); @@ -209,8 +208,8 @@ private void assignActive(final LinkedList activeTasks) { } // 2. re-assigning tasks to clients that previously have seen the same task (as standby task) - for (final Iterator it = activeTasks.iterator(); it.hasNext();) { - final TaskId task = it.next(); + for (final Iterator it = activeTasks.iterator(); it.hasNext();) { + final TaskIdImpl task = it.next(); final ArrayList prevMembers = localState.standbyTaskToPrevMember.get(task); final Member prevMember = findPrevMemberWithLeastLoad(prevMembers, null); if (prevMember != null) { @@ -225,12 +224,12 @@ private void assignActive(final LinkedList activeTasks) { } // To achieve an initially range-based assignment, sort by subtopology - activeTasks.sort(Comparator.comparing(TaskId::subtopologyId).thenComparing(TaskId::partition)); + activeTasks.sort(Comparator.comparing(TaskIdImpl::subtopologyId).thenComparing(TaskIdImpl::partition)); // 3. assign any remaining unassigned tasks final PriorityQueue processByLoad = new PriorityQueue<>(Comparator.comparingDouble(ProcessState::load)); processByLoad.addAll(localState.processIdToState.values()); - for (final TaskId task: activeTasks) { + for (final TaskIdImpl task: activeTasks) { final ProcessState processWithLeastLoad = processByLoad.poll(); if (processWithLeastLoad == null) { throw new TaskAssignorException(String.format("No process available to assign active task %s.", task)); @@ -262,7 +261,7 @@ private void maybeUpdateTotalTasksPerMember(final int taskNo) { } } - private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue queue, TaskId taskId) { + private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue queue, TaskIdImpl taskId) { final ProcessState processWithLeastLoad = queue.poll(); if (processWithLeastLoad == null) { return false; @@ -290,7 +289,7 @@ private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue q * * @return Previous member with the least load that does not have the task, or null if no such member exists. */ - private Member findPrevMemberWithLeastLoad(final ArrayList members, final TaskId taskId) { + private Member findPrevMemberWithLeastLoad(final ArrayList members, final TaskIdImpl taskId) { if (members == null || members.isEmpty()) { return null; } @@ -327,13 +326,13 @@ private boolean hasUnfulfilledTaskQuota(final ProcessState process, final Member return process.memberToTaskCounts().get(member.memberId) < localState.totalTasksPerMember; } - private void assignStandby(final LinkedList standbyTasks) { + private void assignStandby(final LinkedList standbyTasks) { final ArrayList toLeastLoaded = new ArrayList<>(standbyTasks.size() * localState.numStandbyReplicas); // Assuming our current assignment is range-based, we want to sort by partition first. - standbyTasks.sort(Comparator.comparing(TaskId::partition).thenComparing(TaskId::subtopologyId).reversed()); + standbyTasks.sort(Comparator.comparing(TaskIdImpl::partition).thenComparing(TaskIdImpl::subtopologyId).reversed()); - for (TaskId task : standbyTasks) { + for (TaskIdImpl task : standbyTasks) { for (int i = 0; i < localState.numStandbyReplicas; i++) { // prev active task @@ -384,7 +383,7 @@ private void assignStandby(final LinkedList standbyTasks) { } } - private String errorMessage(final int numStandbyReplicas, final int i, final TaskId task) { + private String errorMessage(final int numStandbyReplicas, final int i, final TaskIdImpl task) { return "Unable to assign " + (numStandbyReplicas - i) + " of " + numStandbyReplicas + " standby tasks for task [" + task + "]."; } @@ -401,10 +400,10 @@ private static int computeTasksPerMember(final int numberOfTasks, final int numb } static class StandbyToAssign { - private final TaskId taskId; + private final TaskIdImpl taskId; private final int remainingReplicas; - public StandbyToAssign(final TaskId taskId, final int remainingReplicas) { + public StandbyToAssign(final TaskIdImpl taskId, final int remainingReplicas) { this.taskId = taskId; this.remainingReplicas = remainingReplicas; } @@ -422,8 +421,8 @@ public Member(final String processId, final String memberId) { private static class LocalState { // helper data structures: - Map activeTaskToPrevMember; - Map> standbyTaskToPrevMember; + Map activeTaskToPrevMember; + Map> standbyTaskToPrevMember; Map processIdToState; int numStandbyReplicas; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java index 7bbf22700d962..a9401cd8fc25f 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java @@ -16,27 +16,27 @@ */ package org.apache.kafka.coordinator.group.streams.assignor; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; - import java.util.Comparator; import java.util.Objects; /** - * Implementation of the {@link TaskId} interface. + * The identifier for a task, consisting of the subtopology ID and the partition. This is an internal + * helper used by the built-in assignors; the public assignor API addresses tasks by subtopology ID + * and partition directly. * * @param subtopologyId The unique identifier of the subtopology. * @param partition The partition of the input topics this task is processing. */ -public record TaskIdImpl(String subtopologyId, int partition) implements TaskId { +public record TaskIdImpl(String subtopologyId, int partition) implements Comparable { public TaskIdImpl { Objects.requireNonNull(subtopologyId); } @Override - public int compareTo(final TaskId other) { - return Comparator.comparing(TaskId::subtopologyId) - .thenComparingInt(TaskId::partition) + public int compareTo(final TaskIdImpl other) { + return Comparator.comparing(TaskIdImpl::subtopologyId) + .thenComparingInt(TaskIdImpl::partition) .compare(this, other); } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index 4f5c27349bdb1..0ff0087c401ff 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -18991,8 +18991,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) StreamsGroup group = context.groupMetadataManager.streamsGroup(groupId); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 10L), - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 20L) + Map.of(subtopology1, Map.of(0, 10L)), + Map.of(subtopology1, Map.of(0, 20L)) ), group.taskOffsets(memberId) ); @@ -19017,8 +19017,8 @@ fooTopicName, computeTopicHash(fooTopicName, metadataImage) assertEquals(List.of(), result.records()); assertEquals( new org.apache.kafka.coordinator.group.streams.MemberTaskOffsets( - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 12L), - Map.of(new org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl(subtopology1, 0), 20L) + Map.of(subtopology1, Map.of(0, 12L)), + Map.of(subtopology1, Map.of(0, 20L)) ), group.taskOffsets(memberId) ); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java index 182c768b66f28..143e3079690b6 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/MemberTaskOffsetsTest.java @@ -18,7 +18,6 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.message.StreamsGroupHeartbeatRequestData; -import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.junit.jupiter.api.Test; @@ -47,11 +46,11 @@ public void shouldConvertHeartbeatRequestTaskOffsets() { ); assertEquals( - Map.of(new TaskIdImpl("sub-1", 0), 10L, new TaskIdImpl("sub-1", 1), 20L), + Map.of("sub-1", Map.of(0, 10L, 1, 20L)), result.taskOffsets() ); assertEquals( - Map.of(new TaskIdImpl("sub-1", 0), 15L, new TaskIdImpl("sub-1", 1), 25L), + Map.of("sub-1", Map.of(0, 15L, 1, 25L)), result.taskEndOffsets() ); } @@ -76,8 +75,8 @@ public void shouldRejectDuplicateTaskEntries() { @Test public void shouldRetainBothMapsWhenBothListsAreNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 10L), - Map.of(new TaskIdImpl("sub-1", 0), 15L) + Map.of("sub-1", Map.of(0, 10L)), + Map.of("sub-1", Map.of(0, 15L)) ); assertEquals(previous, previous.update(null, null)); @@ -87,28 +86,28 @@ public void shouldRetainBothMapsWhenBothListsAreNull() { @Test public void shouldUpdateTaskOffsetsAndRetainTaskEndOffsetsWhenEndOffsetsNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 10L), - Map.of(new TaskIdImpl("sub-1", 0), 15L) + Map.of("sub-1", Map.of(0, 10L)), + Map.of("sub-1", Map.of(0, 15L)) ); MemberTaskOffsets result = previous.update(List.of(taskOffset("sub-1", 0, 12L)), null); - assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 12L), result.taskOffsets()); + assertEquals(Map.of("sub-1", Map.of(0, 12L)), result.taskOffsets()); // The end-offsets were not reported, so the previously reported values are retained. - assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 15L), result.taskEndOffsets()); + assertEquals(Map.of("sub-1", Map.of(0, 15L)), result.taskEndOffsets()); } @Test public void shouldUpdateTaskEndOffsetsAndRetainTaskOffsetsWhenOffsetsNull() { MemberTaskOffsets previous = new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 10L), - Map.of(new TaskIdImpl("sub-1", 0), 15L) + Map.of("sub-1", Map.of(0, 10L)), + Map.of("sub-1", Map.of(0, 15L)) ); MemberTaskOffsets result = previous.update(null, List.of(taskOffset("sub-1", 0, 18L))); // The offsets were not reported, so the previously reported values are retained. - assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 10L), result.taskOffsets()); - assertEquals(Map.of(new TaskIdImpl("sub-1", 0), 18L), result.taskEndOffsets()); + assertEquals(Map.of("sub-1", Map.of(0, 10L)), result.taskOffsets()); + assertEquals(Map.of("sub-1", Map.of(0, 18L)), result.taskEndOffsets()); } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java index 2ea17c40d4046..9f81bed979626 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/StreamsGroupTest.java @@ -48,7 +48,6 @@ import org.apache.kafka.coordinator.group.generated.StreamsGroupTopologyValue; import org.apache.kafka.coordinator.group.streams.StreamsGroup.StreamsGroupState; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; -import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.timeline.SnapshotRegistry; @@ -120,8 +119,8 @@ public void testUpdateAndRetrieveTaskOffsets() { assertEquals(Map.of(), streamsGroup.taskOffsets()); MemberTaskOffsets offsets = new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 10L), - Map.of(new TaskIdImpl("sub-1", 0), 20L) + Map.of("sub-1", Map.of(0, 10L)), + Map.of("sub-1", Map.of(0, 20L)) ); streamsGroup.updateTaskOffsets("member-id", offsets); @@ -130,8 +129,8 @@ public void testUpdateAndRetrieveTaskOffsets() { // A new report replaces the previous one. MemberTaskOffsets newerOffsets = new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 15L), - Map.of(new TaskIdImpl("sub-1", 0), 25L) + Map.of("sub-1", Map.of(0, 15L)), + Map.of("sub-1", Map.of(0, 25L)) ); streamsGroup.updateTaskOffsets("member-id", newerOffsets); assertEquals(newerOffsets, streamsGroup.taskOffsets("member-id")); @@ -142,8 +141,8 @@ public void testRemoveMemberClearsTaskOffsets() { StreamsGroup streamsGroup = createStreamsGroup("foo"); streamsGroup.updateMember(new StreamsGroupMember.Builder("member-id").build()); streamsGroup.updateTaskOffsets("member-id", new MemberTaskOffsets( - Map.of(new TaskIdImpl("sub-1", 0), 10L), - Map.of(new TaskIdImpl("sub-1", 0), 20L) + Map.of("sub-1", Map.of(0, 10L)), + Map.of("sub-1", Map.of(0, 20L)) )); streamsGroup.removeMember("member-id"); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index bad35e8eff0a7..f05cd9d52d9a9 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -25,13 +25,11 @@ import org.apache.kafka.coordinator.group.api.assignor.streams.GroupAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.MemberAssignment; import org.apache.kafka.coordinator.group.api.assignor.streams.TaskAssignor; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.apache.kafka.coordinator.group.generated.StreamsGroupMemberMetadataValue; import org.apache.kafka.coordinator.group.streams.TaskAssignmentTestUtil.TaskRole; import org.apache.kafka.coordinator.group.streams.assignor.GroupSpecImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberAssignmentImpl; import org.apache.kafka.coordinator.group.streams.assignor.MemberMetadataAndAssignmentImpl; -import org.apache.kafka.coordinator.group.streams.assignor.TaskIdImpl; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredSubtopology; import org.apache.kafka.coordinator.group.streams.topics.ConfiguredTopology; @@ -137,8 +135,8 @@ public void testCreateMemberMetadataAndAssignmentPopulatesTaskOffsets() { .setClientTags(Map.of()) .build(); - Map taskOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 10L); - Map taskEndOffsets = Map.of(new TaskIdImpl(fooSubtopologyId, 0), 20L); + Map> taskOffsets = Map.of(fooSubtopologyId, Map.of(0, 10L)); + Map> taskEndOffsets = Map.of(fooSubtopologyId, Map.of(0, 20L)); MemberMetadataAndAssignmentImpl memberMetadata = createMemberMetadataAndAssignment( member, diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java index 36abd22c6da2b..50ec8b394d6e0 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.coordinator.group.streams.assignor; -import org.apache.kafka.coordinator.group.api.assignor.streams.TaskId; import org.junit.jupiter.api.Test; @@ -33,7 +32,7 @@ public class ProcessStateTest { void shouldAddActiveTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskIdImpl("id1", 0); + TaskIdImpl taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, true); @@ -45,7 +44,7 @@ void shouldAddActiveTaskToMember() { void shouldAddStandbyTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskIdImpl("id1", 0); + TaskIdImpl taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, false); @@ -94,7 +93,7 @@ void shouldCompareBasedOnLoadAndCapacity() { void shouldReturnTrueIfTaskIsAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskIdImpl("id1", 0); + TaskIdImpl taskId = new TaskIdImpl("id1", 0); processState.addTask("member1", taskId, true); assertTrue(processState.hasTask(taskId)); @@ -104,7 +103,7 @@ void shouldReturnTrueIfTaskIsAssigned() { void shouldReturnFalseIfTaskIsNotAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId taskId = new TaskIdImpl("id1", 0); + TaskIdImpl taskId = new TaskIdImpl("id1", 0); assertFalse(processState.hasTask(taskId)); } @@ -113,12 +112,12 @@ void shouldReturnFalseIfTaskIsNotAssigned() { void shouldReturnAllAssignedTasks() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskId activeTaskId = new TaskIdImpl("id1", 0); - TaskId standbyTaskId = new TaskIdImpl("id1", 1); + TaskIdImpl activeTaskId = new TaskIdImpl("id1", 0); + TaskIdImpl standbyTaskId = new TaskIdImpl("id1", 1); processState.addTask("member1", activeTaskId, true); processState.addTask("member1", standbyTaskId, false); - Set assignedTasks = processState.assignedTasks(); + Set assignedTasks = processState.assignedTasks(); assertTrue(assignedTasks.contains(activeTaskId)); assertTrue(assignedTasks.contains(standbyTaskId)); From 8d3bd5dd499eeacf14ca0f11ef742b824573be02 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 16:08:10 -0400 Subject: [PATCH 12/15] rename taskidimpl back --- .../group/streams/assignor/ProcessState.java | 24 ++++---- .../streams/assignor/StickyTaskAssignor.java | 58 +++++++++---------- .../assignor/{TaskIdImpl.java => TaskId.java} | 14 ++--- .../streams/assignor/ProcessStateTest.java | 18 +++--- 4 files changed, 56 insertions(+), 58 deletions(-) rename group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/{TaskIdImpl.java => TaskId.java} (74%) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java index 22e7232efb3d1..7c2134cfb67ef 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessState.java @@ -37,9 +37,9 @@ public class ProcessState { private double load; private int taskCount; private final Map memberToTaskCounts; - private final Map> assignedActiveTasks; - private final Map> assignedStandbyTasks; - private final Set assignedTasks; + private final Map> assignedActiveTasks; + private final Map> assignedStandbyTasks; + private final Set assignedTasks; private PriorityQueue> membersByLoad; ProcessState(final String processId) { @@ -69,23 +69,23 @@ public Map memberToTaskCounts() { return memberToTaskCounts; } - public Set assignedActiveTasks() { + public Set assignedActiveTasks() { return assignedActiveTasks.values().stream() .flatMap(Set::stream) .collect(Collectors.toSet()); } - public Map> assignedActiveTasksByMember() { + public Map> assignedActiveTasksByMember() { return assignedActiveTasks; } - public Set assignedStandbyTasks() { + public Set assignedStandbyTasks() { return assignedStandbyTasks.values().stream() .flatMap(Set::stream) .collect(Collectors.toSet()); } - public Map> assignedStandbyTasksByMember() { + public Map> assignedStandbyTasksByMember() { return assignedStandbyTasks; } @@ -97,7 +97,7 @@ public Map> assignedStandbyTasksByMember() { * @param isActive Whether the task is an active task (true) or a standby task (false). * @return the number of tasks that `memberId` has assigned after adding the new task. */ - public int addTask(final String memberId, final TaskIdImpl taskId, final boolean isActive) { + public int addTask(final String memberId, final TaskId taskId, final boolean isActive) { int newTaskCount = addTaskInternal(memberId, taskId, isActive); // We cannot efficiently add a task to a specific member and keep the memberByLoad ordered correctly. // So we just drop the heap here. @@ -108,7 +108,7 @@ public int addTask(final String memberId, final TaskIdImpl taskId, final boolean return newTaskCount; } - private int addTaskInternal(final String memberId, final TaskIdImpl taskId, final boolean isActive) { + private int addTaskInternal(final String memberId, final TaskId taskId, final boolean isActive) { taskCount += 1; assignedTasks.add(taskId); if (isActive) { @@ -132,7 +132,7 @@ private int addTaskInternal(final String memberId, final TaskIdImpl taskId, fina * @return the number of tasks that `memberId` has assigned after adding the new task, or -1 if the * task was not assigned to any member. */ - public int addTaskToLeastLoadedMember(final TaskIdImpl taskId, final boolean isActive) { + public int addTaskToLeastLoadedMember(final TaskId taskId, final boolean isActive) { if (memberToTaskCounts.isEmpty()) { return -1; } @@ -190,11 +190,11 @@ public int compareTo(final ProcessState other) { return loadCompare; } - public boolean hasTask(final TaskIdImpl taskId) { + public boolean hasTask(final TaskId taskId) { return assignedTasks.contains(taskId); } - Set assignedTasks() { + Set assignedTasks() { return assignedTasks; } } \ No newline at end of file diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java index 8b53b0d285d80..b7e5245224529 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignor.java @@ -68,24 +68,24 @@ public GroupAssignment assign(final GroupSpec groupSpec, final TopologyDescriber } private GroupAssignment doAssign(final GroupSpec groupSpec, final TopologyDescriber topologyDescriber) { - final LinkedList activeTasks = taskIds(topologyDescriber, true); + final LinkedList activeTasks = taskIds(topologyDescriber, true); assignActive(activeTasks); if (localState.numStandbyReplicas > 0) { - final LinkedList statefulTasks = taskIds(topologyDescriber, false); + final LinkedList statefulTasks = taskIds(topologyDescriber, false); assignStandby(statefulTasks); } return buildGroupAssignment(groupSpec.memberIds()); } - private LinkedList taskIds(final TopologyDescriber topologyDescriber, final boolean isActive) { - final LinkedList ret = new LinkedList<>(); + private LinkedList taskIds(final TopologyDescriber topologyDescriber, final boolean isActive) { + final LinkedList ret = new LinkedList<>(); for (final String subtopology : topologyDescriber.subtopologies()) { if (isActive || topologyDescriber.isStateful(subtopology)) { final int numberOfPartitions = topologyDescriber.maxNumInputPartitions(subtopology); for (int i = 0; i < numberOfPartitions; i++) { - ret.add(new TaskIdImpl(subtopology, i)); + ret.add(new TaskId(subtopology, i)); } } } @@ -129,7 +129,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol for (final Map.Entry> entry : memberAssignmentState.activeTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { - localState.activeTaskToPrevMember.put(new TaskIdImpl(entry.getKey(), partitionNo), member); + localState.activeTaskToPrevMember.put(new TaskId(entry.getKey(), partitionNo), member); } } @@ -137,7 +137,7 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol for (final Map.Entry> entry : memberAssignmentState.standbyTasks().entrySet()) { final Set partitionNoSet = entry.getValue(); for (final int partitionNo : partitionNoSet) { - final TaskIdImpl taskId = new TaskIdImpl(entry.getKey(), partitionNo); + final TaskId taskId = new TaskId(entry.getKey(), partitionNo); localState.standbyTaskToPrevMember.putIfAbsent(taskId, new ArrayList<>(localState.numStandbyReplicas)); localState.standbyTaskToPrevMember.get(taskId).add(member); } @@ -148,14 +148,14 @@ private void initialize(final GroupSpec groupSpec, final TopologyDescriber topol private GroupAssignment buildGroupAssignment(final Collection members) { final Map memberAssignments = new HashMap<>(); - final Map> activeTasksAssignments = localState.processIdToState.entrySet().stream() + final Map> activeTasksAssignments = localState.processIdToState.entrySet().stream() .flatMap(entry -> entry.getValue().assignedActiveTasksByMember().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (set1, set2) -> { set1.addAll(set2); return set1; })); - final Map> standbyTasksAssignments = localState.processIdToState.entrySet().stream() + final Map> standbyTasksAssignments = localState.processIdToState.entrySet().stream() .flatMap(entry -> entry.getValue().assignedStandbyTasksByMember().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (set1, set2) -> { set1.addAll(set2); @@ -178,23 +178,23 @@ private GroupAssignment buildGroupAssignment(final Collection members) { return new GroupAssignment(memberAssignments); } - private Map> toCompactedTaskIds(final Set taskIds) { + private Map> toCompactedTaskIds(final Set taskIds) { final Map> ret = new HashMap<>(); - for (final TaskIdImpl taskId : taskIds) { + for (final TaskId taskId : taskIds) { ret.putIfAbsent(taskId.subtopologyId(), new HashSet<>()); ret.get(taskId.subtopologyId()).add(taskId.partition()); } return ret; } - private void assignActive(final LinkedList activeTasks) { + private void assignActive(final LinkedList activeTasks) { // Assuming our current assignment pairs same partitions (range-based), we want to sort by partition first - activeTasks.sort(Comparator.comparing(TaskIdImpl::partition).thenComparing(TaskIdImpl::subtopologyId)); + activeTasks.sort(Comparator.comparing(TaskId::partition).thenComparing(TaskId::subtopologyId)); // 1. re-assigning existing active tasks to clients that previously had the same active tasks - for (final Iterator it = activeTasks.iterator(); it.hasNext();) { - final TaskIdImpl task = it.next(); + for (final Iterator it = activeTasks.iterator(); it.hasNext();) { + final TaskId task = it.next(); final Member prevMember = localState.activeTaskToPrevMember.get(task); if (prevMember != null) { final ProcessState processState = localState.processIdToState.get(prevMember.processId); @@ -208,8 +208,8 @@ private void assignActive(final LinkedList activeTasks) { } // 2. re-assigning tasks to clients that previously have seen the same task (as standby task) - for (final Iterator it = activeTasks.iterator(); it.hasNext();) { - final TaskIdImpl task = it.next(); + for (final Iterator it = activeTasks.iterator(); it.hasNext();) { + final TaskId task = it.next(); final ArrayList prevMembers = localState.standbyTaskToPrevMember.get(task); final Member prevMember = findPrevMemberWithLeastLoad(prevMembers, null); if (prevMember != null) { @@ -224,12 +224,12 @@ private void assignActive(final LinkedList activeTasks) { } // To achieve an initially range-based assignment, sort by subtopology - activeTasks.sort(Comparator.comparing(TaskIdImpl::subtopologyId).thenComparing(TaskIdImpl::partition)); + activeTasks.sort(Comparator.comparing(TaskId::subtopologyId).thenComparing(TaskId::partition)); // 3. assign any remaining unassigned tasks final PriorityQueue processByLoad = new PriorityQueue<>(Comparator.comparingDouble(ProcessState::load)); processByLoad.addAll(localState.processIdToState.values()); - for (final TaskIdImpl task: activeTasks) { + for (final TaskId task: activeTasks) { final ProcessState processWithLeastLoad = processByLoad.poll(); if (processWithLeastLoad == null) { throw new TaskAssignorException(String.format("No process available to assign active task %s.", task)); @@ -261,7 +261,7 @@ private void maybeUpdateTotalTasksPerMember(final int taskNo) { } } - private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue queue, TaskIdImpl taskId) { + private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue queue, TaskId taskId) { final ProcessState processWithLeastLoad = queue.poll(); if (processWithLeastLoad == null) { return false; @@ -289,7 +289,7 @@ private boolean assignStandbyToMemberWithLeastLoad(PriorityQueue q * * @return Previous member with the least load that does not have the task, or null if no such member exists. */ - private Member findPrevMemberWithLeastLoad(final ArrayList members, final TaskIdImpl taskId) { + private Member findPrevMemberWithLeastLoad(final ArrayList members, final TaskId taskId) { if (members == null || members.isEmpty()) { return null; } @@ -326,13 +326,13 @@ private boolean hasUnfulfilledTaskQuota(final ProcessState process, final Member return process.memberToTaskCounts().get(member.memberId) < localState.totalTasksPerMember; } - private void assignStandby(final LinkedList standbyTasks) { + private void assignStandby(final LinkedList standbyTasks) { final ArrayList toLeastLoaded = new ArrayList<>(standbyTasks.size() * localState.numStandbyReplicas); // Assuming our current assignment is range-based, we want to sort by partition first. - standbyTasks.sort(Comparator.comparing(TaskIdImpl::partition).thenComparing(TaskIdImpl::subtopologyId).reversed()); + standbyTasks.sort(Comparator.comparing(TaskId::partition).thenComparing(TaskId::subtopologyId).reversed()); - for (TaskIdImpl task : standbyTasks) { + for (TaskId task : standbyTasks) { for (int i = 0; i < localState.numStandbyReplicas; i++) { // prev active task @@ -383,7 +383,7 @@ private void assignStandby(final LinkedList standbyTasks) { } } - private String errorMessage(final int numStandbyReplicas, final int i, final TaskIdImpl task) { + private String errorMessage(final int numStandbyReplicas, final int i, final TaskId task) { return "Unable to assign " + (numStandbyReplicas - i) + " of " + numStandbyReplicas + " standby tasks for task [" + task + "]."; } @@ -400,10 +400,10 @@ private static int computeTasksPerMember(final int numberOfTasks, final int numb } static class StandbyToAssign { - private final TaskIdImpl taskId; + private final TaskId taskId; private final int remainingReplicas; - public StandbyToAssign(final TaskIdImpl taskId, final int remainingReplicas) { + public StandbyToAssign(final TaskId taskId, final int remainingReplicas) { this.taskId = taskId; this.remainingReplicas = remainingReplicas; } @@ -421,8 +421,8 @@ public Member(final String processId, final String memberId) { private static class LocalState { // helper data structures: - Map activeTaskToPrevMember; - Map> standbyTaskToPrevMember; + Map activeTaskToPrevMember; + Map> standbyTaskToPrevMember; Map processIdToState; int numStandbyReplicas; diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java similarity index 74% rename from group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java rename to group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java index a9401cd8fc25f..fe1a14b105b62 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskIdImpl.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/assignor/TaskId.java @@ -20,23 +20,21 @@ import java.util.Objects; /** - * The identifier for a task, consisting of the subtopology ID and the partition. This is an internal - * helper used by the built-in assignors; the public assignor API addresses tasks by subtopology ID - * and partition directly. + * The identifier for a task, consisting of the subtopology ID and the partition. * * @param subtopologyId The unique identifier of the subtopology. * @param partition The partition of the input topics this task is processing. */ -public record TaskIdImpl(String subtopologyId, int partition) implements Comparable { +public record TaskId(String subtopologyId, int partition) implements Comparable { - public TaskIdImpl { + public TaskId { Objects.requireNonNull(subtopologyId); } @Override - public int compareTo(final TaskIdImpl other) { - return Comparator.comparing(TaskIdImpl::subtopologyId) - .thenComparingInt(TaskIdImpl::partition) + public int compareTo(final TaskId other) { + return Comparator.comparing(TaskId::subtopologyId) + .thenComparingInt(TaskId::partition) .compare(this, other); } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java index 50ec8b394d6e0..842d2e789cef1 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java @@ -32,7 +32,7 @@ public class ProcessStateTest { void shouldAddActiveTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskIdImpl taskId = new TaskIdImpl("id1", 0); + TaskId taskId = new TaskId("id1", 0); processState.addTask("member1", taskId, true); @@ -44,7 +44,7 @@ void shouldAddActiveTaskToMember() { void shouldAddStandbyTaskToMember() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskIdImpl taskId = new TaskIdImpl("id1", 0); + TaskId taskId = new TaskId("id1", 0); processState.addTask("member1", taskId, false); @@ -73,7 +73,7 @@ void shouldReturnTrueWhenCapacityIsAvailable() { void shouldReturnFalseWhenCapacityIsFull() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - processState.addTask("member1", new TaskIdImpl("id1", 0), true); + processState.addTask("member1", new TaskId("id1", 0), true); assertFalse(processState.hasCapacity()); } @@ -84,7 +84,7 @@ void shouldCompareBasedOnLoadAndCapacity() { ProcessState processState2 = new ProcessState("process2"); processState1.addMember("member1"); processState2.addMember("member2"); - processState1.addTask("member1", new TaskIdImpl("id1", 0), true); + processState1.addTask("member1", new TaskId("id1", 0), true); assertTrue(processState1.compareTo(processState2) > 0); } @@ -93,7 +93,7 @@ void shouldCompareBasedOnLoadAndCapacity() { void shouldReturnTrueIfTaskIsAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskIdImpl taskId = new TaskIdImpl("id1", 0); + TaskId taskId = new TaskId("id1", 0); processState.addTask("member1", taskId, true); assertTrue(processState.hasTask(taskId)); @@ -103,7 +103,7 @@ void shouldReturnTrueIfTaskIsAssigned() { void shouldReturnFalseIfTaskIsNotAssigned() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskIdImpl taskId = new TaskIdImpl("id1", 0); + TaskId taskId = new TaskId("id1", 0); assertFalse(processState.hasTask(taskId)); } @@ -112,12 +112,12 @@ void shouldReturnFalseIfTaskIsNotAssigned() { void shouldReturnAllAssignedTasks() { ProcessState processState = new ProcessState("process1"); processState.addMember("member1"); - TaskIdImpl activeTaskId = new TaskIdImpl("id1", 0); - TaskIdImpl standbyTaskId = new TaskIdImpl("id1", 1); + TaskId activeTaskId = new TaskId("id1", 0); + TaskId standbyTaskId = new TaskId("id1", 1); processState.addTask("member1", activeTaskId, true); processState.addTask("member1", standbyTaskId, false); - Set assignedTasks = processState.assignedTasks(); + Set assignedTasks = processState.assignedTasks(); assertTrue(assignedTasks.contains(activeTaskId)); assertTrue(assignedTasks.contains(standbyTaskId)); From aff0de56ebdfaaaab81144eadf261524517e001c Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 16:29:12 -0400 Subject: [PATCH 13/15] rename memberSpec to membermetadata --- .../streams/TargetAssignmentBuilder.java | 12 +- .../streams/TargetAssignmentBuilderTest.java | 6 +- .../streams/assignor/MockAssignorTest.java | 22 +- .../assignor/StickyTaskAssignorTest.java | 332 +++++++++--------- 4 files changed, 186 insertions(+), 186 deletions(-) diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java index 8dca62d4ffb25..6a3fbe7c1087c 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java @@ -218,10 +218,10 @@ public TargetAssignmentBuilder withTopology( * @throws TaskAssignorException if the target assignment cannot be computed. */ public TargetAssignmentResult build() throws TaskAssignorException { - Map memberSpecs = new HashMap<>(); + Map memberMetadataMap = new HashMap<>(); - // Prepare the member spec for all members. - members.forEach((memberId, member) -> memberSpecs.put(memberId, createMemberMetadataAndAssignment( + // Prepare the member metadata for all members. + members.forEach((memberId, member) -> memberMetadataMap.put(memberId, createMemberMetadataAndAssignment( member, targetAssignment.getOrDefault(memberId, org.apache.kafka.coordinator.group.streams.TasksTuple.EMPTY), taskOffsets.getOrDefault(memberId, MemberTaskOffsets.EMPTY) @@ -235,14 +235,14 @@ public TargetAssignmentResult build() throws TaskAssignorException { } newGroupAssignment = assignor.assign( new GroupSpecImpl( - Collections.unmodifiableMap(memberSpecs), + Collections.unmodifiableMap(memberMetadataMap), assignmentConfigs ), new TopologyMetadata(metadataImage, topology.subtopologies().get()) ); } else { newGroupAssignment = new GroupAssignment( - memberSpecs.keySet().stream().collect(Collectors.toMap(x -> x, x -> (MemberAssignment) MemberAssignmentImpl.empty()))); + memberMetadataMap.keySet().stream().collect(Collectors.toMap(x -> x, x -> (MemberAssignment) MemberAssignmentImpl.empty()))); } // Compute delta from previous to new target assignment and create the @@ -250,7 +250,7 @@ public TargetAssignmentResult build() throws TaskAssignorException { List records = new ArrayList<>(); Map newTargetAssignment = new HashMap<>(); - memberSpecs.keySet().forEach(memberId -> { + memberMetadataMap.keySet().forEach(memberId -> { org.apache.kafka.coordinator.group.streams.TasksTuple oldMemberAssignment = targetAssignment.get(memberId); org.apache.kafka.coordinator.group.streams.TasksTuple newMemberAssignment = newMemberAssignment(newGroupAssignment, memberId); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index f05cd9d52d9a9..26f82c438ed6c 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -435,9 +435,9 @@ public void prepareMemberAssignment( public org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.TargetAssignmentResult build() { // Prepare expected member specs. - Map memberSpecs = new HashMap<>(); + Map memberMetadataMap = new HashMap<>(); members.forEach((memberId, member) -> - memberSpecs.put(memberId, createMemberMetadataAndAssignment( + memberMetadataMap.put(memberId, createMemberMetadataAndAssignment( member, targetAssignment.getOrDefault(memberId, TasksTuple.EMPTY), MemberTaskOffsets.EMPTY @@ -450,7 +450,7 @@ public org.apache.kafka.coordinator.group.streams.TargetAssignmentBuilder.Target TopologyMetadata topologyMetadata = new TopologyMetadata(metadataImage, subtopologies); // Prepare the expected assignment spec. - GroupSpecImpl groupSpec = new GroupSpecImpl(memberSpecs, new HashMap<>()); + GroupSpecImpl groupSpec = new GroupSpecImpl(memberMetadataMap, new HashMap<>()); // We use `any` here to always return an assignment but use `verify` later on // to ensure that the input was correct. diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java index 3409dc9645d7a..e8c6d2ff96e2a 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/MockAssignorTest.java @@ -65,7 +65,7 @@ public void testZeroMembers() { @Test public void testDoubleAssignment() { - final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -77,7 +77,7 @@ public void testDoubleAssignment() { Map.of() ); - final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of("test-subtopology", Set.of(0)), @@ -91,7 +91,7 @@ public void testDoubleAssignment() { TaskAssignorException ex = assertThrows(TaskAssignorException.class, () -> assignor.assign( new GroupSpecImpl( - Map.of("member1", memberSpec1, "member2", memberSpec2), + Map.of("member1", memberMetadata1, "member2", memberMetadata2), new HashMap<>() ), new TopologyDescriberImpl(5, List.of("test-subtopology")) @@ -118,7 +118,7 @@ public void testBasicScenario() { @Test public void testSingleMember() { - final MemberMetadataAndAssignmentImpl memberSpec = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -132,7 +132,7 @@ public void testSingleMember() { final GroupAssignment result = assignor.assign( new GroupSpecImpl( - Map.of("test_member", memberSpec), + Map.of("test_member", memberMetadata), new HashMap<>() ), new TopologyDescriberImpl(4, List.of("test-subtopology")) @@ -150,7 +150,7 @@ public void testSingleMember() { @Test public void testTwoMembersTwoSubtopologies() { - final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -162,7 +162,7 @@ public void testTwoMembersTwoSubtopologies() { Map.of() ); - final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), Map.of(), @@ -176,7 +176,7 @@ public void testTwoMembersTwoSubtopologies() { final GroupAssignment result = assignor.assign( new GroupSpecImpl( - mkMap(mkEntry("test_member1", memberSpec1), mkEntry("test_member2", memberSpec2)), + mkMap(mkEntry("test_member1", memberMetadata1), mkEntry("test_member2", memberMetadata2)), new HashMap<>() ), new TopologyDescriberImpl(4, List.of("test-subtopology1", "test-subtopology2")) @@ -203,7 +203,7 @@ public void testTwoMembersTwoSubtopologies() { @Test public void testTwoMembersTwoSubtopologiesStickiness() { - final MemberMetadataAndAssignmentImpl memberSpec1 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata1 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( @@ -218,7 +218,7 @@ public void testTwoMembersTwoSubtopologiesStickiness() { Map.of() ); - final MemberMetadataAndAssignmentImpl memberSpec2 = new MemberMetadataAndAssignmentImpl( + final MemberMetadataAndAssignmentImpl memberMetadata2 = new MemberMetadataAndAssignmentImpl( Optional.empty(), Optional.empty(), mkMap( @@ -234,7 +234,7 @@ public void testTwoMembersTwoSubtopologiesStickiness() { ); final GroupAssignment result = assignor.assign( new GroupSpecImpl( - mkMap(mkEntry("test_member1", memberSpec1), mkEntry("test_member2", memberSpec2)), + mkMap(mkEntry("test_member1", memberMetadata1), mkEntry("test_member2", memberMetadata2)), new HashMap<>() ), new TopologyDescriberImpl(4, List.of("test-subtopology1", "test-subtopology2")) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java index cdd8addc41f9c..9ef52fdbbd0c4 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/StickyTaskAssignorTest.java @@ -57,13 +57,13 @@ public void testToStringReturnsName() { @Test public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCount() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); final GroupAssignment result = assignor.assign( new GroupSpecImpl( - mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)), + mkMap(mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)), new HashMap<>() ), new TopologyDescriberImpl(3, false, List.of("test-subtopology")) @@ -82,15 +82,15 @@ public void shouldAssignOneActiveTaskToEachProcessWhenTaskCountSameAsProcessCoun @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { - final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); - final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), - mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), - mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); + final MemberMetadataAndAssignmentImpl memberMetadata11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata32 = createMemberMetadata("process3"); + final Map members = mkMap(mkEntry("member1_1", memberMetadata11), mkEntry("member1_2", memberMetadata12), + mkEntry("member2_1", memberMetadata21), mkEntry("member2_2", memberMetadata22), + mkEntry("member3_1", memberMetadata31), mkEntry("member3_2", memberMetadata32)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -110,15 +110,15 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithNoStandByTasks() { @Test public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { final Map> tasks = mkMap(mkEntry("test-subtopology1", Sets.newSet(0, 1, 2)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2))); - final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); - final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); - final Map members = mkMap(mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), - mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22), - mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); + final MemberMetadataAndAssignmentImpl memberMetadata11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata32 = createMemberMetadata("process3"); + final Map members = mkMap(mkEntry("member1_1", memberMetadata11), mkEntry("member1_2", memberMetadata12), + mkEntry("member2_1", memberMetadata21), mkEntry("member2_2", memberMetadata22), + mkEntry("member3_1", memberMetadata31), mkEntry("member3_2", memberMetadata32)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -139,9 +139,9 @@ public void shouldAssignTopicGroupIdEvenlyAcrossClientsWithStandByTasks() { @Test public void shouldNotMigrateActiveTaskToOtherProcess() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - Map members = mkMap(mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + Map members = mkMap(mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -158,9 +158,9 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { testMember1.activeTasks().get("test-subtopology").size() + testMember2.activeTasks().get("test-subtopology").size()); // flip the previous active tasks assignment around. - memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - members = mkMap(mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + members = mkMap(mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -179,11 +179,11 @@ public void shouldNotMigrateActiveTaskToOtherProcess() { @Test public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -207,11 +207,11 @@ public void shouldMigrateActiveTasksToNewProcessWithoutChangingAllAssignments() @Test public void shouldAssignBasedOnCapacity() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata22 = createMemberMetadata("process2"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); + mkEntry("member1", memberMetadata1), mkEntry("member2_1", memberMetadata21), mkEntry("member2_2", memberMetadata22)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -234,10 +234,10 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { final Map> activeTasks = mkMap( mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3, 4, 5)), mkEntry("test-subtopology2", Sets.newSet(0))); - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", activeTasks, Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", activeTasks, Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -261,14 +261,14 @@ public void shouldAssignTasksEvenlyWithUnequalTopicGroupSizes() { @Test public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { - MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); - MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); + MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + MemberMetadataAndAssignmentImpl memberMetadata4 = createMemberMetadata("process4"); + MemberMetadataAndAssignmentImpl memberMetadata5 = createMemberMetadata("process5"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), - mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), + mkEntry("member3", memberMetadata3), mkEntry("member4", memberMetadata4), mkEntry("member5", memberMetadata5)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -295,14 +295,14 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { assertNull(testMember5.activeTasks().get("test-subtopology")); // change up the assignment and make sure it is still sticky - memberSpec1 = createMemberMetadata("process1"); - memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - memberSpec3 = createMemberMetadata("process3"); - memberSpec4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - memberSpec5 = createMemberMetadata("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + memberMetadata1 = createMemberMetadata("process1"); + memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + memberMetadata3 = createMemberMetadata("process3"); + memberMetadata4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + memberMetadata5 = createMemberMetadata("process5", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), - mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), + mkEntry("member3", memberMetadata3), mkEntry("member4", memberMetadata4), mkEntry("member5", memberMetadata5)); result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -331,11 +331,11 @@ public void shouldKeepActiveTaskStickinessWhenMoreClientThanActiveTasks() { @Test public void shouldAssignTasksToClientWithPreviousStandbyTasks() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(2)))); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -358,10 +358,10 @@ public void shouldAssignTasksToClientWithPreviousStandbyTasks() { @Test public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurrentActiveTasks() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(0)))); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Set.of(1)))); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, mkMap(mkEntry(NUM_STANDBY_REPLICAS_CONFIG, "1"))), @@ -384,17 +384,17 @@ public void shouldNotAssignStandbyTasksToClientWithPreviousStandbyTasksAndCurren @Test public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata21 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(2))), mkMap(mkEntry("test-subtopology", Set.of(1)))); - final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata22 = createMemberMetadata("process2", Map.of(), Map.of()); Map members = mkMap( - mkEntry("member1", memberSpec1), - mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); + mkEntry("member1", memberMetadata1), + mkEntry("member2_1", memberMetadata21), mkEntry("member2_2", memberMetadata22)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -418,13 +418,13 @@ public void shouldAssignBasedOnCapacityWhenMultipleClientHaveStandbyTasks() { @Test public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTaskIsAssignedTo() { final Map> tasks = mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))); - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata4 = createMemberMetadata("process4", mkMap(mkEntry("test-subtopology", Set.of(3))), Map.of()); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), - mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), + mkEntry("member3", memberMetadata3), mkEntry("member4", memberMetadata4)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -455,12 +455,12 @@ public void shouldAssignStandbyTasksToDifferentClientThanCorrespondingActiveTask @Test public void shouldAssignMultipleReplicasOfStandbyTask() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Set.of(0))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Set.of(1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Set.of(2))), Map.of()); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), - mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), + mkEntry("member3", memberMetadata3)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -475,9 +475,9 @@ public void shouldAssignMultipleReplicasOfStandbyTask() { @Test public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHavingTheTaskAssigned() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); Map members = mkMap( - mkEntry("member1", memberSpec1)); + mkEntry("member1", memberMetadata1)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -490,12 +490,12 @@ public void shouldNotAssignStandbyTaskReplicasWhenNoClientAvailableWithoutHaving @Test public void shouldAssignActiveAndStandbyTasks() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); Map members = mkMap( - mkEntry("member1", memberSpec1), - mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), + mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -509,14 +509,14 @@ public void shouldAssignActiveAndStandbyTasks() { @Test public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { - final MemberMetadataAndAssignmentImpl memberSpec11 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec12 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec13 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata11 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata12 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata13 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); Map members = mkMap( - mkEntry("member1_1", memberSpec11), mkEntry("member1_2", memberSpec12), mkEntry("member1_3", memberSpec13), - mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1_1", memberMetadata11), mkEntry("member1_2", memberMetadata12), mkEntry("member1_3", memberMetadata13), + mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -530,15 +530,15 @@ public void shouldAssignAtLeastOneTaskToEachClientIfPossible() { @Test public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); - final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); - final MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); - final MemberMetadataAndAssignmentImpl memberSpec6 = createMemberMetadata("process6"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata4 = createMemberMetadata("process4"); + final MemberMetadataAndAssignmentImpl memberMetadata5 = createMemberMetadata("process5"); + final MemberMetadataAndAssignmentImpl memberMetadata6 = createMemberMetadata("process6"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), - mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3), + mkEntry("member4", memberMetadata4), mkEntry("member5", memberMetadata5), mkEntry("member6", memberMetadata6)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -551,15 +551,15 @@ public void shouldAssignEachActiveTaskToOneClientWhenMoreClientsThanTasks() { @Test public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); - final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); - final MemberMetadataAndAssignmentImpl memberSpec5 = createMemberMetadata("process5"); - final MemberMetadataAndAssignmentImpl memberSpec6 = createMemberMetadata("process6"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata4 = createMemberMetadata("process4"); + final MemberMetadataAndAssignmentImpl memberMetadata5 = createMemberMetadata("process5"); + final MemberMetadataAndAssignmentImpl memberMetadata6 = createMemberMetadata("process6"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), - mkEntry("member4", memberSpec4), mkEntry("member5", memberSpec5), mkEntry("member6", memberSpec6)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3), + mkEntry("member4", memberMetadata4), mkEntry("member5", memberMetadata5), mkEntry("member6", memberMetadata6)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, @@ -574,11 +574,11 @@ public void shouldBalanceActiveAndStandbyTasksAcrossAvailableClients() { @Test public void shouldAssignMoreTasksToClientWithMoreCapacity() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec21 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec22 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata21 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata22 = createMemberMetadata("process2"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2_1", memberSpec21), mkEntry("member2_2", memberSpec22)); + mkEntry("member1", memberMetadata1), mkEntry("member2_1", memberMetadata21), mkEntry("member2_2", memberMetadata22)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -591,12 +591,12 @@ public void shouldAssignMoreTasksToClientWithMoreCapacity() { @Test public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame() { - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); - final MemberMetadataAndAssignmentImpl memberSpec4 = createMemberMetadata("process4"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata4 = createMemberMetadata("process4"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("member4", memberSpec4)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3), mkEntry("member4", memberMetadata4)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -611,11 +611,11 @@ public void shouldReBalanceTasksAcrossAllClientsWhenCapacityAndTaskCountTheSame( @Test public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -629,11 +629,11 @@ public void shouldReBalanceTasksAcrossClientsWhenCapacityLessThanTaskCount() { @Test public void shouldRebalanceTasksToClientsBasedOnCapacity() { - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec31 = createMemberMetadata("process3"); - final MemberMetadataAndAssignmentImpl memberSpec32 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata31 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata32 = createMemberMetadata("process3"); Map members = mkMap( - mkEntry("member2", memberSpec2), mkEntry("member3_1", memberSpec31), mkEntry("member3_2", memberSpec32)); + mkEntry("member2", memberMetadata2), mkEntry("member3_1", memberMetadata31), mkEntry("member3_2", memberMetadata32)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -648,11 +648,11 @@ public void shouldRebalanceTasksToClientsBasedOnCapacity() { public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClientAdded() { final Set p1PrevTasks = Sets.newSet(0, 2); final Set p2PrevTasks = Sets.newSet(1, 3); - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", p1PrevTasks)), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", p2PrevTasks)), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -670,10 +670,10 @@ public void shouldMoveMinimalNumberOfTasksWhenPreviouslyAboveCapacityAndNewClien @Test public void shouldNotMoveAnyTasksWhenNewTasksAdded() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), Map.of()); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -690,11 +690,11 @@ public void shouldNotMoveAnyTasksWhenNewTasksAdded() { @Test public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 1))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 3))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -714,20 +714,20 @@ public void shouldAssignNewTasksToNewClientWhenPreviousTasksAssignedToOldClients @Test public void shouldAssignTasksNotPreviouslyActiveToNewClient() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3", + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3", mkMap(mkEntry("test-subtopology2", Sets.newSet(0, 1, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(2)), mkEntry("test-subtopology1", Sets.newSet(2)))); final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("process4", Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 1, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 2, 3)))); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3), mkEntry("newMember", newMemberSpec)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -746,10 +746,10 @@ public void shouldAssignTasksNotPreviouslyActiveToNewClient() { @Test public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology0", Sets.newSet(1)), mkEntry("test-subtopology1", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology0", Sets.newSet(0)), mkEntry("test-subtopology1", Sets.newSet(1)), mkEntry("test-subtopology2", Sets.newSet(2))), mkMap(mkEntry("test-subtopology0", Sets.newSet(1, 2, 3)), mkEntry("test-subtopology1", Sets.newSet(0, 2, 3)), mkEntry("test-subtopology2", Sets.newSet(0, 1, 3)))); final MemberMetadataAndAssignmentImpl bounce1 = createMemberMetadata("bounce1", @@ -759,7 +759,7 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { Map.of(), mkMap(mkEntry("test-subtopology0", Sets.newSet(2, 3)), mkEntry("test-subtopology1", Sets.newSet(0)))); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("bounce_member1", bounce1), mkEntry("bounce_member2", bounce2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("bounce_member1", bounce1), mkEntry("bounce_member2", bounce2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -778,10 +778,10 @@ public void shouldAssignTasksNotPreviouslyActiveToMultipleNewClients() { @Test public void shouldAssignTasksToNewClient() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(1, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -793,11 +793,11 @@ public void shouldAssignTasksToNewClient() { @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingClients() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5))), Map.of()); final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("process3"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -819,11 +819,11 @@ public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExisting @Test public void shouldAssignTasksToNewClientWithoutFlippingAssignmentBetweenExistingAndBouncedClients() { - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1, 2, 6))), Map.of()); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", Map.of(), mkMap(mkEntry("test-subtopology", Sets.newSet(3, 4, 5)))); final MemberMetadataAndAssignmentImpl newMemberSpec = createMemberMetadata("newProcess"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("newMember", newMemberSpec)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("newMember", newMemberSpec)); GroupAssignment result = assignor.assign( new GroupSpecImpl(members, new HashMap<>()), @@ -1105,19 +1105,19 @@ public void shouldHandleEdgeCaseWithMoreStandbyReplicasThanAvailableClients() { public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssignments() { // Initial setup: Node 1 has active tasks 0,1 and standby tasks 2,3 // Node 2 has active tasks 2,3 and standby tasks 0,1 - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1))), mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(2, 3))), mkMap(mkEntry("test-subtopology", Sets.newSet(0, 1)))); // Node 3 joins as new client - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2), mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2), mkEntry("member3", memberMetadata3)); final GroupAssignment result = assignor.assign( new GroupSpecImpl(members, mkMap(mkEntry(NUM_STANDBY_REPLICAS_CONFIG, "1"))), @@ -1152,12 +1152,12 @@ public void shouldReassignTasksWhenNewNodeJoinsWithExistingActiveAndStandbyAssig @Test public void shouldRangeAssignTasksWhenScalingUp() { // Two clients, the second one is new - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", Map.of("test-subtopology1", Set.of(0, 1), "test-subtopology2", Set.of(0, 1)), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled final GroupAssignment result = assignor.assign( @@ -1200,10 +1200,10 @@ public void shouldRangeAssignTasksWhenScalingUp() { @Test public void shouldRangeAssignTasksWhenStartingEmpty() { // Two clients starting empty (no previous tasks) - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1"); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2"); + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1"); + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2"); final Map members = mkMap( - mkEntry("member1", memberSpec1), mkEntry("member2", memberSpec2)); + mkEntry("member1", memberMetadata1), mkEntry("member2", memberMetadata2)); // Two subtopologies with 2 tasks each (4 tasks total) with standby replicas enabled final GroupAssignment result = assignor.assign( @@ -1261,18 +1261,18 @@ public void shouldAssignStandbyTaskToPreviousOwnerBasedOnBelowQuotaCondition() { // Process1: active=[0], standby=[1] (previously had both active and standby tasks) // Process2: active=[1] (had the active task that process1 had as standby) // Process3: no previous tasks - final MemberMetadataAndAssignmentImpl memberSpec1 = createMemberMetadata("process1", + final MemberMetadataAndAssignmentImpl memberMetadata1 = createMemberMetadata("process1", mkMap(mkEntry("test-subtopology", Sets.newSet(0))), mkMap(mkEntry("test-subtopology", Sets.newSet(1)))); - final MemberMetadataAndAssignmentImpl memberSpec2 = createMemberMetadata("process2", + final MemberMetadataAndAssignmentImpl memberMetadata2 = createMemberMetadata("process2", mkMap(mkEntry("test-subtopology", Sets.newSet(1))), Map.of()); - final MemberMetadataAndAssignmentImpl memberSpec3 = createMemberMetadata("process3"); + final MemberMetadataAndAssignmentImpl memberMetadata3 = createMemberMetadata("process3"); final Map members = mkMap( - mkEntry("member1", memberSpec1), - mkEntry("member2", memberSpec2), - mkEntry("member3", memberSpec3)); + mkEntry("member1", memberMetadata1), + mkEntry("member2", memberMetadata2), + mkEntry("member3", memberMetadata3)); // We have 2 active tasks + 1 standby replica = 4 total tasks // Quota per process = 4 tasks / 3 processes = 1.33 -> 2 tasks per process From 29f7c13ef758eda70fcbcea60ae3abd3a9774015 Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 17:10:13 -0400 Subject: [PATCH 14/15] clean the newline --- .../coordinator/group/streams/assignor/ProcessStateTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java index 842d2e789cef1..ba63afddd30a4 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/assignor/ProcessStateTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.coordinator.group.streams.assignor; - import org.junit.jupiter.api.Test; import java.util.Set; From 6750fbb0fc436847f8f2ccc65372ee0686b7ca8f Mon Sep 17 00:00:00 2001 From: Gabriella Fu <2234862823@qq.com> Date: Fri, 17 Jul 2026 18:02:58 -0400 Subject: [PATCH 15/15] wire memberassigmentstate to member.assignedTasks().warmupTasks() to get the actual warmuptask --- .../streams/MemberAssignmentState.java | 20 ++++++----- .../streams/TargetAssignmentBuilder.java | 4 ++- .../streams/TargetAssignmentBuilderTest.java | 33 +++++++++++++++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java index e35719d41a91f..3bd901364fd8f 100644 --- a/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java +++ b/group-coordinator/group-coordinator-api/src/main/java/org/apache/kafka/coordinator/group/api/assignor/streams/MemberAssignmentState.java @@ -23,12 +23,16 @@ import java.util.Set; /** - * Interface representing the current assignment state for a streams group member. + * Interface representing the current assignment state for a streams group member, used by the + * {@link TaskAssignor} to compute a new, sticky target assignment. * - *

This is the assignment the member currently owns and is used by the {@link TaskAssignor} - * to compute a new, sticky target assignment. Note that the assignor does not assign warm-up - * tasks itself (they are decided during reconciliation), but the currently-assigned warm-up - * tasks are exposed here so that the assignor can take them into account. + *

The active and standby tasks are the member's current target assignment (the last + * assignment committed for the member). Using the committed target as the stickiness baseline keeps + * in-flight task moves converging rather than reverting them. Warm-up tasks are not assigned by the + * assignor (they are decided during reconciliation); the tasks the member is currently + * warming up are exposed here so that the assignor can take them into account. As a result, a task + * that is being moved to this member can appear both as an active or standby task (from the target) + * and as a warm-up task (currently in progress). * *

All accessors are keyed by subtopology ID. The task-set accessors ({@link #activeTasks()}, * {@link #standbyTasks()}, {@link #warmupTasks()}) map each subtopology ID to its set of @@ -40,17 +44,17 @@ public interface MemberAssignmentState { /** - * @return The current active tasks keyed by subtopology Id. + * @return The member's current target active tasks keyed by subtopology Id. */ Map> activeTasks(); /** - * @return The current standby tasks keyed by subtopology Id. + * @return The member's current target standby tasks keyed by subtopology Id. */ Map> standbyTasks(); /** - * @return The current warm-up tasks keyed by subtopology Id. + * @return The tasks the member is currently warming up, keyed by subtopology Id. */ Map> warmupTasks(); diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java index 6a3fbe7c1087c..a1700bc48a42b 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilder.java @@ -127,7 +127,9 @@ static MemberMetadataAndAssignmentImpl createMemberMetadataAndAssignment( member.rackId(), targetAssignment.activeTasks(), targetAssignment.standbyTasks(), - targetAssignment.warmupTasks(), + // Warm-up tasks are decided during reconciliation and recorded in the member's current + // assignment, not in the target assignment, so they are sourced from there. + member.assignedTasks().warmupTasks(), member.processId(), member.clientTags(), taskOffsets.taskOffsets(), diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java index 26f82c438ed6c..09798e5dd957a 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/streams/TargetAssignmentBuilderTest.java @@ -87,7 +87,9 @@ public void testBuildEmptyAssignmentWhenTopologyNotReady() { } @ParameterizedTest - @EnumSource(TaskRole.class) + // Warm-up tasks are sourced from the member's current assignment (set during reconciliation), + // not from the target assignment, so this test only varies the target's active and standby tasks. + @EnumSource(value = TaskRole.class, names = {"ACTIVE", "STANDBY"}) public void testCreateMemberMetadataAndAssignment(TaskRole taskRole) { String fooSubtopologyId = Uuid.randomUuid().toString(); String barSubtopologyId = Uuid.randomUuid().toString(); @@ -98,6 +100,7 @@ public void testCreateMemberMetadataAndAssignment(TaskRole taskRole) { .setInstanceId("instanceId") .setProcessId("processId") .setClientTags(clientTags) + .setAssignedTasks(TasksTupleWithEpochs.EMPTY) .build(); TasksTuple assignment = mkTasksTuple(taskRole, @@ -116,7 +119,7 @@ public void testCreateMemberMetadataAndAssignment(TaskRole taskRole) { Optional.of("rackId"), assignment.activeTasks(), assignment.standbyTasks(), - assignment.warmupTasks(), + Map.of(), "processId", clientTags, Map.of(), @@ -133,6 +136,7 @@ public void testCreateMemberMetadataAndAssignmentPopulatesTaskOffsets() { .setInstanceId("instanceId") .setProcessId("processId") .setClientTags(Map.of()) + .setAssignedTasks(TasksTupleWithEpochs.EMPTY) .build(); Map> taskOffsets = Map.of(fooSubtopologyId, Map.of(0, 10L)); @@ -148,6 +152,30 @@ public void testCreateMemberMetadataAndAssignmentPopulatesTaskOffsets() { assertEquals(taskEndOffsets, memberMetadata.taskEndOffsets()); } + @Test + public void testCreateMemberMetadataAndAssignmentSourcesWarmupTasksFromCurrentAssignment() { + String fooSubtopologyId = Uuid.randomUuid().toString(); + + // Warm-up tasks are decided during reconciliation and stored in the member's current + // assignment; they must be sourced from there, not from the target assignment. + StreamsGroupMember member = new StreamsGroupMember.Builder("member-id") + .setInstanceId("instanceId") + .setRackId("rackId") + .setProcessId("processId") + .setClientTags(Map.of()) + .setAssignedTasks(new TasksTupleWithEpochs(Map.of(), Map.of(), Map.of(fooSubtopologyId, Set.of(1, 2, 3)))) + .build(); + + MemberMetadataAndAssignmentImpl memberMetadata = createMemberMetadataAndAssignment( + member, + // The target assignment never carries warm-up tasks; even if it did, it must be ignored. + mkTasksTuple(TaskRole.WARMUP, mkTasks(fooSubtopologyId, 7, 8, 9)), + MemberTaskOffsets.EMPTY + ); + + assertEquals(Map.of(fooSubtopologyId, Set.of(1, 2, 3)), memberMetadata.warmupTasks()); + } + @Test public void testEmpty() { TargetAssignmentBuilderTestContext context = new TargetAssignmentBuilderTestContext( @@ -410,6 +438,7 @@ public void addGroupMember( memberBuilder.setUserEndpoint(new StreamsGroupMemberMetadataValue.Endpoint().setHost("host").setPort(9090)); memberBuilder.setInstanceId(null); memberBuilder.setRackId(null); + memberBuilder.setAssignedTasks(TasksTupleWithEpochs.EMPTY); members.put(memberId, memberBuilder.build()); targetAssignment.put(memberId, targetTasks); }