IGNITE-28917 Implemented tracking of previous cluster version feature set and sending it to joining nodes

modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeClusterData.java

@@ -20,7 +20,9 @@
 import java.util.UUID;
 import org.apache.ignite.internal.Order;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteComponentFeatureSet;
+import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteNodeFeatureSet;
 import org.apache.ignite.plugin.extensions.communication.Message;
+import org.jetbrains.annotations.Nullable;
 
 /** */
 public class RollingUpgradeClusterData implements Message {
@@ -36,6 +38,10 @@ public class RollingUpgradeClusterData implements Message {
     @Order(2)
     IgniteComponentFeatureSet[] activeFeatures;
 
+    /** */
+    @Order(3)
+    @Nullable IgniteComponentFeatureSet[] prevActiveFeatures;
+
     /** */
     public RollingUpgradeClusterData() {
         // No-op.
@@ -45,10 +51,22 @@ public RollingUpgradeClusterData() {
     public RollingUpgradeClusterData(
         boolean isVersionUpgradeEnabled,
         UUID curFinalizeProcId,
-        IgniteComponentFeatureSet[] activeFeatures
+        IgniteNodeFeatureSet activeFeatures,
+        @Nullable IgniteNodeFeatureSet prevActiveFeatures
     ) {
         this.isVersionUpgradeEnabled = isVersionUpgradeEnabled;
         this.curFinalizeProcId = curFinalizeProcId;
-        this.activeFeatures = activeFeatures;
+        this.activeFeatures = activeFeatures.values();
+        this.prevActiveFeatures = prevActiveFeatures == null ? null : prevActiveFeatures.values();
+    }
+
+    /** */
+    public IgniteNodeFeatureSet activeFeatures() {
+        return new IgniteNodeFeatureSet(activeFeatures);
+    }
+
+    /** */
+    @Nullable public IgniteNodeFeatureSet previousActiveFeatures() {
+        return prevActiveFeatures == null ? null : new IgniteNodeFeatureSet(prevActiveFeatures);
     }
 }

modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java

@@ -218,12 +218,12 @@ IgniteComponentUpgradeState state(String cmpName) {
 
     /** {@inheritDoc} */
     @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
-        RollingUpgradeClusterData gridData = data.commonData();
+        RollingUpgradeClusterData clusterData = data.commonData();
 
-        isVerUpgradeEnabled = gridData.isVersionUpgradeEnabled;
-        curFinalizeProcId = gridData.curFinalizeProcId;
+        isVerUpgradeEnabled = clusterData.isVersionUpgradeEnabled;
+        curFinalizeProcId = clusterData.curFinalizeProcId;
 
-        featureMgr.onGridDataReceived(new IgniteNodeFeatureSet(gridData.activeFeatures));
+        featureMgr.onGridDataReceived(clusterData);
     }
 
     /** {@inheritDoc} */
@@ -396,7 +396,8 @@ private RollingUpgradeClusterData collectRollingUpgradeClusterData() {
         return new RollingUpgradeClusterData(
             isVerUpgradeEnabled,
             curFinalizeProcId,
-            featureMgr.activeFeatures().values()
+            featureMgr.activeFeatures(),
+            featureMgr.previousActiveFeatures()
         );
     }
 

modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/feature/IgniteFeatureManager.java

@@ -21,6 +21,7 @@
 import java.util.Collection;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.rollingupgrade.RollingUpgradeClusterData;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
@@ -35,12 +36,25 @@ public class IgniteFeatureManager {
     /** */
     private final IgniteNodeFeatureSet locVerFeatures;
 
-    /** */
-    private final GridFutureAdapter<Void> locVerFeaturesActivationFut;
+    /**
+     * During the RU process, updated nodes operate in accordance with both the old logical version (prior to RU completion)
+     * and the new one (after RU completion). This variable stores the features of the previous version under which this
+     * node operated.
+     *
+     * <p>
+     *     If a node joins the cluster after the Rolling Upgrade has been finalized, this value is inherited from the
+     *     existing cluster nodes (after Rolling Upgrade finalization, all cluster nodes are guaranteed to hold the same
+     *     value for this field).
+     *</p>
+     */
+    @Nullable private volatile IgniteNodeFeatureSet prevActiveFeatures;
 
     /** */
     private volatile IgniteNodeFeatureSet activeFeatures;
 
+    /** */
+    private final GridFutureAdapter<Void> locVerFeaturesActivationFut;
+
     /** */
     public IgniteFeatureManager(GridKernalContext ctx, IgniteCoreFeatureSet coreFeatures) {
         this.ctx = ctx;
@@ -67,6 +81,11 @@ public IgniteNodeFeatureSet activeFeatures() {
         return finalActiveFeatures;
     }
 
+    /** @return The feature set corresponding to the previous version under which this node operated. */
+    @Nullable public IgniteNodeFeatureSet previousActiveFeatures() {
+        return prevActiveFeatures;
+    }
+
     /** @return {@code true} if the specified {@link IgniteFeature} is active in the cluster; {@code false} otherwise. */
     public boolean isActive(IgniteFeature feature) {
         final IgniteNodeFeatureSet finalActiveFeatures = activeFeatures;
@@ -91,15 +110,22 @@ public void listenActivation(IgniteFeature feature, IgniteRunnable lsnr) {
     }
 
     /** */
-    public void onGridDataReceived(IgniteNodeFeatureSet activeClusterFeatures) {
-        boolean hasSameFeatures = ctx.clientNode()
+    public void onGridDataReceived(RollingUpgradeClusterData clusterData) {
+        IgniteNodeFeatureSet activeClusterFeatures = clusterData.activeFeatures();
+
+        boolean hasSameFeaturesAsCluster = ctx.clientNode()
             ? activeClusterFeatures.containsAll(locVerFeatures)
             : locVerFeatures.equals(activeClusterFeatures);
 
-        if (hasSameFeatures)
+        if (hasSameFeaturesAsCluster) {
+            prevActiveFeatures = clusterData.previousActiveFeatures();
+
             activateLocalVersionFeatures();
-        else
-            this.activeFeatures = activeClusterFeatures;
+        }
+        else {
+            activeFeatures = activeClusterFeatures;
+            prevActiveFeatures = activeClusterFeatures;
+        }
     }
 
     /** */

modules/core/src/test/java/org/apache/ignite/internal/processors/rollingupgrade/AbstractRollingUpgradeTest.java

@@ -39,6 +39,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgnitionEx;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -49,7 +50,10 @@
 import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeature;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteCoreFeatureSet;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeature;
+import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureManager;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteFeatureSet;
+import org.apache.ignite.internal.processors.rollingupgrade.feature.IgniteNodeFeatureSet;
+import org.apache.ignite.internal.processors.rollingupgrade.feature.IgnitePluginFeatureSet;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.TestIgniteReleaseFeatures_2_18_0;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginComponentFeatureSetProvider;
 import org.apache.ignite.internal.processors.rollingupgrade.feature.TestPluginFeature;
@@ -74,6 +78,7 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.events.EventType.EVT_NODE_VALIDATION_FAILED;
 import static org.apache.ignite.internal.IgniteVersionUtils.semanticVersion;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
 
 /**
  * Provides the ability to override a node's version and supported {@link IgniteFeature}s in order to
@@ -167,6 +172,28 @@ public abstract class AbstractRollingUpgradeTest extends GridCommonAbstractTest
         stopAllGrids();
     }
 
+    /** {@inheritDoc} */
+    @Override protected void stopGrid(int idx) {
+        UUID stoppingNodeId = grid(idx).context().localNodeId();
+
+        super.stopGrid(idx);
+
+        // Rolling Upgrade tests are highly susceptible to the topology state. Since the node stop procedure is asynchronous,
+        // we explicitly wait for all nodes to handle the node-left event and update their local topology snapshots.
+        for (Ignite ignite : IgnitionEx.allGridsx()) {
+            try {
+                assertTrue(waitForCondition(
+                    () -> ((IgniteEx)ignite).context().discovery().node(stoppingNodeId) == null,
+                    getTestTimeout()));
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                Thread.currentThread().interrupt();
+
+                throw new IgniteException(e);
+            }
+        }
+    }
+
     /** */
     protected IgniteConfiguration getConfiguration(int idx, String ver) throws Exception {
         return getConfiguration(getTestIgniteInstanceName(idx), ver);
@@ -272,6 +299,7 @@ protected void startCluster(String ver) throws Exception {
         startClientGrid(2, ver);
 
         checkVersionUpgradeInactive(ver);
+        checkPreviousClusterFeatures(null);
     }
 
     /** */
@@ -357,6 +385,38 @@ protected void checkVersionUpgradeInProgress(String logicalVer, String srcVer, S
         );
     }
 
+    /** */
+    protected void checkPreviousClusterFeatures(@Nullable String expVer) throws Exception {
+        TestVersions expVersions = expVer == null ? null : TestVersions.parse(expVer);
+
+        IgniteCoreFeatureSet expPrevCoreFeatures = expVersions != null
+            ? new IgniteCoreFeatureSet(
+                IgniteProductVersion.fromString(expVersions.coreVersion()),
+                IgniteFeatureSet.buildFrom(readDeclaredCoreFeatures(expVersions.coreVersion())))
+            : null;
+
+        IgnitePluginFeatureSet expPrevPluginFeatures = expVersions != null && expVersions.containsPlugin()
+            ? new IgnitePluginFeatureSet(
+                TestPluginFeature.COMPONENT_NAME,
+                IgniteProductVersion.fromString(expVersions.pluginVersion()),
+                IgniteFeatureSet.buildFrom(readDeclaredPluginFeatures(expVersions.pluginVersion())))
+            : null;
+
+        for (Ignite ignite : Ignition.allGrids()) {
+            IgniteNodeFeatureSet prevFeatures = ru(ignite).features().previousActiveFeatures();
+
+            if (expVersions == null)
+                assertNull(prevFeatures);
+            else {
+                assertNotNull(prevFeatures);
+                assertEquals(expPrevCoreFeatures, prevFeatures.componentFeatures(IgniteCoreFeature.COMPONENT_NAME));
+
+                if (expVersions.containsPlugin())
+                    assertEquals(expPrevPluginFeatures, prevFeatures.componentFeatures(TestPluginFeature.COMPONENT_NAME));
+            }
+        }
+    }
+
     /** */
     protected void checkVersionUpgradeInactive(String expVer) throws Exception {
         checkVersionUpgradeEnabledStatus(false);
@@ -467,14 +527,23 @@ protected void upgradeNodeVersion(int nodeIdx, String logicalVer, String srcVer,
 
     /** */
     protected void finalizeClusterVersion(int nodeIdx, String expVer) throws Exception {
+        IgniteFeatureManager featureMgr = ru(nodeIdx).features();
+
+        String prevLogicalVer = featureMgr.localVersionFeatures().equals(featureMgr.activeFeatures())
+            ? null // No actual version upgrade was performed.
+            : resolveCompoundVersion(ru(nodeIdx).features().activeFeatures());
+
+        checkPreviousClusterFeatures(prevLogicalVer);
+
         ru(nodeIdx).finalizeClusterVersion();
 
         checkVersionUpgradeInactive(expVer);
+        checkPreviousClusterFeatures(prevLogicalVer);
     }
 
     /** */
     protected void restartNode(int nodeIdx) throws Exception {
-        String ver = resolveNodeCompoundVersions(nodeIdx);
+        String ver = resolveNodeLocalCompoundVersion(nodeIdx);
         boolean isClient = grid(nodeIdx).context().clientNode();
 
         stopGrid(nodeIdx);
@@ -489,7 +558,7 @@ protected void forAllNodes(ConsumerX<Integer> nodeProcessor) throws Exception {
 
     /** */
     protected void checkUpgradeFailed(int nodeIdx, String targetVer, String errMsg) throws Exception {
-        String srcVer = resolveNodeCompoundVersions(nodeIdx);
+        String srcVer = resolveNodeLocalCompoundVersion(nodeIdx);
         boolean isClient = grid(nodeIdx).context().clientNode();
 
         stopGrid(nodeIdx);
@@ -500,11 +569,16 @@ protected void checkUpgradeFailed(int nodeIdx, String targetVer, String errMsg)
     }
 
     /** */
-    String resolveNodeCompoundVersions(int nodeIdx) {
-        return Arrays.stream(ru(nodeIdx).features().localVersionFeatures().values())
+    protected String resolveNodeLocalCompoundVersion(int nodeIdx) {
+        return resolveCompoundVersion(ru(nodeIdx).features().localVersionFeatures());
+    }
+
+    /** */
+    protected String resolveCompoundVersion(IgniteNodeFeatureSet features) {
+        return Arrays.stream(features.values())
             .sorted(Comparator.comparing(IgniteComponentFeatureSet::componentName))
             .map(f -> semanticVersion(f.version()))
-            .collect(Collectors.joining("|"));
+            .collect(Collectors.joining(" | "));
     }
 
     /** */
@@ -567,7 +641,7 @@ protected static class TestVersions {
 
         /** */
         public TestVersions(List<String> cmpVersions) {
-            assertTrue(!cmpVersions.isEmpty());
+            assertFalse(cmpVersions.isEmpty());
             assertTrue(cmpVersions.size() <= 2);
 
             this.cmpVersions.put(IgniteCoreFeature.COMPONENT_NAME, cmpVersions.get(0));