feat(kernels): fold the out-of-range mask into SubtractWiden too

benchmarks/Clast.DatabaseDecimal.Benchmarks/NullableColumnarBenchmarks.cs

@@ -29,7 +29,10 @@ public class FoldedMaskBenchmarks
     private long[] _right = null!;
     private long[] _result = null!;
     private ulong[] _mask = null!;
+    private int[] _narrowLeft = null!;
+    private int[] _narrowRight = null!;
 
+    private static readonly DecimalType T32S2 = DecimalType.Numeric(9, 2);
     private static readonly DecimalType T64S2 = DecimalType.Numeric(18, 2);
 
     [GlobalSetup]
@@ -40,11 +43,15 @@ public void Setup()
         _right = new long[N];
         _result = new long[N];
         _mask = new ulong[DecimalRange.MaskWordCount(N)];
+        _narrowLeft = new int[N];
+        _narrowRight = new int[N];
 
         for (int i = 0; i < N; i++)
         {
             _left[i] = rng.NextInt64(-1_000_000_000L, 1_000_000_000L);
             _right[i] = rng.NextInt64(-1_000_000_000L, 1_000_000_000L);
+            _narrowLeft[i] = rng.Next(-100_000_000, 100_000_000);
+            _narrowRight[i] = rng.Next(-100_000_000, 100_000_000);
         }
     }
 
@@ -65,6 +72,18 @@ public int Add_Int64_FoldedMask() =>
     [Benchmark]
     public int Subtract_Int64_FoldedMask() =>
         SpanAddKernel.Subtract(_left, T64S2, _right, T64S2, _result, T64S2, _mask);
+
+    /// <summary>
+    /// The widening pair, whose vectorised path widens each 32-bit chunk into
+    /// two 64-bit vectors and so reports mask bits for two half-chunks per step.
+    /// </summary>
+    [Benchmark]
+    public int AddWiden_Int32_To_Int64_FoldedMask() =>
+        SpanAddKernel.AddWiden(_narrowLeft, T32S2, _narrowRight, T32S2, _result, T64S2, _mask);
+
+    [Benchmark]
+    public int SubtractWiden_Int32_To_Int64_FoldedMask() =>
+        SpanAddKernel.SubtractWiden(_narrowLeft, T32S2, _narrowRight, T32S2, _result, T64S2, _mask);
 }
 
 /// <summary>

src/Clast.DatabaseDecimal/Arithmetic/SpanAddKernel.cs

@@ -1169,6 +1169,87 @@ public static int Subtract(
         return count;
     }
 
+    /// <inheritdoc cref="Subtract(ReadOnlySpan{int}, DecimalType, ReadOnlySpan{int}, DecimalType, Span{int}, DecimalType, Span{ulong}, DecimalRounding)"/>
+    public static int SubtractWiden(
+        ReadOnlySpan<int> left, DecimalType leftType,
+        ReadOnlySpan<int> right, DecimalType rightType,
+        Span<long> result, DecimalType resultType,
+        Span<ulong> outOfRangeMask,
+        DecimalRounding rounding = DecimalRounding.HalfEven)
+    {
+        ValidateLengths(left.Length, right.Length, result.Length);
+        Span<ulong> mask = MaskWords.PrepareOut(left.Length, outOfRangeMask, nameof(outOfRangeMask));
+        DecimalRange.GetBounds(resultType, out long lower, out long upper);
+
+        int ld = resultType.Scale - leftType.Scale;
+        int rd = resultType.Scale - rightType.Scale;
+
+        if (ld == 0 && rd == 0)
+            return SubtractWidenSameScale32To64Masked(left, right, result, lower, upper, mask);
+
+        int count = 0;
+        for (int i = 0; i < left.Length; i++)
+        {
+            long value = checked(ScaleHelper.WidenByDelta32To64(left[i], ld, rounding)
+                - ScaleHelper.WidenByDelta32To64(right[i], rd, rounding));
+            result[i] = value;
+            if (value < lower || value > upper) { mask[i >> 6] |= 1UL << (i & 63); count++; }
+        }
+        return count;
+    }
+
+    /// <inheritdoc cref="Subtract(ReadOnlySpan{int}, DecimalType, ReadOnlySpan{int}, DecimalType, Span{int}, DecimalType, Span{ulong}, DecimalRounding)"/>
+    public static int SubtractWiden(
+        ReadOnlySpan<long> left, DecimalType leftType,
+        ReadOnlySpan<long> right, DecimalType rightType,
+        Span<Int128> result, DecimalType resultType,
+        Span<ulong> outOfRangeMask,
+        DecimalRounding rounding = DecimalRounding.HalfEven)
+    {
+        ValidateLengths(left.Length, right.Length, result.Length);
+        Span<ulong> mask = MaskWords.PrepareOut(left.Length, outOfRangeMask, nameof(outOfRangeMask));
+        DecimalRange.GetBounds(resultType, out Int128 lower, out Int128 upper);
+
+        int ld = resultType.Scale - leftType.Scale;
+        int rd = resultType.Scale - rightType.Scale;
+
+        int count = 0;
+        for (int i = 0; i < left.Length; i++)
+        {
+            Int128 value = checked(ScaleHelper.WidenByDelta64To128(left[i], ld, rounding)
+                - ScaleHelper.WidenByDelta64To128(right[i], rd, rounding));
+            result[i] = value;
+            if (value < lower || value > upper) { mask[i >> 6] |= 1UL << (i & 63); count++; }
+        }
+        return count;
+    }
+
+    /// <inheritdoc cref="Subtract(ReadOnlySpan{int}, DecimalType, ReadOnlySpan{int}, DecimalType, Span{int}, DecimalType, Span{ulong}, DecimalRounding)"/>
+    public static int SubtractWiden(
+        ReadOnlySpan<Int128> left, DecimalType leftType,
+        ReadOnlySpan<Int128> right, DecimalType rightType,
+        Span<Int256> result, DecimalType resultType,
+        Span<ulong> outOfRangeMask,
+        DecimalRounding rounding = DecimalRounding.HalfEven)
+    {
+        ValidateLengths(left.Length, right.Length, result.Length);
+        Span<ulong> mask = MaskWords.PrepareOut(left.Length, outOfRangeMask, nameof(outOfRangeMask));
+        DecimalRange.GetBounds(resultType, out Int256 lower, out Int256 upper);
+
+        int ld = resultType.Scale - leftType.Scale;
+        int rd = resultType.Scale - rightType.Scale;
+
+        int count = 0;
+        for (int i = 0; i < left.Length; i++)
+        {
+            Int256 value = checked(ScaleHelper.WidenByDelta128To256(left[i], ld, rounding)
+                - ScaleHelper.WidenByDelta128To256(right[i], rd, rounding));
+            result[i] = value;
+            if (value < lower || value > upper) { mask[i >> 6] |= 1UL << (i & 63); count++; }
+        }
+        return count;
+    }
+
     // ================================================================
     // Inline rescale helpers — delta and rounding are both loop-invariant,
     // so the branch predictor handles the per-element branches perfectly.
@@ -1942,6 +2023,55 @@ private static int AddWidenSameScale32To64Masked(ReadOnlySpan<int> left, ReadOnl
         return count;
     }
 
+    private static int SubtractWidenSameScale32To64Masked(ReadOnlySpan<int> left, ReadOnlySpan<int> right, Span<long> result, long lower, long upper, Span<ulong> mask)
+    {
+        int i = 0;
+        int count = 0;
+#if NET5_0_OR_GREATER
+        if (Vector.IsHardwareAccelerated && left.Length >= Vector<int>.Count)
+        {
+            ReadOnlySpan<Vector<int>> lv = MemoryMarshal.Cast<int, Vector<int>>(left);
+            ReadOnlySpan<Vector<int>> rv = MemoryMarshal.Cast<int, Vector<int>>(right);
+            Span<Vector<long>> ov = MemoryMarshal.Cast<long, Vector<long>>(result);
+            int chunks = lv.Length;
+            // As in the bool-returning sibling, the difference of two widened
+            // 32-bit values cannot overflow 64 bits, so there is no overflow
+            // accumulator — only the declared precision has to be reported, on
+            // both halves of each widened pair.
+            int halfLanes = Vector<long>.Count;
+            Vector<long> loVec = new Vector<long>(lower);
+            Vector<long> hiVec = new Vector<long>(upper);
+            for (int k = 0; k < chunks; k++)
+            {
+                Vector<int> a = lv[k];
+                Vector<int> b = rv[k];
+                Vector.Widen(a, out Vector<long> aLo, out Vector<long> aHi);
+                Vector.Widen(b, out Vector<long> bLo, out Vector<long> bHi);
+                Vector<long> low = aLo - bLo;
+                Vector<long> high = aHi - bHi;
+                Vector<long> badLow = Vector.LessThan(low, loVec) | Vector.GreaterThan(low, hiVec);
+                Vector<long> badHigh = Vector.LessThan(high, loVec) | Vector.GreaterThan(high, hiVec);
+                ov[k * 2] = low;
+                ov[k * 2 + 1] = high;
+                int b0 = k * Vector<int>.Count;
+                if (badLow != Vector<long>.Zero)
+                    count += FlagLanes64(badLow, b0, mask);
+                if (badHigh != Vector<long>.Zero)
+                    count += FlagLanes64(badHigh, b0 + halfLanes, mask);
+            }
+            i = chunks * Vector<int>.Count;
+        }
+#endif
+        for (; i < left.Length; i++)
+        {
+            var value = (long)left[i] - right[i];
+            result[i] = value;
+            if (value < lower || value > upper) { mask[i >> 6] |= 1UL << (i & 63); count++; }
+        }
+
+        return count;
+    }
+
     [MethodImpl(MethodImplOptions.NoInlining)]
     private static void ThrowOverflow() => throw new OverflowException();
 }

tests/Clast.DatabaseDecimal.Tests/NullableColumnarTests.cs

@@ -294,6 +294,108 @@ public void Subtract256_FoldedMask_MatchesKernelPlusSeparatePass(int length)
         Assert.True(expectedCount > 0);
     }
 
+    [Theory]
+    [MemberData(nameof(Lengths))]
+    public void SubtractWiden32To64_FoldedMask_MatchesKernelPlusSeparatePass(int length)
+    {
+        var operandType = DecimalType.Numeric(9, 2);
+        // A result precision the widened difference can exceed, so the mask has
+        // something to say.
+        var resultType = DecimalType.Numeric(9, 2);
+        var rng = new Random(41);
+        int[] left = new int[length], right = new int[length];
+        for (int i = 0; i < length; i++)
+        {
+            left[i] = rng.Next(-900_000_000, 900_000_000);
+            right[i] = rng.Next(-900_000_000, 900_000_000);
+        }
+        left[0] = -900_000_000; right[0] = 900_000_000;
+
+        long[] expected = new long[length];
+        SpanAddKernel.SubtractWiden(left, operandType, right, operandType, expected, resultType,
+            DecimalRounding.HalfEven, DecimalOverflow.Ignore);
+        ulong[] expectedMask = NewMask(length);
+        int expectedCount = DecimalRange.WriteOutOfRangeMask(expected, resultType, expectedMask);
+
+        long[] actual = new long[length];
+        ulong[] actualMask = NewMask(length);
+        int actualCount = SpanAddKernel.SubtractWiden(left, operandType, right, operandType, actual, resultType, actualMask);
+
+        Assert.Equal(expected, actual);
+        Assert.Equal(expectedMask, actualMask);
+        Assert.Equal(expectedCount, actualCount);
+        Assert.True(expectedCount > 0);
+    }
+
+    [Theory]
+    [MemberData(nameof(Lengths))]
+    public void SubtractWiden64To128_FoldedMask_MatchesKernelPlusSeparatePass(int length)
+    {
+        var operandType = DecimalType.Numeric(18, 2);
+        var resultType = DecimalType.Numeric(18, 2);
+        var rng = new Random(42);
+        long[] left = new long[length], right = new long[length];
+        for (int i = 0; i < length; i++)
+        {
+            left[i] = RandomInt64(rng, -900_000_000_000_000_000L, 900_000_000_000_000_000L);
+            right[i] = RandomInt64(rng, -900_000_000_000_000_000L, 900_000_000_000_000_000L);
+        }
+        left[0] = -900_000_000_000_000_000L; right[0] = 900_000_000_000_000_000L;
+
+        Int128[] expected = new Int128[length];
+        SpanAddKernel.SubtractWiden(left, operandType, right, operandType, expected, resultType,
+            DecimalRounding.HalfEven, DecimalOverflow.Ignore);
+        ulong[] expectedMask = NewMask(length);
+        int expectedCount = DecimalRange.WriteOutOfRangeMask(expected, resultType, expectedMask);
+
+        Int128[] actual = new Int128[length];
+        ulong[] actualMask = NewMask(length);
+        int actualCount = SpanAddKernel.SubtractWiden(left, operandType, right, operandType, actual, resultType, actualMask);
+
+        Assert.Equal(expected, actual);
+        Assert.Equal(expectedMask, actualMask);
+        Assert.Equal(expectedCount, actualCount);
+        Assert.True(expectedCount > 0);
+    }
+
+    [Theory]
+    [MemberData(nameof(Lengths))]
+    public void SubtractWiden128To256_FoldedMask_MatchesKernelPlusSeparatePass(int length)
+    {
+        var operandType = DecimalType.Numeric(38, 2);
+        var resultType = DecimalType.Numeric(38, 2);
+        var rng = new Random(43);
+        Int128[] left = new Int128[length], right = new Int128[length];
+        for (int i = 0; i < length; i++)
+        {
+            left[i] = (Int128)RandomInt64(rng) * 90;
+            right[i] = (Int128)RandomInt64(rng) * 90;
+        }
+        // 9e37 each way, so the difference is 1.8e38 against NUMERIC(38,2)'s
+        // bound of 10^38. Built by repeated multiplication because 10^37 does
+        // not fit a long literal.
+        Int128 nine37 = Int128.One;
+        for (int k = 0; k < 37; k++) nine37 *= 10;
+        nine37 *= 9;
+        left[0] = -nine37;
+        right[0] = nine37;
+
+        Int256[] expected = new Int256[length];
+        SpanAddKernel.SubtractWiden(left, operandType, right, operandType, expected, resultType,
+            DecimalRounding.HalfEven, DecimalOverflow.Ignore);
+        ulong[] expectedMask = NewMask(length);
+        int expectedCount = DecimalRange.WriteOutOfRangeMask(expected, resultType, expectedMask);
+
+        Int256[] actual = new Int256[length];
+        ulong[] actualMask = NewMask(length);
+        int actualCount = SpanAddKernel.SubtractWiden(left, operandType, right, operandType, actual, resultType, actualMask);
+
+        Assert.Equal(expected, actual);
+        Assert.Equal(expectedMask, actualMask);
+        Assert.Equal(expectedCount, actualCount);
+        Assert.True(expectedCount > 0);
+    }
+
     [Fact]
     public void FoldedMask_RescalingPath_MatchesKernelPlusSeparatePass()
     {