diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 877c21311876d5..2cb5c28fef4305 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -2153,59 +2153,58 @@ private static int LastIndexOfValueType(ref TValue searchSpace else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) { Vector256 equals, values = Vector256.Create(value); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + nint offset = length - Vector256.Count; - // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + // Loop until either we've finished all elements -or- there's one or less than a vector's-worth remaining. + while (offset > 0) { - equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace))); + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset)))); + if (equals == Vector256.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + offset -= Vector256.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector256.Count != 0) + // Process the first vector in the search space. + + equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace))); + + if (equals != Vector256.Zero) { - equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace))); - if (equals != Vector256.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } else { Vector128 equals, values = Vector128.Create(value); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + nint offset = length - Vector128.Count; - // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + // Loop until either we've finished all elements -or- there's one or less than a vector's-worth remaining. + while (offset > 0) { - equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace))); + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset)))); + if (equals == Vector128.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + offset -= Vector128.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector128.Count != 0) + + // Process the first vector in the search space. + + equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace))); + + if (equals != Vector128.Zero) { - equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace))); - if (equals != Vector128.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } @@ -2291,63 +2290,60 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) { Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + nint offset = length - Vector256.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector256.LoadUnsafe(ref currentSearchSpace); + current = Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset)); equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)); + if (equals == Vector256.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + offset -= Vector256.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector256.Count != 0) + // Process the first vector in the search space. + + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)); + + if (equals != Vector256.Zero) { - current = Vector256.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)); - if (equals != Vector256.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } else { Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + nint offset = length - Vector128.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector128.LoadUnsafe(ref currentSearchSpace); + current = Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset)); equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)); if (equals == Vector128.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + offset -= Vector128.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector128.Count != 0) + // Process the first vector in the search space. + + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)); + + if (equals != Vector128.Zero) { - current = Vector128.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)); - if (equals != Vector128.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } @@ -2433,63 +2429,61 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) { Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + nint offset = length - Vector256.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector256.LoadUnsafe(ref currentSearchSpace); + current = Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset)); equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2)); + if (equals == Vector256.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + offset -= Vector256.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector256.Count != 0) + // Process the first vector in the search space. + + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2)); + + if (equals != Vector256.Zero) { - current = Vector256.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2)); - if (equals != Vector256.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } else { Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + nint offset = length - Vector128.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector128.LoadUnsafe(ref currentSearchSpace); + current = Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset)); equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2)); + if (equals == Vector128.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + offset -= Vector128.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector128.Count != 0) + // Process the first vector in the search space. + + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2)); + + if (equals != Vector128.Zero) { - current = Vector128.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2)); - if (equals != Vector128.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } @@ -2547,67 +2541,61 @@ private static int LastIndexOfAnyValueType(ref TValue searchSp else if (Vector256.IsHardwareAccelerated && length >= Vector256.Count) { Vector256 equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256.Count); + nint offset = length - Vector256.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector256.LoadUnsafe(ref currentSearchSpace); + current = Vector256.LoadUnsafe(ref searchSpace, (nuint)(offset)); equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2) | Vector256.Equals(current, values3)); if (equals == Vector256.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256.Count); + offset -= Vector256.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector256.Count != 0) + // Process the first vector in the search space. + + current = Vector256.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2) | Vector256.Equals(current, values3)); + + if (equals != Vector256.Zero) { - current = Vector256.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) - | Vector256.Equals(current, values2) | Vector256.Equals(current, values3)); - if (equals != Vector256.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } else { Vector128 equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3); - ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128.Count); + nint offset = length - Vector128.Count; // Loop until either we've finished all elements or there's less than a vector's-worth remaining. - do + while (offset > 0) { - current = Vector128.LoadUnsafe(ref currentSearchSpace); - equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) - | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); + current = Vector128.LoadUnsafe(ref searchSpace, (nuint)(offset)); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); + if (equals == Vector128.Zero) { - currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128.Count); + offset -= Vector128.Count; continue; } - return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals); + return ComputeLastIndex(offset, equals); } - while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace)); - // If any elements remain, process the first vector in the search space. - if ((uint)length % Vector128.Count != 0) + // Process the first vector in the search space. + + current = Vector128.LoadUnsafe(ref searchSpace); + equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); + + if (equals != Vector128.Zero) { - current = Vector128.LoadUnsafe(ref searchSpace); - equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) - | Vector128.Equals(current, values2) | Vector128.Equals(current, values3)); - if (equals != Vector128.Zero) - { - return ComputeLastIndex(ref searchSpace, ref searchSpace, equals); - } + return ComputeLastIndex(offset: 0, equals); } } @@ -2631,19 +2619,19 @@ private static int ComputeFirstIndex(ref T searchSpace, ref T current, Vector } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ComputeLastIndex(ref T searchSpace, ref T current, Vector128 equals) where T : struct + private static int ComputeLastIndex(nint offset, Vector128 equals) where T : struct { uint notEqualsElements = equals.ExtractMostSignificantBits(); int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()) + index; + return (int)offset + index; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int ComputeLastIndex(ref T searchSpace, ref T current, Vector256 equals) where T : struct + private static int ComputeLastIndex(nint offset, Vector256 equals) where T : struct { uint notEqualsElements = equals.ExtractMostSignificantBits(); int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero) - return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf()) + index; + return (int)offset + index; } private interface INegator where T : struct