Skip to content

remove GC hole from LastIndexOf* #75855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adamsitnik wants to merge 1 commit into from
Closed

remove GC hole from LastIndexOf* #75855

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 52 additions & 52 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2153,58 +2153,58 @@ private static int LastIndexOfValueType<TValue, TNegator>(ref TValue searchSpace
else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
{
Vector256<TValue> equals, values = Vector256.Create(value);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
int offset = length - Vector256<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace, (nuint)offset)));
Copy link
Member

@gfoidl gfoidl Sep 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(nuint)offset will be a sign-extending move, so less ideal.
Either

Suggested change
equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace, (nuint)offset)));
equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace, (uint)offset)));

or declare as nint offset (then the cast is just a mov instead of movsxd).

if (equals == Vector256<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
offset -= Vector256<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector256<TValue>.Count != 0)
{
equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
if (equals != Vector256<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
else
{
Vector128<TValue> equals, values = Vector128.Create(value);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
int offset = length - Vector128<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace, (nuint)offset)));
if (equals == Vector128<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
offset -= Vector128<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector128<TValue>.Count != 0)
{
equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
if (equals != Vector128<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
Expand Down Expand Up @@ -2291,22 +2291,22 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
{
Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
int offset = length - Vector256<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector256.LoadUnsafe(ref currentSearchSpace);
current = Vector256.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
if (equals == Vector256<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
offset -= Vector256<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector256<TValue>.Count != 0)
Expand All @@ -2315,29 +2315,29 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
if (equals != Vector256<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
else
{
Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
int offset = length - Vector128<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector128.LoadUnsafe(ref currentSearchSpace);
current = Vector128.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
if (equals == Vector128<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
offset -= Vector128<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector128<TValue>.Count != 0)
Expand All @@ -2346,7 +2346,7 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
if (equals != Vector128<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
Expand Down Expand Up @@ -2433,22 +2433,22 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
{
Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
int offset = length - Vector256<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector256.LoadUnsafe(ref currentSearchSpace);
current = Vector256.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
if (equals == Vector256<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
offset -= Vector256<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector256<TValue>.Count != 0)
Expand All @@ -2457,29 +2457,29 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
if (equals != Vector256<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
else
{
Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
int offset = length - Vector128<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector128.LoadUnsafe(ref currentSearchSpace);
current = Vector128.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
if (equals == Vector128<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
offset -= Vector128<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector128<TValue>.Count != 0)
Expand All @@ -2488,7 +2488,7 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
if (equals != Vector128<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
Expand Down Expand Up @@ -2547,23 +2547,23 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
{
Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
int offset = length - Vector256<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector256.LoadUnsafe(ref currentSearchSpace);
current = Vector256.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
| Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
if (equals == Vector256<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
offset -= Vector256<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector256<TValue>.Count != 0)
Expand All @@ -2573,30 +2573,30 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
| Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
if (equals != Vector256<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
else
{
Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
int offset = length - Vector128<TValue>.Count;

// Loop until either we've finished all elements or there's less than a vector's-worth remaining.
do
{
current = Vector128.LoadUnsafe(ref currentSearchSpace);
current = Vector128.LoadUnsafe(ref searchSpace, (nuint)offset);
equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
| Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
if (equals == Vector128<TValue>.Zero)
{
currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
offset -= Vector128<TValue>.Count;
continue;
}

return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
return ComputeLastIndex(offset, equals);
}
while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
while (offset >= 0);

// If any elements remain, process the first vector in the search space.
if ((uint)length % Vector128<TValue>.Count != 0)
Expand All @@ -2606,7 +2606,7 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
| Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
if (equals != Vector128<TValue>.Zero)
{
return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
return ComputeLastIndex(0, equals);
}
}
}
Expand All @@ -2631,19 +2631,19 @@ private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
private static int ComputeLastIndex<T>(int offset, Vector128<T> equals) where T : struct
{
uint notEqualsElements = equals.ExtractMostSignificantBits();
int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
return offset + index;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct
private static int ComputeLastIndex<T>(int offset, Vector256<T> equals) where T : struct
{
uint notEqualsElements = equals.ExtractMostSignificantBits();
int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
return offset + index;
}

private interface INegator<T> where T : struct
Expand Down