[release/8.0-staging] Fix SysV first/second return register GC info mismatch (#116208)

* JIT: Fix SysV first/second return register GC info mismatch when generating calls

* Fix struct ReturnKind on SysV AMD64.

On SysV AMD64, structs returned in a float and int reg pair were being
classified as RT_Scalar_XX. This causes downstream consumers
(e.g., HijackFrame::GcScanRoots) to look for obj/byref's in
the second int reg. Per the ABI, however, the first float is passed
through a float reg and the obj/byref is passed through the _first_
int reg. We now detect and fix this case by skipping the first float
type in the ReturnKind encoding and moving the second type into the
first.

Fix #115815

---------

Co-authored-by: zengandrew <7494393+zengandrew@users.noreply.github.com>

Author: jakobbotsch

src/coreclr/jit/codegenxarch.cpp

@@ -6173,6 +6173,18 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
     {
         retSize       = emitTypeSize(retTypeDesc->GetReturnRegType(0));
         secondRetSize = emitTypeSize(retTypeDesc->GetReturnRegType(1));
+
+        if (retTypeDesc->GetABIReturnReg(1) == REG_INTRET)
+        {
+            // If the second return register is REG_INTRET, then the first
+            // return is expected to be in a SIMD register.
+            // The emitter has hardcoded belief that retSize corresponds to
+            // REG_INTRET and secondRetSize to REG_INTRET_1, so fix up the
+            // situation here.
+            assert(!EA_IS_GCREF_OR_BYREF(retSize));
+            retSize       = secondRetSize;
+            secondRetSize = EA_UNKNOWN;
+        }
     }
     else
     {

src/coreclr/jit/gcencode.cpp

@@ -49,8 +49,21 @@ ReturnKind GCInfo::getReturnKind()
         case 1:
             return VarTypeToReturnKind(retTypeDesc.GetReturnRegType(0));
         case 2:
-            return GetStructReturnKind(VarTypeToReturnKind(retTypeDesc.GetReturnRegType(0)),
-                                       VarTypeToReturnKind(retTypeDesc.GetReturnRegType(1)));
+        {
+            var_types first = retTypeDesc.GetReturnRegType(0);
+            var_types second = retTypeDesc.GetReturnRegType(1);
+#ifdef UNIX_AMD64_ABI
+            if (varTypeUsesFloatReg(first))
+            {
+                // first does not consume an int register in this case so an obj/ref
+                // in the second ReturnKind would actually be found in the first int reg.
+                first = second;
+                second = TYP_UNDEF;
+            }
+#endif // UNIX_AMD64_ABI
+            return GetStructReturnKind(VarTypeToReturnKind(first),
+                                       VarTypeToReturnKind(second));
+        }
         default:
 #ifdef DEBUG
             for (unsigned i = 0; i < regCount; i++)

src/coreclr/vm/method.cpp

@@ -1168,6 +1168,15 @@ ReturnKind MethodDesc::ParseReturnKindFromSig(INDEBUG(bool supportStringConstruc
                                     regKinds[i] = RT_Scalar;
                                 }
                             }
+
+                            if (eeClass->GetEightByteClassification(0) == SystemVClassificationTypeSSE)
+                            {
+                                // Skip over SSE types since they do not consume integer registers.
+                                // An obj/byref in the 2nd eight bytes will be in the first integer register.
+                                regKinds[0] = regKinds[1];
+                                regKinds[1] = RT_Scalar;
+                            }
+
                             ReturnKind structReturnKind = GetStructReturnKind(regKinds[0], regKinds[1]);
                             return structReturnKind;
                         }

src/tests/JIT/Regression/JitBlue/Runtime_115815/Runtime_115815.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Threading;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using Xunit;
+
+public class Runtime_115815
+{
+    [Fact]
+    public static void TestEntryPoint()
+    {
+        var destination = new KeyValuePair<Container, double>[1_000];
+
+        // loop to make this method fully interruptible + to get into OSR version
+        for (int i = 0; i < destination.Length * 1000; i++)
+        {
+            destination[i / 1000] = default;
+        }
+
+        for (int i = 0; i < 5; i++)
+        {
+            for (int j = 0; j < destination.Length; j++)
+            {
+                destination[j] = GetValue(j);
+            }
+
+            Thread.Sleep(10);
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static KeyValuePair<Container, double> GetValue(int i)
+        => KeyValuePair.Create(new Container(i.ToString()), (double)i);
+
+    private struct Container
+    {
+        public string Name;
+        public Container(string name) { this.Name = name; }
+    }
+}

src/tests/JIT/Regression/JitBlue/Runtime_115815/Runtime_115815.csproj

@@ -0,0 +1,8 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="$(MSBuildProjectName).cs" />
+  </ItemGroup>
+</Project>