Unity GC write barrier patch

This is a temporary patch to allow our NULL GC to work with upstream
changes.  Check the upper bounds in JIT_WriteBarrier_PreGrow64 and
JIT_WriteBarrier_WriteWatch_PreGrow64.

We must remove this when we return to default coreclr gc.

Author: bholmes

src/coreclr/clrdefinitions.cmake

@@ -118,6 +118,7 @@ if(FEATURE_UNITY_EMBEDDING_INTERFACE)
     add_definitions(-DFEATURE_UNITY_ASSEMBLY_MEMORY_PATH)
     add_definitions(-DFEATURE_UNITY_MODULE_NUM_TYPEDEFS)
     add_definitions(-DFEATURE_UNITY_EXPLICIT_LAYOUT_INHERITANCE)
+    add_definitions(-DFEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH)
 endif()
 if(FEATURE_GDBJIT)
     add_definitions(-DFEATURE_GDBJIT)

src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm

@@ -56,6 +56,17 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Lower
 
         nop ; padding for alignment of constant
 
+ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+
+PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Upper
+        mov     r8, 0F0F0F0F0F0F0F0F0h
+
+        cmp     rdx, r8
+        jae     Exit
+
+        nop ; padding for alignment of constant
+endif
+
 PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable
         mov     rax, 0F0F0F0F0F0F0F0F0h
 
@@ -384,6 +395,18 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower
         ; Touch the card table entry, if not already dirty.
         shr     rcx, 0Bh
         NOP_2_BYTE ; padding for alignment of constant
+
+ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Upper
+        mov     r8, 0F0F0F0F0F0F0F0F0h
+
+        cmp     rdx, r8
+        jae     Exit
+
+        nop ; padding for alignment of constant
+endif
+
 PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable
         mov     rax, 0F0F0F0F0F0F0F0F0h
         cmp     byte ptr [rcx + rax], 0FFh

src/coreclr/vm/amd64/jithelpers_fastwritebarriers.S

@@ -32,6 +32,23 @@ PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Lower
 
         nop // padding for alignment of constant
 
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+
+PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_Upper
+        movabs  r8, 0xF0F0F0F0F0F0F0F0
+
+        cmp     rsi, r8
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x4b
+#else
+        .byte 0x73, 0x2b
+#endif
+        // jae     Exit_PreGrow64
+
+        nop // padding for alignment of constant
+#endif
+
 PATCH_LABEL JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable
         movabs  rax, 0xF0F0F0F0F0F0F0F0
 
@@ -415,6 +432,24 @@ PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower
         // Touch the card table entry, if not already dirty.
         shr     rdi, 0x0B
         NOP_2_BYTE // padding for alignment of constant
+
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Upper
+        movabs  r10, 0xF0F0F0F0F0F0F0F0
+
+        cmp     rsi, r10
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+        .byte 0x73, 0x3b
+#else
+        .byte 0x73, 0x2b
+#endif
+        // jae     Exit_WriteWatch_PreGrow64
+
+        nop // padding for alignment of constant
+#endif
+
 PATCH_LABEL JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable
         movabs  rax, 0xF0F0F0F0F0F0F0F0
         cmp     byte ptr [rdi + rax], 0xFF

src/coreclr/vm/amd64/jitinterfaceamd64.cpp

@@ -26,6 +26,9 @@ EXTERN_C void JIT_WriteBarrier_End();
 
 EXTERN_C void JIT_WriteBarrier_PreGrow64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Lower();
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_Upper();
+#endif
 EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardTable();
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 EXTERN_C void JIT_WriteBarrier_PreGrow64_Patch_Label_CardBundleTable();
@@ -79,6 +82,9 @@ EXTERN_C void JIT_WriteBarrier_Bit_Region64_End();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64(Object **dst, Object *ref);
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_WriteWatchTable();
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Lower();
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_Upper();
+#endif
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardTable();
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 EXTERN_C void JIT_WriteBarrier_WriteWatch_PreGrow64_Patch_Label_CardBundleTable();
@@ -172,9 +178,15 @@ void WriteBarrierManager::Validate()
 #endif
 
     pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+    pUpperBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Upper, 2);
+#endif
     pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
 
     _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+    _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
+#endif
     _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
@@ -239,10 +251,16 @@ void WriteBarrierManager::Validate()
 
     pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2);
     pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+    pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Upper, 2);
+#endif
     pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2);
 
     _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pWriteWatchTableImmediate) & 0x7) == 0);
     _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pLowerBoundImmediate) & 0x7) == 0);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+    _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pUpperBoundImmediate) & 0x7) == 0);
+#endif
     _ASSERTE_ALL_BUILDS((reinterpret_cast<UINT64>(pCardTableImmediate) & 0x7) == 0);
 
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
@@ -433,11 +451,17 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
         case WRITE_BARRIER_PREGROW64:
         {
             m_pLowerBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Lower, 2);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+            m_pUpperBoundImmediate      = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_Upper, 2);
+#endif
             m_pCardTableImmediate       = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardTable, 2);
 
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
             _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+            _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
+#endif
 
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
             m_pCardBundleTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_PreGrow64, Patch_Label_CardBundleTable, 2);
@@ -529,11 +553,17 @@ int WriteBarrierManager::ChangeWriteBarrierTo(WriteBarrierType newWriteBarrier,
         {
             m_pWriteWatchTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_WriteWatchTable, 2);
             m_pLowerBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Lower, 2);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+            m_pUpperBoundImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_Upper, 2);
+#endif
             m_pCardTableImmediate = CALC_PATCH_LOCATION(JIT_WriteBarrier_WriteWatch_PreGrow64, Patch_Label_CardTable, 2);
 
             // Make sure that we will be bashing the right places (immediates should be hardcoded to 0x0f0f0f0f0f0f0f0f0).
             _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pWriteWatchTableImmediate);
             _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pLowerBoundImmediate);
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+            _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pUpperBoundImmediate);
+#endif
             _ASSERTE_ALL_BUILDS(0xf0f0f0f0f0f0f0f0 == *(UINT64*)m_pCardTableImmediate);
 
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
@@ -788,6 +818,12 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
         case WRITE_BARRIER_WRITE_WATCH_BYTE_REGIONS64:
         case WRITE_BARRIER_WRITE_WATCH_BIT_REGIONS64:
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#ifdef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
+        case WRITE_BARRIER_PREGROW64:
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#endif // FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
         {
             // Change immediate if different from new g_ephermeral_high.
             if (*(UINT64*)m_pUpperBoundImmediate != (size_t)g_ephemeral_high)
@@ -797,11 +833,13 @@ int WriteBarrierManager::UpdateEphemeralBounds(bool isRuntimeSuspended)
                 stompWBCompleteActions |= SWB_ICACHE_FLUSH;
             }
         }
+#ifndef FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
         FALLTHROUGH;
         case WRITE_BARRIER_PREGROW64:
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         case WRITE_BARRIER_WRITE_WATCH_PREGROW64:
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+#endif // FEATURE_UNITY_NULLGC_WRITEBARRIER_PATCH
         {
             // Change immediate if different from new g_ephermeral_low.
             if (*(UINT64*)m_pLowerBoundImmediate != (size_t)g_ephemeral_low)