Support for NativeAOT (#31)

src/coreclr/classlibnative/bcltype/arraynative.inl

@@ -294,6 +294,9 @@ FORCEINLINE void InlinedMemmoveGCRefsHelper(void *dest, const void *src, size_t
     }
     CONTRACTL_END;
 
+#if FEATURE_SATORI_GC
+    GCHeapUtilities::GetGCHeap()->BulkMoveWithWriteBarrier(dest, src, len);
+#else
     _ASSERTE(dest != nullptr);
     _ASSERTE(src != nullptr);
     _ASSERTE(dest != src);
@@ -307,16 +310,7 @@ FORCEINLINE void InlinedMemmoveGCRefsHelper(void *dest, const void *src, size_t
     _ASSERTE(CheckPointer(dest));
     _ASSERTE(CheckPointer(src));
 
-    bool localAssignment = false;
+    GCHeapMemoryBarrier();
-    if (len >= sizeof(size_t))
-    {
-        localAssignment = CheckEscapeSatoriRange((size_t)dest, (size_t)src, len);
-    }
-
-    if (!localAssignment)
-    {
-        GCHeapMemoryBarrier();
-    }
 
     // To be able to copy forwards, the destination buffer cannot start inside the source buffer
     if ((size_t)dest - (size_t)src >= len)
@@ -328,7 +322,8 @@ FORCEINLINE void InlinedMemmoveGCRefsHelper(void *dest, const void *src, size_t
         InlinedBackwardGCSafeCopyHelper(dest, src, len);
     }
 
-    SetCardsAfterBulkCopy((Object**)dest, (Object**)src, len);
+    InlinedSetCardsAfterBulkCopyHelper((Object**)dest, len);
+#endif // FEATURE_SATORI_GC
 }
 
 #endif // !_ARRAYNATIVE_INL_

src/coreclr/clrdefinitions.cmake

@@ -179,6 +179,7 @@ if (FEATURE_ENABLE_NO_ADDRESS_SPACE_RANDOMIZATION)
 endif(FEATURE_ENABLE_NO_ADDRESS_SPACE_RANDOMIZATION)
 add_definitions(-DFEATURE_SVR_GC)
 add_definitions(-DFEATURE_SATORI_GC)
+add_definitions(-DFEATURE_SATORI_EXTERNAL_OBJECTS)
 add_definitions(-DFEATURE_SYMDIFF)
 add_compile_definitions(FEATURE_TIERED_COMPILATION)
 if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_LOONGARCH64 OR CLR_CMAKE_TARGET_ARCH_RISCV64)

src/coreclr/gc/gcee.cpp

@@ -494,6 +494,10 @@ void GCHeap::UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t
 #endif // FEATURE_BASICFREEZE
 }
 
+void GCHeap::BulkMoveWithWriteBarrier(void* dst, const void* src, size_t byteCount)
+{
+}
+
 bool GCHeap::RuntimeStructuresValid()
 {
     return GCScan::GetGcRuntimeStructuresValid();

src/coreclr/gc/gcimpl.h

@@ -330,6 +330,9 @@ public:
     static void ReportGenerationBounds();
 
     virtual int RefreshMemoryLimit();
+
+    // Inherited via IGCHeapInternal
+    virtual void BulkMoveWithWriteBarrier(void* dst, const void* src, size_t byteCount);
 };
 
 #endif  // GCIMPL_H_

src/coreclr/gc/gcinterface.h

@@ -798,6 +798,8 @@ public:
 
     virtual size_t GetLastGCGenerationSize(int gen) PURE_VIRTUAL
 
+    virtual void BulkMoveWithWriteBarrier(void* dst, const void* src, size_t byteCount) PURE_VIRTUAL
+
     /*
     ===========================================================================
     Miscellaneous routines used by the VM.

src/coreclr/gc/satori/SatoriAllocator.cpp

@@ -48,10 +48,10 @@ void SatoriAllocator::Initialize(SatoriHeap* heap)
 
     for (int i = 0; i < Satori::ALLOCATOR_BUCKET_COUNT; i++)
     {
-        m_queues[i] = new SatoriRegionQueue(QueueKind::Allocator);
+        m_queues[i] = new (nothrow) SatoriRegionQueue(QueueKind::Allocator);
     }
 
-    m_workChunks = new SatoriWorkList();
+    m_workChunks = new (nothrow) SatoriWorkList();
 
     m_immortalRegion = nullptr;
     m_immortalAlocLock.Initialize();

src/coreclr/gc/satori/SatoriGC.cpp

@@ -186,6 +186,12 @@ int SatoriGC::WaitForFullGCComplete(int millisecondsTimeout)
 unsigned SatoriGC::WhichGeneration(Object* obj)
 {
     SatoriObject* so = (SatoriObject*)obj;
+    if (so->IsExternal())
+    {
+        // never collected -> 3
+        return 3;
+    }
+
     return (unsigned)so->ContainingRegion()->Generation();
 }
 
@@ -267,14 +273,27 @@ bool SatoriGC::RegisterForFinalization(int gen, Object* obj)
     SatoriObject* so = (SatoriObject*)obj;
     _ASSERTE(so->RawGetMethodTable()->HasFinalizer());
 
+    if (so->IsExternal())
+    {
+        // noop
+        return true;
+    }
+
     if (so->IsFinalizationSuppressed())
     {
         so->UnSuppressFinalization();
         return true;
     }
     else
-    {        
+    {
-        return so->ContainingRegion()->RegisterForFinalization(so);
+        SatoriRegion* region = so->ContainingRegion();
+        if (region->Generation() == 3)
+        {
+            // no need to register immortal objects
+            return true;
+        }
+
+        return region->RegisterForFinalization(so);
     }
 }
 
@@ -316,19 +335,19 @@ HRESULT SatoriGC::Initialize()
 // actually checks if object is considered reachable as a result of a marking phase.
 bool SatoriGC::IsPromoted(Object* object)
 {
-    _ASSERTE(object == nullptr || m_heap->IsHeapAddress((size_t)object));
     SatoriObject* o = (SatoriObject*)object;
 
     // objects outside of the collected generation (including null) are considered marked.
     // (existing behavior)
     return o == nullptr ||
+        o->IsExternal() ||
         o->IsMarkedOrOlderThan(m_heap->Recycler()->GetCondemnedGeneration());
 }
 
 bool SatoriGC::IsHeapPointer(void* object, bool small_heap_only)
 {
     //small_heap_only is unused - there is no special heap for large objects.
-    return m_heap->IsHeapAddress((size_t)object);
+    return SatoriHeap::IsInHeap((size_t)object);
 }
 
 unsigned SatoriGC::GetCondemnedGeneration()
@@ -572,7 +591,12 @@ bool SatoriGC::StressHeap(gc_alloc_context* acontext)
 segment_handle SatoriGC::RegisterFrozenSegment(segment_info* pseginfo)
 {
     // N/A
+    // non-null means success;
+#if FEATURE_SATORI_EXTERNAL_OBJECTS
+    return (segment_handle)1;
+#else
     return NULL;
+#endif
 }
 
 void SatoriGC::UnregisterFrozenSegment(segment_handle seg)
@@ -580,9 +604,21 @@ void SatoriGC::UnregisterFrozenSegment(segment_handle seg)
     // N/A
 }
 
+void SatoriGC::UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed)
+{
+    // N/A
+}
+
 bool SatoriGC::IsInFrozenSegment(Object* object)
 {
-    return ((SatoriObject*)object)->ContainingRegion()->Generation() == 3;
+    // is never collected (immortal)
+    SatoriObject* so = (SatoriObject*)object;
+    if (so->IsExternal())
+    {
+        return true;
+    }
+
+    return (unsigned)so->ContainingRegion()->Generation() == 3;
 }
 
 void SatoriGC::ControlEvents(GCEventKeyword keyword, GCEventLevel level)
@@ -670,6 +706,138 @@ void SatoriGC::EnumerateConfigurationValues(void* context, ConfigurationValueFun
     GCConfig::EnumerateConfigurationValues(context, configurationValueFunc);
 }
 
-void SatoriGC::UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed)
+bool SatoriGC::CheckEscapeSatoriRange(size_t dst, size_t src, size_t len)
 {
+    SatoriRegion* curRegion = (SatoriRegion*)GCToEEInterface::GetAllocContext()->gc_reserved_1;
+    if (!curRegion || !curRegion->IsEscapeTracking())
+    {
+        // not tracking escapes, not a local assignment.
+        return false;
+    }
+
+    _ASSERTE(curRegion->IsEscapeTrackedByCurrentThread());
+
+    // if dst is within the curRegion and is not exposed, we are done
+    if (((dst ^ curRegion->Start()) >> 21) == 0)
+    {
+        if (!curRegion->AnyExposed(dst, len))
+        {
+            // thread-local assignment
+            return true;
+        }
+    }
+
+    if (!SatoriHeap::IsInHeap(dst))
+    {
+        // dest not in heap, must be stack, so, local
+        return true;
+    }
+
+    if (((src ^ curRegion->Start()) >> 21) == 0)
+    {
+        // if src is in current region, the elements could be escaping
+        if (!curRegion->AnyExposed(src, len))
+        {
+            // one-element array copy is embarrasingly common. specialcase that.
+            if (len == sizeof(size_t))
+            {
+                SatoriObject* obj = *(SatoriObject**)src;
+                if (obj->SameRegion(curRegion))
+                {
+                    curRegion->EscapeRecursively(obj);
+                }
+            }
+            else
+            {
+                SatoriObject* containingSrcObj = curRegion->FindObject(src);
+                containingSrcObj->ForEachObjectRef(
+                    [&](SatoriObject** ref)
+                    {
+                        SatoriObject* child = *ref;
+                        if (child->SameRegion(curRegion))
+                        {
+                            curRegion->EscapeRecursively(child);
+                        }
+                    },
+                    src,
+                    src + len
+                );
+            }
+        }
+
+        return false;
+    }
+
+    if (SatoriHeap::IsInHeap(src))
+    {
+        // src is not in current region but in heap,
+        // it can't escape anything that belongs to the current thread, but it is not a local assignment.
+        return false;
+    }
+
+    // This is a very rare case where we are copying refs out of non-heap area like stack or native heap.
+    // We do not have a containing type and that is somewhat inconvenient.
+    // 
+    // There are not many scenarios that lead here. In particular, boxing uses a newly
+    // allocated and not yet escaped target, so it does not end up here.
+    // One possible way to get here is a copy-back after a reflection call with a boxed nullable
+    // argument that happen to escape.
+    // 
+    // We could handle this is by conservatively escaping any value that matches an unescaped pointer in curRegion.
+    // However, considering how uncommon this is, we will just give up tracking.
+    curRegion->StopEscapeTracking();
+    return false;
+}
+
+void SatoriGC::SetCardsAfterBulkCopy(size_t dst, size_t src, size_t len)
+{
+    SatoriPage* page = m_heap->PageForAddressChecked(dst);
+    if (page)
+    {
+        SatoriRecycler* recycler = m_heap->Recycler();
+        if (!recycler->IsBarrierConcurrent())
+        {
+            page->SetCardsForRange(dst, dst + len);
+        }
+
+        if (recycler->IsBarrierConcurrent())
+        {
+            page->DirtyCardsForRangeUnordered(dst, dst + len);
+        }
+    }
+}
+
+void SatoriGC::BulkMoveWithWriteBarrier(void* dst, const void* src, size_t byteCount)
+{
+    if (dst == src || byteCount == 0)
+        return;
+
+    // Make sure everything is pointer aligned
+    _ASSERTE(((size_t)dst & (sizeof(size_t) - 1)) == 0);
+    _ASSERTE(((size_t)src & (sizeof(size_t) - 1)) == 0);
+    _ASSERTE(((size_t)byteCount & (sizeof(size_t) - 1)) == 0);
+
+    bool localAssignment = false;
+    if (byteCount >= sizeof(size_t))
+    {
+        localAssignment = CheckEscapeSatoriRange((size_t)dst, (size_t)src, byteCount);
+    }
+
+    if (!localAssignment)
+    {
+#if !defined(TARGET_X86) && !defined(TARGET_AMD64)
+        MemoryBarrier();
+#endif
+    }
+
+    // NOTE! memmove needs to copy with size_t granularity
+    // I do not see how it would not, since everything is aligned.
+    // If we need to handle unaligned moves, we may need to write our own memmove
+    memmove(dst, src, byteCount);
+
+    if (byteCount >= sizeof(size_t) &&
+       (!localAssignment || m_heap->Recycler()->IsBarrierConcurrent()))
+    {
+        SetCardsAfterBulkCopy((size_t)dst, (size_t)src, byteCount);
+    }
 }

src/coreclr/gc/satori/SatoriGC.h

@@ -53,6 +53,9 @@ private:
 
     GCEvent* m_waitForGCEvent;
 
+    bool CheckEscapeSatoriRange(size_t dst, size_t src, size_t len);
+    void SetCardsAfterBulkCopy(size_t dst, size_t src, size_t len);
+
 public:
     // Inherited via IGCHeap
     virtual bool IsValidSegmentSize(size_t size) override;
@@ -151,6 +154,9 @@ public:
 
     // Inherited via IGCHeapInternal
     virtual void UpdateFrozenSegment(segment_handle seg, uint8_t* allocated, uint8_t* committed) override;
+
+    // Inherited via IGCHeapInternal
+    virtual void BulkMoveWithWriteBarrier(void* dst, const void* src, size_t byteCount) override;
 };
 
 #endif

src/coreclr/gc/satori/SatoriHandlePartitioner.h

@@ -39,7 +39,7 @@ public:
     static void Initialize()
     {
         s_partitionCount = SatoriUtil::HandlePartitionsCount();
-        s_scanTickets = new uint8_t[s_partitionCount]{};
+        s_scanTickets = new (nothrow) uint8_t[s_partitionCount]{};
     }
 
     static void StartNextScan()

src/coreclr/gc/satori/SatoriHeap.cpp

@@ -37,12 +37,15 @@
 #include "SatoriPage.h"
 #include "SatoriPage.inl"
 
-void InitWriteBarrier(void* pageMap, size_t highest_address)
+int8_t* SatoriHeap::s_pageByteMap;
+
+static void InitWriteBarrier(void* pageMap, void* pageByteMap, size_t highest_address)
 {
     WriteBarrierParameters args = {};
     args.operation = WriteBarrierOp::Initialize;
     args.is_runtime_suspended = true;
     args.card_table = (uint32_t*)pageMap;
+    args.card_bundle_table = (uint32_t*)pageByteMap;
     args.highest_address = (uint8_t*)highest_address;
 
     // dummy values to make asserts happy, we will not use this
@@ -53,27 +56,26 @@ void InitWriteBarrier(void* pageMap, size_t highest_address)
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
-void UpdateWriteBarrier(void* pageMap, size_t highest_address)
+static void UpdateWriteBarrier(void* pageMap, void* pageByteMap, size_t highest_address)
 {
     WriteBarrierParameters args = {};
     args.operation = WriteBarrierOp::StompResize;
     args.is_runtime_suspended = false;
     args.card_table = (uint32_t*)pageMap;
+    args.card_bundle_table = (uint32_t*)pageByteMap;
     args.highest_address = (uint8_t*)highest_address;
 
     // dummy values to make asserts happy, we will not use this
     args.lowest_address = (uint8_t*)1;
     args.ephemeral_low = (uint8_t*)-1;
     args.ephemeral_high = (uint8_t*)-1;
-    args.card_bundle_table = (uint32_t*)-1;
 
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
 SatoriHeap* SatoriHeap::Create()
 {
-    // we need to cover the whole possible address space (48bit, 52 on rare ARM64).
+    // we need to cover the whole possible address space (48bit, 52 may be supported as needed).
-    // Half-byte per 4Gb
     const int availableAddressSpaceBits = 48;
     const int pageCountBits = availableAddressSpaceBits - Satori::PAGE_BITS;
     const int mapSize = (1 << pageCountBits) * sizeof(SatoriPage*);
@@ -88,10 +90,16 @@ SatoriHeap* SatoriHeap::Create()
         return nullptr;
     }
 
+    SatoriHeap::s_pageByteMap = new (nothrow) int8_t[1 << pageCountBits]{};
+    if (!SatoriHeap::s_pageByteMap)
+    {
+        return nullptr;
+    }
+
     SatoriHeap* heap = (SatoriHeap*)reserved;
     heap->m_reservedMapSize = mapSize;
     heap->m_committedMapSize = commitSize - sizeof(SatoriHeap) + sizeof(SatoriPage*);
-    InitWriteBarrier(heap->m_pageMap, heap->CommittedMapLength() * Satori::PAGE_SIZE_GRANULARITY - 1);
+    InitWriteBarrier(heap->m_pageMap, s_pageByteMap, heap->CommittedMapLength() * Satori::PAGE_SIZE_GRANULARITY - 1);
     heap->m_mapLock.Initialize();
     heap->m_nextPageIndex = 1;
     heap->m_usedMapLength = 1;
@@ -115,7 +123,7 @@ bool SatoriHeap::CommitMoreMap(size_t currentCommittedMapSize)
         {
             // we did the commit
             m_committedMapSize = min(currentCommittedMapSize + commitSize, m_reservedMapSize);
-            UpdateWriteBarrier(m_pageMap, CommittedMapLength() * Satori::PAGE_SIZE_GRANULARITY - 1);
+            UpdateWriteBarrier(m_pageMap, s_pageByteMap, CommittedMapLength() * Satori::PAGE_SIZE_GRANULARITY - 1);
         }
     }
 
@@ -150,6 +158,7 @@ bool SatoriHeap::TryAddRegularPage(SatoriPage*& newPage)
                 // Also the object must be published before other thread could see it, and publishing is a release.
                 // Thus an ordinary write is ok even for weak memory cases.
                 m_pageMap[i] = newPage;
+                s_pageByteMap[i] = 1;
 
                 // ensure the next is advanced to at least i + 1
                 while ((nextPageIndex = m_nextPageIndex) < i + 1 &&
@@ -210,6 +219,7 @@ SatoriPage* SatoriHeap::AddLargePage(size_t minSize)
                 for (size_t j = 0; j < mapMarkCount; j++)
                 {
                     m_pageMap[i + j] = newPage;
+                    s_pageByteMap[i + j] = 1;
                 }
 
                 size_t usedMapLength;

src/coreclr/gc/satori/SatoriHeap.h

@@ -77,9 +77,17 @@ public:
         return nullptr;
     }
 
-    bool IsHeapAddress(size_t address)
+    static bool IsInHeap(size_t address)
     {
-        return PageForAddressChecked(address) != nullptr;
+        _ASSERTE((address & (sizeof(size_t) - 1)) == 0);
+
+        size_t mapIndex = address >> Satori::PAGE_BITS;
+        if (s_pageByteMap[mapIndex] == 0)
+        {
+            return false;
+        }
+
+        return true;
     }
 
     SatoriRegion* RegionForAddressChecked(size_t address);
@@ -127,6 +135,8 @@ private:
     SatoriSpinLock m_mapLock;
     SatoriPage* m_pageMap[1];
 
+    static int8_t* s_pageByteMap;
+
     bool CommitMoreMap(size_t currentlyCommitted);
 
     size_t CommittedMapLength()

src/coreclr/gc/satori/SatoriObject.cpp

@@ -44,6 +44,11 @@ void SatoriObject::Initialize()
 
 void SatoriObject::EscapeCheckOnHandleCreation()
 {
+    if (IsExternal())
+    {
+        return;
+    }
+
     SatoriRegion* region = ContainingRegion();
     if (region->IsEscapeTrackedByCurrentThread())
     {
@@ -108,7 +113,7 @@ void SatoriObject::Validate()
                 {
                     _ASSERTE(ContainingRegion()->IsExposed(ref));
                     SatoriObject* child = VolatileLoad(ref);
-                    if (child->ContainingRegion() == ContainingRegion())
+                    if (child->SameRegion(ContainingRegion()))
                     {
                         _ASSERTE(child->IsEscaped());
                     }

src/coreclr/gc/satori/SatoriObject.h

@@ -54,7 +54,9 @@ public:
     SatoriObject* Next();
 
     size_t Size();
+    bool SameRegion(SatoriRegion* otherRegion);
     bool IsFree();
+    bool IsExternal();
 
     bool IsEscaped();
     bool IsMarked();

src/coreclr/gc/satori/SatoriObject.inl

@@ -63,11 +63,31 @@ FORCEINLINE SatoriObject* SatoriObject::Next()
     return (SatoriObject*)End();
 }
 
+inline bool SatoriObject::IsExternal()
+{
+#if FEATURE_SATORI_EXTERNAL_OBJECTS
+    return !SatoriHeap::IsInHeap(this->Start());
+#else
+    _ASSERTE(SatoriHeap::IsInHeap(this->Start()));
+    return false;
+#endif
+}
+
 inline SatoriRegion* SatoriObject::ContainingRegion()
 {
+#if FEATURE_SATORI_EXTERNAL_OBJECTS
+    _ASSERTE(!IsExternal() || this == nullptr);
+#endif
+
     return (SatoriRegion*)((size_t)this & ~(Satori::REGION_SIZE_GRANULARITY - 1));
 }
 
+
+inline bool SatoriObject::SameRegion(SatoriRegion* otherRegion)
+{
+    return (((size_t)this ^ (size_t)otherRegion) >> Satori::REGION_BITS) == 0;
+}
+
 inline bool SatoriObject::IsFree()
 {
     return RawGetMethodTable() == s_emptyObjectMt;
@@ -214,19 +234,17 @@ inline void SatoriObject::ForEachObjectRef(F lambda, bool includeCollectibleAllo
     CGCDescSeries* cur = map->GetHighestSeries();
 
     // GetNumSeries is actually signed.
-    // Negative value means the pattern repeats -numSeries times such as in a case of arrays
+    // Negative value means the pattern repeats componentNum times (struct arrays)
     ptrdiff_t numSeries = (ptrdiff_t)map->GetNumSeries();
     if (numSeries >= 0)
     {
         CGCDescSeries* last = map->GetLowestSeries();
 
-        // series size is offset by the object size
+        // series size is offset by the object size, so need to compensate for that.
         size_t size = mt->GetBaseSize();
-
-        // object arrays are handled here too, so need to compensate for that.
         if (mt->HasComponentSize())
         {
-            size += (size_t)((ArrayBase*)this)->GetNumComponents() * sizeof(size_t);
+            size += (size_t)((ArrayBase*)this)->GetNumComponents() * mt->RawGetComponentSize();
         }
 
         do
@@ -289,19 +307,17 @@ inline void SatoriObject::ForEachObjectRef(F lambda, size_t start, size_t end)
     CGCDescSeries* cur = map->GetHighestSeries();
 
     // GetNumSeries is actually signed.
-    // Negative value means the pattern repeats -numSeries times such as in a case of arrays
+    // Negative value means the pattern repeats componentNum times (struct arrays)
     ptrdiff_t cnt = (ptrdiff_t)map->GetNumSeries();
     if (cnt >= 0)
     {
         CGCDescSeries* last = map->GetLowestSeries();
 
-        // series size is offset by the object size
+        // series size is offset by the object size, so need to compensate for that.
         size_t size = mt->GetBaseSize();
-
-        // object arrays are handled here too, so need to compensate for that.
         if (mt->HasComponentSize())
         {
-            size += (size_t)((ArrayBase*)this)->GetNumComponents() * sizeof(size_t);
+            size += (size_t)((ArrayBase*)this)->GetNumComponents() * mt->RawGetComponentSize();
         }
 
         do

src/coreclr/gc/satori/SatoriRecycler.cpp

@@ -79,35 +79,35 @@ void SatoriRecycler::Initialize(SatoriHeap* heap)
     m_perfCounterFrequencyMHz = GCToOSInterface::QueryPerformanceFrequency() / 1000;
 
     m_heap = heap;
-    m_trimmer = new SatoriTrimmer(heap);
+    m_trimmer = new (nothrow) SatoriTrimmer(heap);
 
-    m_ephemeralRegions = new SatoriRegionQueue(QueueKind::RecyclerEphemeral);
+    m_ephemeralRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerEphemeral);
-    m_ephemeralFinalizationTrackingRegions = new SatoriRegionQueue(QueueKind::RecyclerEphemeralFinalizationTracking);
+    m_ephemeralFinalizationTrackingRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerEphemeralFinalizationTracking);
-    m_tenuredRegions = new SatoriRegionQueue(QueueKind::RecyclerTenured);
+    m_tenuredRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerTenured);
-    m_tenuredFinalizationTrackingRegions = new SatoriRegionQueue(QueueKind::RecyclerTenuredFinalizationTracking);
+    m_tenuredFinalizationTrackingRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerTenuredFinalizationTracking);
 
-    m_finalizationPendingRegions = new SatoriRegionQueue(QueueKind::RecyclerFinalizationPending);
+    m_finalizationPendingRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerFinalizationPending);
 
-    m_stayingRegions = new SatoriRegionQueue(QueueKind::RecyclerStaying);
+    m_stayingRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerStaying);
-    m_relocatingRegions = new SatoriRegionQueue(QueueKind::RecyclerRelocating);
+    m_relocatingRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerRelocating);
-    m_relocatedRegions = new SatoriRegionQueue(QueueKind::RecyclerRelocated);
+    m_relocatedRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerRelocated);
-    m_relocatedToHigherGenRegions = new SatoriRegionQueue(QueueKind::RecyclerRelocatedToHigherGen);
+    m_relocatedToHigherGenRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerRelocatedToHigherGen);
 
     for (int i = 0; i < Satori::FREELIST_COUNT; i++)
     {
-        m_relocationTargets[i] = new SatoriRegionQueue(QueueKind::RecyclerRelocationTarget);
+        m_relocationTargets[i] = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerRelocationTarget);
     }
 
-    m_deferredSweepRegions = new SatoriRegionQueue(QueueKind::RecyclerDeferredSweep);
+    m_deferredSweepRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerDeferredSweep);
     m_deferredSweepCount = 0;
     m_gen1AddedSinceLastCollection = 0;
     m_gen2AddedSinceLastCollection = 0;
 
-    m_reusableRegions = new SatoriRegionQueue(QueueKind::RecyclerReusable);
+    m_reusableRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerReusable);
-    m_reusableRegionsAlternate = new SatoriRegionQueue(QueueKind::RecyclerReusable);
+    m_reusableRegionsAlternate = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerReusable);
-    m_demotedRegions = new SatoriRegionQueue(QueueKind::RecyclerDemoted);
+    m_demotedRegions = new (nothrow) SatoriRegionQueue(QueueKind::RecyclerDemoted);
 
-    m_workList = new SatoriWorkList();
+    m_workList = new (nothrow) SatoriWorkList();
     m_gcState = GC_STATE_NONE;
     m_isBarrierConcurrent = false;
 
@@ -408,7 +408,7 @@ void SatoriRecycler::TryStartGC(int generation, gc_reason reason)
 
 bool IsHelperThread()
 {
-    return GCToEEInterface::GetThread() == nullptr;
+    return GCToEEInterface::WasCurrentThreadCreatedByGC();
 }
 
 int64_t SatoriRecycler::HelpQuantum()
@@ -456,8 +456,8 @@ bool SatoriRecycler::HelpOnceCore()
     if (!m_isBarrierConcurrent)
     {
         // toggling is a PW fence.
-        ToggleWriteBarrier(true, /* eeSuspended */ false);
         m_isBarrierConcurrent = true;
+        ToggleWriteBarrier(true, /* eeSuspended */ false);
     }
 
     if (MarkOwnStackAndDrainQueues(deadline))
@@ -551,14 +551,14 @@ void SatoriRecycler::ConcurrentPhasePrepFn(gc_alloc_context* gcContext, void* pa
 
         if (region->IsDemoted())
         {
-            recycler->MarkDemoted(region, *markContext);
+            recycler->MarkDemoted(region, markContext);
         }
     }
 
     region = context->LargeRegion();
     if (region && region->IsDemoted())
     {
-        recycler->MarkDemoted(region, *markContext);
+        recycler->MarkDemoted(region, markContext);
     }
 }
 
@@ -686,7 +686,11 @@ void SatoriRecycler::AskForHelp()
         totalHelpers < MaxHelpers() &&
         Interlocked::CompareExchange(&m_totalHelpers, totalHelpers + 1, totalHelpers) == totalHelpers)
     {
-        GCToEEInterface::CreateThread(HelperThreadFn, this, false, "Satori GC Helper Thread");
+        if (!GCToEEInterface::CreateThread(HelperThreadFn, this, false, "Satori GC Helper Thread"))
+        {
+            Interlocked::Decrement(&m_totalHelpers);
+            return;
+        }
     }
 
     if (!m_gateSignaled && Interlocked::CompareExchange(&m_gateSignaled, 1, 0) == 0)
@@ -1178,13 +1182,13 @@ void SatoriRecycler::MarkFn(PTR_PTR_Object ppObject, ScanContext* sc, uint32_t f
         return;
     }
 
-    MarkContext* context = (MarkContext*)sc->_unused1;
+    MarkContext* markContext = (MarkContext*)sc->_unused1;
     SatoriObject* o = (SatoriObject*)location;
 
     if (flags & GC_CALL_INTERIOR)
     {
         // byrefs may point to stack, use checked here
-        SatoriRegion* containingRegion = context->m_heap->RegionForAddressChecked(location);
+        SatoriRegion* containingRegion = markContext->m_heap->RegionForAddressChecked(location);
         if (!containingRegion ||
             (isConservative && (containingRegion->Generation() < 0)))
         {
@@ -1197,13 +1201,17 @@ void SatoriRecycler::MarkFn(PTR_PTR_Object ppObject, ScanContext* sc, uint32_t f
             return;
         }
     }
+    else if (o->IsExternal())
+    {
+        return;
+    }
 
-    if (o->ContainingRegion()->Generation() <= context->m_condemnedGeneration)
+    if (o->ContainingRegion()->Generation() <= markContext->m_condemnedGeneration)
     {
         if (!o->IsMarked())
         {
             o->SetMarkedAtomic();
-            context->PushToMarkQueues(o);
+            markContext->PushToMarkQueues(o);
         }
 
         if (flags & GC_CALL_PINNED)
@@ -1226,9 +1234,9 @@ void SatoriRecycler::UpdateFn(PTR_PTR_Object ppObject, ScanContext* sc, uint32_t
     SatoriObject* o = (SatoriObject*)location;
     if (flags & GC_CALL_INTERIOR)
     {
-        MarkContext* context = (MarkContext*)sc->_unused1;
+        MarkContext* markContext = (MarkContext*)sc->_unused1;
         // byrefs may point to stack, use checked here
-        SatoriRegion* containingRegion = context->m_heap->RegionForAddressChecked(location);
+        SatoriRegion* containingRegion = markContext->m_heap->RegionForAddressChecked(location);
         if (!containingRegion ||
             (isConservative && (containingRegion->Generation() < 0)))
         {
@@ -1273,13 +1281,13 @@ void SatoriRecycler::MarkFnConcurrent(PTR_PTR_Object ppObject, ScanContext* sc,
 #endif //DEBUG_DestroyedHandleValue
 
     SatoriRegion* containingRegion;
-    MarkContext* context = (MarkContext*)sc->_unused1;
+    MarkContext* markContext = (MarkContext*)sc->_unused1;
     SatoriObject* o = (SatoriObject*)location;
 
     if (flags & GC_CALL_INTERIOR)
     {
         // byrefs may point to stack, use checked here
-        containingRegion = context->m_heap->RegionForAddressChecked(location);
+        containingRegion = markContext->m_heap->RegionForAddressChecked(location);
         if (!containingRegion)
         {
             return;
@@ -1312,6 +1320,11 @@ void SatoriRecycler::MarkFnConcurrent(PTR_PTR_Object ppObject, ScanContext* sc,
     }
     else
     {
+        if (o->IsExternal())
+        {
+            return;
+        }
+
         containingRegion = o->ContainingRegion();
         // can't mark in regions which are tracking escapes, bitmap is in use
         if (containingRegion->MaybeEscapeTrackingAcquire())
@@ -1321,12 +1334,12 @@ void SatoriRecycler::MarkFnConcurrent(PTR_PTR_Object ppObject, ScanContext* sc,
     }
 
     // no need for fences here. a published obj cannot become gen2 concurrently. 
-    if (containingRegion->Generation() <= context->m_condemnedGeneration)
+    if (containingRegion->Generation() <= markContext->m_condemnedGeneration)
     {
         if (!o->IsMarked())
         {
             o->SetMarkedAtomic();
-            context->PushToMarkQueues(o);
+            markContext->PushToMarkQueues(o);
         }
 
         if (flags & GC_CALL_PINNED)
@@ -1338,12 +1351,11 @@ void SatoriRecycler::MarkFnConcurrent(PTR_PTR_Object ppObject, ScanContext* sc,
 
 bool SatoriRecycler::MarkOwnStackAndDrainQueues(int64_t deadline)
 {
-    MarkContext c = MarkContext(this);
+    MarkContext markContext = MarkContext(this);
-    gc_alloc_context* aContext = GCToEEInterface::GetAllocContext();
 
-    // NB: helper threads do not have contexts, so we must check
+    if (!IsHelperThread())
-    if (aContext)
     {
+        gc_alloc_context* aContext = GCToEEInterface::GetAllocContext();
         int threadScanTicket = VolatileLoadWithoutBarrier(&aContext->alloc_count);
         int currentScanTicket = GetRootScanTicket();
         if (threadScanTicket != currentScanTicket)
@@ -1352,7 +1364,7 @@ bool SatoriRecycler::MarkOwnStackAndDrainQueues(int64_t deadline)
             if (Interlocked::CompareExchange(&aContext->alloc_count, currentScanTicket, threadScanTicket) == threadScanTicket)
             {
                 MaybeAskForHelp();
-                MarkOwnStack(aContext, &c);
+                MarkOwnStack(aContext, &markContext);
             }
         }
     }
@@ -1368,14 +1380,14 @@ bool SatoriRecycler::MarkOwnStackAndDrainQueues(int64_t deadline)
             MaybeAskForHelp();
             do
             {
-                MarkDemoted(curRegion, c);
+                MarkDemoted(curRegion, &markContext);
                 PushToEphemeralQueuesIgnoringDemoted(curRegion);
 
                 if (deadline && ((GCToOSInterface::QueryPerformanceCounter() - deadline) > 0))
                 {
-                    if (c.m_WorkChunk != nullptr)
+                    if (markContext.m_WorkChunk != nullptr)
                     {
-                        m_workList->Push(c.m_WorkChunk);
+                        m_workList->Push(markContext.m_WorkChunk);
                     }
 
                     return true;
@@ -1387,17 +1399,17 @@ bool SatoriRecycler::MarkOwnStackAndDrainQueues(int64_t deadline)
     bool revisit = false;
     if (isBlockingPhase)
     {
-        DrainMarkQueues(c.m_WorkChunk);
+        DrainMarkQueues(markContext.m_WorkChunk);
     }
     else
     {
-        revisit = DrainMarkQueuesConcurrent(c.m_WorkChunk, deadline);
+        revisit = DrainMarkQueuesConcurrent(markContext.m_WorkChunk, deadline);
     }
 
     return revisit;
 }
 
-void SatoriRecycler::MarkOwnStack(gc_alloc_context* aContext, MarkContext* mc)
+void SatoriRecycler::MarkOwnStack(gc_alloc_context* aContext, MarkContext* markContext)
 {
     bool isBlockingPhase = IsBlockingPhase();
     if (!isBlockingPhase)
@@ -1411,7 +1423,7 @@ void SatoriRecycler::MarkOwnStack(gc_alloc_context* aContext, MarkContext* mc)
 
     ScanContext sc;
     sc.promotion = TRUE;
-    sc._unused1 = mc;
+    sc._unused1 = markContext;
 
     if (SatoriUtil::IsConservativeMode())
         GCToEEInterface::GcScanCurrentStackRoots(isBlockingPhase ? MarkFn<true> : MarkFnConcurrent<true>, &sc);
@@ -1419,7 +1431,7 @@ void SatoriRecycler::MarkOwnStack(gc_alloc_context* aContext, MarkContext* mc)
         GCToEEInterface::GcScanCurrentStackRoots(isBlockingPhase ? MarkFn<false> : MarkFnConcurrent<false>, &sc);
 }
 
-void SatoriRecycler::MarkDemoted(SatoriRegion* curRegion, MarkContext& c)
+void SatoriRecycler::MarkDemoted(SatoriRegion* curRegion, MarkContext* markContext)
 {
     _ASSERTE(curRegion->Generation() == 1);
 
@@ -1435,7 +1447,7 @@ void SatoriRecycler::MarkDemoted(SatoriRegion* curRegion, MarkContext& c)
             {
                 SatoriObject* o = gen2Objects->Item(i);
                 o->SetMarkedAtomic();
-                c.PushToMarkQueues(o);
+                markContext->PushToMarkQueues(o);
             }
 
             gen2Objects = gen2Objects->Next();
@@ -1452,8 +1464,8 @@ void SatoriRecycler::MarkAllStacksFinalizationAndDemotedRoots()
     // mark roots for all stacks
     ScanContext sc;
     sc.promotion = TRUE;
-    MarkContext c = MarkContext(this);
+    MarkContext markContext = MarkContext(this);
-    sc._unused1 = &c;
+    sc._unused1 = &markContext;
 
     bool isBlockingPhase = IsBlockingPhase();
 
@@ -1481,7 +1493,7 @@ void SatoriRecycler::MarkAllStacksFinalizationAndDemotedRoots()
                 if (!o->IsMarkedOrOlderThan(m_condemnedGeneration))
                 {
                     o->SetMarkedAtomic();
-                    c.PushToMarkQueues(o);
+                    markContext.PushToMarkQueues(o);
                 }
             }
         );
@@ -1497,7 +1509,7 @@ void SatoriRecycler::MarkAllStacksFinalizationAndDemotedRoots()
             MaybeAskForHelp();
             do
             {
-                MarkDemoted(curRegion, c);
+                MarkDemoted(curRegion, &markContext);
                 PushToEphemeralQueuesIgnoringDemoted(curRegion);
             } while ((curRegion = m_demotedRegions->TryPop()));
         }
@@ -1516,7 +1528,7 @@ void SatoriRecycler::MarkAllStacksFinalizationAndDemotedRoots()
             {
                 if (curRegion->IsDemoted() && curRegion->ReusableFor() != SatoriRegion::ReuseLevel::Gen0)
                 {
-                    MarkDemoted(curRegion, c);
+                    MarkDemoted(curRegion, &markContext);
                 }
 
                 if (curRegion->HasFreeSpaceInTopBucket())
@@ -1531,9 +1543,9 @@ void SatoriRecycler::MarkAllStacksFinalizationAndDemotedRoots()
         }
     }
 
-    if (c.m_WorkChunk != nullptr)
+    if (markContext.m_WorkChunk != nullptr)
     {
-        m_workList->Push(c.m_WorkChunk);
+        m_workList->Push(markContext.m_WorkChunk);
     }
 }
 
@@ -1557,7 +1569,7 @@ bool SatoriRecycler::DrainMarkQueuesConcurrent(SatoriWorkChunk* srcChunk, int64_
     auto markChildFn = [&](SatoriObject** ref)
     {
         SatoriObject* child = VolatileLoadWithoutBarrier(ref);
-        if (child)
+        if (child && !child->IsExternal())
         {
             objectCount++;
             SatoriRegion* childRegion = child->ContainingRegion();
@@ -1741,7 +1753,9 @@ void SatoriRecycler::DrainMarkQueues(SatoriWorkChunk* srcChunk)
     auto markChildFn = [&](SatoriObject** ref)
     {
         SatoriObject* child = *ref;
-        if (child && !child->IsMarkedOrOlderThan(m_condemnedGeneration))
+        if (child &&
+            !child->IsExternal() &&
+            !child->IsMarkedOrOlderThan(m_condemnedGeneration))
         {
             child->SetMarkedAtomic();
             // put more work, if found, into dstChunk
@@ -1916,7 +1930,7 @@ bool SatoriRecycler::MarkThroughCardsConcurrent(int64_t deadline)
                                     [&](SatoriObject** ref)
                                     {
                                         SatoriObject* child = VolatileLoadWithoutBarrier(ref);
-                                        if (child)
+                                        if (child && !child->IsExternal())
                                         {
                                             SatoriRegion* childRegion = child->ContainingRegion();
                                             if (!childRegion->MaybeEscapeTrackingAcquire())
@@ -2063,7 +2077,7 @@ bool SatoriRecycler::ScanDirtyCardsConcurrent(int64_t deadline)
                                         [&](SatoriObject** ref)
                                         {
                                             SatoriObject* child = VolatileLoadWithoutBarrier(ref);
-                                            if (child)
+                                            if (child && !child->IsExternal())
                                             {
                                                 SatoriRegion* childRegion = child->ContainingRegion();
                                                 // cannot mark stuff in thread local regions. just mark as dirty to visit later.
@@ -2219,7 +2233,9 @@ void SatoriRecycler::MarkThroughCards()
                                     [&](SatoriObject** ref)
                                     {
                                         SatoriObject* child = VolatileLoadWithoutBarrier(ref);
-                                        if (child && !child->IsMarkedOrOlderThan(1))
+                                        if (child &&
+                                            !child->IsExternal() &&
+                                            !child->IsMarkedOrOlderThan(1))
                                         {
                                             child->SetMarkedAtomic();
                                             if (!dstChunk || !dstChunk->TryPush(child))
@@ -2343,7 +2359,9 @@ void SatoriRecycler::CleanCards()
                                         [&](SatoriObject** ref)
                                         {
                                             SatoriObject* child = *ref;
-                                            if (child && !child->IsMarkedOrOlderThan(m_condemnedGeneration))
+                                            if (child &&
+                                                !child->IsExternal() &&
+                                                !child->IsMarkedOrOlderThan(m_condemnedGeneration))
                                             {
                                                 child->SetMarkedAtomic();
                                                 if (!dstChunk || !dstChunk->TryPush(child))
@@ -2451,7 +2469,7 @@ void SatoriRecycler::UpdatePointersThroughCards()
                                     {
                                         // prevent re-reading o, someone else could be doing the same update.
                                         SatoriObject* child = VolatileLoadWithoutBarrier(ppObject);
-                                        if (child)
+                                        if (child && !child->IsExternal())
                                         {
                                             ptrdiff_t ptr = ((ptrdiff_t*)child)[-1];
                                             if (ptr < 0)
@@ -2497,6 +2515,7 @@ bool SatoriRecycler::MarkHandles(int64_t deadline)
         {
             // there is work for us, so maybe there is more
             MaybeAskForHelp();
+            _ASSERTE(m_condemnedGeneration > 0);
 
             sc.thread_number = p;
             GCScan::GcScanHandles(
@@ -2589,7 +2608,7 @@ void SatoriRecycler::ScanAllFinalizableRegionsWorker()
     }
 }
 
-void SatoriRecycler::ScanFinalizableRegions(SatoriRegionQueue* queue, MarkContext* c)
+void SatoriRecycler::ScanFinalizableRegions(SatoriRegionQueue* queue, MarkContext* markContext)
 {
     SatoriRegion* region = queue->TryPop();
     if (region)
@@ -2640,7 +2659,7 @@ void SatoriRecycler::ScanFinalizableRegions(SatoriRegionQueue* queue, MarkContex
                             else
                             {
                                 finalizable->SetMarkedAtomic();
-                                c->PushToMarkQueues(finalizable);
+                                markContext->PushToMarkQueues(finalizable);
 
                                 if (m_heap->FinalizationQueue()->TryScheduleForFinalization(finalizable))
                                 {
@@ -2790,7 +2809,7 @@ void SatoriRecycler::PromoteSurvivedHandlesAndFreeRelocatedRegionsWorker()
     {
         ScanContext sc;
         sc.promotion = TRUE;
-        // no need for context. we do not create more work here.
+        // no need for scan context. we do not create more work here.
         sc._unused1 = nullptr;
 
         SatoriHandlePartitioner::ForEachUnscannedPartition(

src/coreclr/gc/satori/SatoriRecycler.h

@@ -82,6 +82,11 @@ public:
     void ScheduleMarkAsChildRanges(SatoriObject* o);
     bool ScheduleUpdateAsChildRanges(SatoriObject* o);
 
+    inline bool IsBarrierConcurrent()
+    {
+        return m_isBarrierConcurrent;
+    }
+
 private:
     SatoriHeap* m_heap;
 
@@ -140,7 +145,7 @@ private:
     bool m_isLowLatencyMode;
     bool m_promoteAllRegions;  
     bool m_allowPromotingRelocations;
-    bool m_isBarrierConcurrent;
+    volatile bool m_isBarrierConcurrent;
 
     int m_prevCondemnedGeneration;
 
@@ -215,10 +220,10 @@ private:
     void IncrementCardScanTicket();
     uint8_t GetCardScanTicket();
 
-    void MarkOwnStack(gc_alloc_context* aContext, MarkContext* mc);
+    void MarkOwnStack(gc_alloc_context* aContext, MarkContext* markContext);
     void MarkThroughCards();
     bool MarkThroughCardsConcurrent(int64_t deadline);
-    void MarkDemoted(SatoriRegion* curRegion, MarkContext& c);
+    void MarkDemoted(SatoriRegion* curRegion, MarkContext* markContext);
     void MarkAllStacksFinalizationAndDemotedRoots();
 
     void PushToMarkQueuesSlow(SatoriWorkChunk*& currentWorkChunk, SatoriObject* o);
@@ -236,7 +241,7 @@ private:
     void LongWeakPtrScanWorker();
 
     void ScanFinalizables();
-    void ScanFinalizableRegions(SatoriRegionQueue* regions, MarkContext* c);
+    void ScanFinalizableRegions(SatoriRegionQueue* regions, MarkContext* markContext);
     void ScanAllFinalizableRegionsWorker();
     void QueueCriticalFinalizablesWorker();
 

src/coreclr/gc/satori/SatoriRegion.cpp

@@ -740,7 +740,7 @@ void SatoriRegion::MarkFn(PTR_PTR_Object ppObject, ScanContext* sc, uint32_t fla
 
 inline void SatoriRegion::PushToMarkStackIfHasPointers(SatoriObject* obj)
 {
-    _ASSERTE(obj->ContainingRegion() == this);
+    _ASSERTE(obj->SameRegion(this));
     _ASSERTE(!obj->GetNextInLocalMarkStack());
 
     if (obj->RawGetMethodTable()->ContainsPointersOrCollectible())
@@ -769,8 +769,8 @@ bool SatoriRegion::AnyExposed(size_t first, size_t length)
     _ASSERTE(length % 8 == 0);
 
     size_t last = first + length - sizeof(size_t);
-    _ASSERTE(((SatoriObject*)first)->ContainingRegion() == this);
+    _ASSERTE(((SatoriObject*)first)->SameRegion(this));
-    _ASSERTE(((SatoriObject*)last)->ContainingRegion() == this);
+    _ASSERTE(((SatoriObject*)last)->SameRegion(this));
 
     size_t bitmapIndexF;
     size_t maskF = (size_t)-1 << ((SatoriObject*)first)->GetMarkBitAndWord(&bitmapIndexF);
@@ -802,7 +802,7 @@ bool SatoriRegion::AnyExposed(size_t first, size_t length)
 void SatoriRegion::EscapeRecursively(SatoriObject* o)
 {
     _ASSERTE(this->IsEscapeTrackedByCurrentThread());
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
 
     if (IsEscaped(o))
     {
@@ -828,7 +828,7 @@ void SatoriRegion::EscapeRecursively(SatoriObject* o)
 
                 // recursively escape all currently reachable objects
                 SatoriObject* child = *ref;
-                if (child->ContainingRegion() == this && !IsEscaped(child))
+                if (child->SameRegion(this) && !IsEscaped(child))
                 {
                     SetEscaped(child);
                     m_escapedSize += child->Size();
@@ -857,7 +857,7 @@ void SatoriRegion::EscsapeAll()
 // used when escaping all objects in the region anyways
 void SatoriRegion::EscapeShallow(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
     _ASSERTE(!IsEscaped(o));
     _ASSERTE(!IsPinned(o));
 
@@ -1000,7 +1000,7 @@ void SatoriRegion::ThreadLocalMark()
             [this](SatoriObject** ref)
             {
                 SatoriObject* child = *ref;
-                if (child->ContainingRegion() == this && !IsMarked(child))
+                if (child->SameRegion(this) && !IsMarked(child))
                 {
                     SetMarked(child);
                     PushToMarkStackIfHasPointers(child);
@@ -1239,7 +1239,7 @@ void SatoriRegion::UpdateFn(PTR_PTR_Object ppObject, ScanContext* sc, uint32_t f
     if (reloc)
     {
         *ppObject = (Object*)(((size_t)*ppObject) - reloc);
-        _ASSERTE(((SatoriObject*)(*ppObject))->ContainingRegion() == region);
+        _ASSERTE(((SatoriObject*)(*ppObject))->SameRegion(region));
     }
 };
 
@@ -1281,13 +1281,13 @@ void SatoriRegion::ThreadLocalUpdatePointers()
                         SatoriObject* child = *ppObject;
                         // ignore objects otside of the current region, we are not relocating those.
                         // (this also rejects nulls)
-                        if (child->ContainingRegion() == this)
+                        if (child->SameRegion(this))
                         {
                             size_t reloc = child->GetLocalReloc();
                             if (reloc)
                             {
                                 *ppObject = (SatoriObject*)((size_t)*ppObject - reloc);
-                                _ASSERTE((*ppObject)->ContainingRegion() == this);
+                                _ASSERTE((*ppObject)->SameRegion(this));
                             }
                         }
                     }
@@ -1319,7 +1319,7 @@ void SatoriRegion::ThreadLocalUpdatePointers()
                 if (reloc)
                 {
                     finalizable = (SatoriObject*)((size_t)finalizable - reloc);
-                    _ASSERTE(finalizable->ContainingRegion() == this);
+                    _ASSERTE(finalizable->SameRegion(this));
                 }
 
                 (size_t&)finalizable |= finalizePending;
@@ -1529,7 +1529,7 @@ tryAgain:
 
 bool SatoriRegion::RegisterForFinalization(SatoriObject* finalizable)
 {
-    _ASSERTE(finalizable->ContainingRegion() == this);
+    _ASSERTE(finalizable->SameRegion(this));
     _ASSERTE(this->m_hasFinalizables || this->IsAttachedToAllocatingOwner());
 
     LockFinalizableTrackers();
@@ -1723,12 +1723,12 @@ void SatoriRegion::UpdateFinalizableTrackers()
         ForEachFinalizable(
             [&](SatoriObject* finalizable)
             {
-                if (finalizable->ContainingRegion() != this)
+                if (!finalizable->SameRegion(this))
                 {
                     ptrdiff_t ptr = ((ptrdiff_t*)finalizable)[-1];
                     _ASSERTE(finalizable->RawGetMethodTable() == ((SatoriObject*)-ptr)->RawGetMethodTable());
                     finalizable = (SatoriObject*)-ptr;
-                    _ASSERTE(finalizable->ContainingRegion() == this);
+                    _ASSERTE(finalizable->SameRegion(this));
                 }
 
                 return finalizable;

src/coreclr/gc/satori/SatoriRegion.inl

@@ -425,7 +425,7 @@ inline SatoriWorkChunk* &SatoriRegion::DemotedObjects()
 
 inline void SatoriRegion::SetMarked(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
 
     size_t word = o->Start();
     size_t bitmapIndex = (word >> 9) & (SatoriRegion::BITMAP_LENGTH - 1);
@@ -436,7 +436,7 @@ inline void SatoriRegion::SetMarked(SatoriObject* o)
 
 inline void SatoriRegion::SetMarkedAtomic(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
     _ASSERTE(!o->IsFree());
 
     size_t word = o->Start();
@@ -459,7 +459,7 @@ inline void SatoriRegion::SetMarkedAtomic(SatoriObject* o)
 
 inline void SatoriRegion::ClearMarked(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
 
     size_t word = o->Start();
     size_t bitmapIndex = (word >> 9) & (SatoriRegion::BITMAP_LENGTH - 1);
@@ -470,7 +470,7 @@ inline void SatoriRegion::ClearMarked(SatoriObject* o)
 
 inline bool SatoriRegion::IsMarked(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
 
     size_t word = o->Start();
     size_t bitmapIndex = (word >> 9) & (SatoriRegion::BITMAP_LENGTH - 1);
@@ -481,7 +481,7 @@ inline bool SatoriRegion::IsMarked(SatoriObject* o)
 
 inline bool SatoriRegion::CheckAndClearMarked(SatoriObject* o)
 {
-    _ASSERTE(o->ContainingRegion() == this);
+    _ASSERTE(o->SameRegion(this));
 
     size_t word = o->Start();
     size_t bitmapIndex = (word >> 9) & (SatoriRegion::BITMAP_LENGTH - 1);
@@ -625,7 +625,7 @@ void SatoriRegion::UpdatePointersInPromotedObjects()
             {
                 // prevent re-reading o, UpdatePointersThroughCards could be doing the same update.
                 SatoriObject* child = VolatileLoadWithoutBarrier(ppObject);
-                if (child)
+                if (child && !child->IsExternal())
                 {
                     ptrdiff_t ptr = ((ptrdiff_t*)child)[-1];
                     if (ptr < 0)

src/coreclr/gc/satori/SatoriTrimmer.cpp

@@ -41,7 +41,7 @@ SatoriTrimmer::SatoriTrimmer(SatoriHeap* heap)
 {
     m_lastGen2Count = 0;
     m_heap  = heap;
-    m_state = TRIMMER_STATE_RUNNING;
+    m_state = TRIMMER_STATE_STOPPED;
 
     m_gate = new (nothrow) GCEvent;
     m_gate->CreateAutoEventNoThrow(false);

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs

@@ -89,15 +89,6 @@ namespace System.Runtime
             return RhpEndNoGCRegion();
         }
 
-        //
-        // internalcalls for System.Runtime.__Finalizer.
-        //
-
-        // Fetch next object which needs finalization or return null if we've reached the end of the list.
-        [RuntimeImport(Redhawk.BaseName, "RhpGetNextFinalizableObject")]
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        internal static extern object RhpGetNextFinalizableObject();
-
         //
         // internalcalls for System.Runtime.InteropServices.GCHandle.
         //
@@ -278,6 +269,9 @@ namespace System.Runtime
         [DllImport(Redhawk.BaseName)]
         internal static extern void RhpSignalFinalizationComplete(uint fCount, int observedFullGcCount);
 
+        [DllImport(Redhawk.BaseName)]
+        internal static extern object RhpGetNextFinalizableObject();
+
         [DllImport(Redhawk.BaseName)]
         internal static extern ulong RhpGetTickCount64();
 

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/__Finalizer.cs

@@ -55,9 +55,10 @@ namespace System.Runtime
         {
             uint finalizerCount = 0;
             // Drain the queue of finalizable objects.
+            object? target = null;
             while (true)
             {
-                object target = InternalCalls.RhpGetNextFinalizableObject();
+                InternalCalls.RhpGetNextFinalizableObject(Unsafe.AsPointer(ref target));
                 if (target == null)
                     return finalizerCount;
 

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -40,8 +40,9 @@ set(COMMON_RUNTIME_SOURCES
     ${GC_DIR}/gcconfig.cpp
     ${GC_DIR}/gchandletable.cpp
     ${GC_DIR}/gccommon.cpp
-    ${GC_DIR}/gceewks.cpp
+# TODO: VS Satori
-    ${GC_DIR}/gcwks.cpp
+#    ${GC_DIR}/gceewks.cpp
+#    ${GC_DIR}/gcwks.cpp
     ${GC_DIR}/gcscan.cpp
     ${GC_DIR}/handletable.cpp
     ${GC_DIR}/handletablecache.cpp
@@ -52,11 +53,26 @@ set(COMMON_RUNTIME_SOURCES
 
     ${CLR_SRC_NATIVE_DIR}/minipal/cpufeatures.c
     ${CLR_SRC_NATIVE_DIR}/minipal/time.c
+    ${GC_DIR}/satori/SatoriGC.cpp
+    ${GC_DIR}/satori/SatoriHandlePartitioner.cpp
+    ${GC_DIR}/satori/SatoriHeap.cpp
+    ${GC_DIR}/satori/SatoriPage.cpp
+    ${GC_DIR}/satori/SatoriObject.cpp
+    ${GC_DIR}/satori/SatoriRegion.cpp
+    ${GC_DIR}/satori/SatoriRegionQueue.cpp
+    ${GC_DIR}/satori/SatoriAllocator.cpp
+    ${GC_DIR}/satori/SatoriRecycler.cpp
+    ${GC_DIR}/satori/SatoriTrimmer.cpp
+    ${GC_DIR}/satori/SatoriFinalizationQueue.cpp
+    ${GC_DIR}/satori/SatoriAllocationContext.cpp
+    ${GC_DIR}/satori/SatoriUtil.cpp
+    ${GC_DIR}/satori/SatoriLock.cpp
 )
 
 set(SERVER_GC_SOURCES
-    ${GC_DIR}/gceesvr.cpp
+# TODO: VS Satori
-    ${GC_DIR}/gcsvr.cpp
+#    ${GC_DIR}/gceesvr.cpp
+#    ${GC_DIR}/gcsvr.cpp
 )
 
 set(STANDALONEGC_DISABLED_SOURCES
@@ -114,7 +130,26 @@ if (WIN32)
     ${GC_DIR}/handletable.inl
     ${GC_DIR}/handletablepriv.h
     ${GC_DIR}/objecthandle.h
-    ${GC_DIR}/softwarewritewatch.h)
+    ${GC_DIR}/softwarewritewatch.h
+    ${GC_DIR}/satori/SatoriGC.h
+    ${GC_DIR}/satori/SatoriHeap.h
+    ${GC_DIR}/satori/SatoriPage.h
+    ${GC_DIR}/satori/SatoriRegion.h
+    ${GC_DIR}/satori/SatoriRegion.inl
+    ${GC_DIR}/satori/SatoriObject.h
+    ${GC_DIR}/satori/SatoriObject.inl
+    ${GC_DIR}/satori/SatoriQueue.h
+    ${GC_DIR}/satori/SatoriRegionQueue.h
+    ${GC_DIR}/satori/SatoriAllocator.h
+    ${GC_DIR}/satori/SatoriRecycler.h
+    ${GC_DIR}/satori/SatoriTrimmer.h
+    ${GC_DIR}/satori/SatoriFinalizationQueue.h
+    ${GC_DIR}/satori/SatoriWorkChunk.h
+    ${GC_DIR}/satori/SatoriWorkList.h
+    ${GC_DIR}/satori/SatoriAllocationContext.h
+    ${GC_DIR}/satori/SatoriUtil.h
+    ${GC_DIR}/satori/SatoriLock.h
+)
 
   include_directories(windows)
 
@@ -236,6 +271,8 @@ if (NOT CLR_CMAKE_TARGET_ARCH_WASM)
   add_definitions(-DFEATURE_HIJACK)
 endif()
 
+add_definitions(-DFEATURE_SATORI_GC)
+add_definitions(-DFEATURE_SATORI_EXTERNAL_OBJECTS)
 add_definitions(-DFEATURE_BASICFREEZE)
 add_definitions(-DFEATURE_CONSERVATIVE_GC)
 if(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)

src/coreclr/nativeaot/Runtime/FinalizerHelpers.cpp

@@ -219,24 +219,31 @@ EXTERN_C UInt32_BOOL QCALLTYPE RhpWaitForFinalizerRequest()
 // Fetch next object which needs finalization or return null if we've reached the end of the list.
 FCIMPL0(OBJECTREF, RhpGetNextFinalizableObject)
 {
+    Thread* pThread = ThreadStore::GetCurrentThread();
+    pThread->DeferTransitionFrame();
+    pThread->DisablePreemptiveMode();
+
+    OBJECTREF refNext = NULL;
     while (true)
     {
         // Get the next finalizable object. If we get back NULL we've reached the end of the list.
-        OBJECTREF refNext = GCHeapUtilities::GetGCHeap()->GetNextFinalizable();
+        refNext = GCHeapUtilities::GetGCHeap()->GetNextFinalizable();
-        if (refNext == NULL)
+        if (refNext != NULL)
-            return NULL;
-
-        // The queue may contain objects which have been marked as finalized already (via GC.SuppressFinalize()
-        // for instance). Skip finalization for these but reset the flag so that the object can be put back on
-        // the list with RegisterForFinalization().
-        if (refNext->GetHeader()->GetBits() & BIT_SBLK_FINALIZER_RUN)
         {
-            refNext->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
+            // The queue may contain objects which have been marked as finalized already (via GC.SuppressFinalize()
-            continue;
+            // for instance). Skip finalization for these but reset the flag so that the object can be put back on
+            // the list with RegisterForFinalization().
+            if (refNext->GetHeader()->GetBits() & BIT_SBLK_FINALIZER_RUN)
+            {
+                refNext->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
+                continue;
+            }
         }
 
-        // We've found the first finalizable object, return it to the caller.
+        break;
-        return refNext;
     }
+
+    pThread->EnablePreemptiveMode();
+    return refNext;
 }
 FCIMPLEND

src/coreclr/nativeaot/Runtime/Full/CMakeLists.txt

@@ -26,7 +26,8 @@ if (CLR_CMAKE_TARGET_WIN32)
   endif()
 endif (CLR_CMAKE_TARGET_WIN32)
 
-add_library(Runtime.WorkstationGC STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIME_SOURCES} ${RUNTIME_SOURCES_ARCH_ASM} ${RUNTIME_ARCH_ASM_OBJECTS})
+# TODO: Satori WKS does not really exist, do the same as SVR
+add_library(Runtime.WorkstationGC STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIME_SOURCES} ${RUNTIME_SOURCES_ARCH_ASM} ${SERVER_GC_SOURCES} ${RUNTIME_ARCH_ASM_OBJECTS})
 add_dependencies(Runtime.WorkstationGC aot_eventing_headers)
 
 add_library(Runtime.ServerGC STATIC ${COMMON_RUNTIME_SOURCES} ${FULL_RUNTIME_SOURCES} ${RUNTIME_SOURCES_ARCH_ASM} ${SERVER_GC_SOURCES} ${RUNTIME_ARCH_ASM_OBJECTS})
@@ -47,6 +48,7 @@ if (CLR_CMAKE_TARGET_ARCH_AMD64)
 endif (CLR_CMAKE_TARGET_ARCH_AMD64)
 
 target_compile_definitions(Runtime.ServerGC PRIVATE -DFEATURE_SVR_GC)
+target_compile_definitions(Runtime.WorkstationGC PRIVATE -DFEATURE_SVR_GC)
 
 if (CLR_CMAKE_TARGET_WIN32)
   add_library(standalonegc-disabled.GuardCF STATIC ${STANDALONEGC_DISABLED_SOURCES})

src/coreclr/nativeaot/Runtime/GCHelpers.cpp

@@ -399,6 +399,9 @@ MethodTable* GetLastAllocEEType()
 
 FCIMPL0(int64_t, RhGetTotalAllocatedBytes)
 {
+#if FEATURE_SATORI_GC
+    return GCHeapUtilities::GetGCHeap()->GetTotalAllocatedBytes();
+#else
     uint64_t allocated_bytes = GCHeapUtilities::GetGCHeap()->GetTotalAllocatedBytes() - Thread::GetDeadThreadsNonAllocBytes();
 
     // highest reported allocated_bytes. We do not want to report a value less than that even if unused_bytes has increased.
@@ -415,6 +418,7 @@ FCIMPL0(int64_t, RhGetTotalAllocatedBytes)
     }
 
     return current_high;
+#endif
 }
 FCIMPLEND
 
@@ -450,6 +454,16 @@ EXTERN_C void QCALLTYPE RhEnableNoGCRegionCallback(NoGCRegionCallbackFinalizerWo
 
 EXTERN_C int64_t QCALLTYPE RhGetTotalAllocatedBytesPrecise()
 {
+#if FEATURE_SATORI_GC
+    Thread* pThread = ThreadStore::GetCurrentThread();
+    pThread->DeferTransitionFrame();
+    pThread->DisablePreemptiveMode();
+
+    GCHeapUtilities::GetGCHeap()->GarbageCollect(1);
+
+    pThread->EnablePreemptiveMode();
+    return GCHeapUtilities::GetGCHeap()->GetTotalAllocatedBytes();
+#else
     int64_t allocated;
 
     // We need to suspend/restart the EE to get each thread's
@@ -469,6 +483,7 @@ EXTERN_C int64_t QCALLTYPE RhGetTotalAllocatedBytesPrecise()
     GCToEEInterface::RestartEE(true);
 
     return allocated;
+#endif
 }
 
 static Object* GcAllocInternal(MethodTable* pEEType, uint32_t uFlags, uintptr_t numElements, Thread* pThread)

src/coreclr/nativeaot/Runtime/GCMemoryHelpers.cpp

@@ -7,6 +7,7 @@
 
 #include "common.h"
 #include "gcenv.h"
+#include "gcheaputilities.h"
 #include "PalRedhawkCommon.h"
 #include "CommonMacros.inl"
 #include "GCMemoryHelpers.inl"
@@ -44,6 +45,9 @@ FCIMPLEND
 
 FCIMPL3(void, RhBulkMoveWithWriteBarrier, uint8_t* pDest, uint8_t* pSrc, size_t cbDest)
 {
+#ifdef FEATURE_SATORI_GC
+    GCHeapUtilities::GetGCHeap()->BulkMoveWithWriteBarrier(pDest, pSrc, cbDest);
+#else
     if (cbDest == 0 || pDest == pSrc)
         return;
 
@@ -67,5 +71,5 @@ FCIMPL3(void, RhBulkMoveWithWriteBarrier, uint8_t* pDest, uint8_t* pSrc, size_t
     {
         InlinedBulkWriteBarrier(pDest, cbDest);
     }
+#endif //FEATURE_SATORI_GC
 }
-FCIMPLEND

src/coreclr/nativeaot/Runtime/GCMemoryHelpers.inl

@@ -170,6 +170,7 @@ extern "C" {
 }
 static const uint32_t INVALIDGCVALUE = 0xcccccccd;
 
+#if !FEATURE_SATORI_GC
 FORCEINLINE void InlineWriteBarrier(void * dst, void * ref)
 {
     ASSERT(((uint8_t*)dst >= g_lowest_address) && ((uint8_t*)dst < g_highest_address))
@@ -183,6 +184,7 @@ FORCEINLINE void InlineWriteBarrier(void * dst, void * ref)
             *pCardByte = 0xFF;
     }
 }
+#endif  //!FEATURE_SATORI_GC
 
 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 extern "C" uint32_t* g_card_bundle_table;
@@ -216,6 +218,7 @@ inline static void SoftwareWriteWatchSetDirtyRegion(void* address, size_t length
 }
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 
+#if !FEATURE_SATORI_GC
 FORCEINLINE void InlineCheckedWriteBarrier(void * dst, void * ref)
 {
     // if the dst is outside of the heap (unboxed value classes) then we
@@ -225,9 +228,14 @@ FORCEINLINE void InlineCheckedWriteBarrier(void * dst, void * ref)
 
     InlineWriteBarrier(dst, ref);
 }
+#endif  //!FEATURE_SATORI_GC
 
 FORCEINLINE void InlinedBulkWriteBarrier(void* pMemStart, size_t cbMemSize)
 {
+#if FEATURE_SATORI_GC
+    ASSERT(!"InlinedBulkWriteBarrier, should use GC helper instead.");
+#endif
+
     // Caller is expected to check whether the writes were even into the heap
     ASSERT(cbMemSize >= sizeof(uintptr_t));
     ASSERT((pMemStart >= g_lowest_address) && (pMemStart < g_highest_address));

src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc

@@ -407,4 +407,5 @@ endif
 
 ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 EXTERN g_write_watch_table  : QWORD
+EXTERN g_sw_ww_enabled_for_gc_heap  : BYTE
 endif

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S

@@ -4,6 +4,8 @@
 .intel_syntax noprefix
 #include <unixasmmacros.inc>
 
+#ifndef FEATURE_SATORI_GC
+
 #ifdef WRITE_BARRIER_CHECK
 
 .macro UPDATE_GC_SHADOW BASENAME, REFREG, DESTREG
@@ -326,3 +328,506 @@ LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired):
     add     rsi, 0x8
     ret
 LEAF_END RhpByRefAssignRef, _TEXT
+
+#else  //FEATURE_SATORI_GC   ######################################################
+
+//
+//   rdi - dest address 
+//   rsi - object
+//
+LEAF_ENTRY RhpCheckedAssignRef, _TEXT
+
+        // See if this is in GCHeap
+        mov     rax, rdi
+        shr     rax, 30                    // round to page size ( >> PAGE_BITS )
+        add     rax, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rax], 0
+        jne     C_FUNC(RhpAssignRef)
+
+NotInHeap:
+ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+    mov     [rdi], rsi
+    ret
+LEAF_END RhpCheckedAssignRef, _TEXT
+
+//
+//   rdi - dest address 
+//   rsi - object
+//
+.balign 16
+LEAF_ENTRY RhpAssignRef, _TEXT
+    // check for escaping assignment
+    // 1) check if we own the source region
+
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rax, rsi
+        shr     rax, 30                 // round to page size ( >> PAGE_BITS )
+        add     rax, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rax], 0
+        je      JustAssign              // src not in heap
+#endif
+
+        mov     rdx, rsi
+        and     rdx, 0xFFFFFFFFFFE00000  // source region
+
+#ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign              // assigning null
+#endif
+
+#ifdef TARGET_OSX
+        mov     rax,  gs:[0]            // thread tag
+#else
+        mov     rax,  fs:[0]            // thread tag
+#endif
+        cmp     qword ptr [rdx], rax     
+        jne     AssignAndMarkCards      // no card marking, src is not a heap object
+
+    // 2) check if the src and dst are from the same region
+        mov     rax, rsi
+        xor     rax, rdi
+        shr     rax, 21
+        jnz     RecordEscape            // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        mov     rax, rdi
+        and     rax, 0x1FFFFF
+        shr     rax, 3
+        bt      qword ptr [rdx], rax
+        jb      RecordEscape            // target is exposed. record an escape.
+
+    JustAssign:
+ALTERNATE_ENTRY RhpAssignRefAVLocationNotHeap
+       mov     [rdi], rsi              // no card marking, src is not a heap object
+        // set rdi, rsi per contract with JIT_ByRefWriteBarrier
+        add     rdi, 8
+        mov     rsi, r10
+        ret
+
+    AssignAndMarkCards:
+ALTERNATE_ENTRY RhpAssignRefAVLocation
+        mov     [rdi], rsi
+
+        // set rdi per contract with JIT_ByRefWriteBarrier
+        mov     rax, rdi
+        add     rdi, 8
+
+        xor     rsi, rdi
+        shr     rsi, 21
+
+        // set rsi per contract with JIT_ByRefWriteBarrier
+        mov     rsi, r10
+
+        jz      CheckConcurrent         // same region, just check if barrier is not concurrent
+
+
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [rdx + 16], 2
+        jl      MarkCards
+
+    CheckConcurrent:
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     MarkCards
+        ret
+
+    MarkCards:
+    // fetch card location for rax  (saved rdi)
+        mov     r9 , [C_VAR(g_card_table)]     // fetch the page map
+        mov     rdx,  rax
+        shr     rax, 30
+        mov     rax, qword ptr [r9 + rax * 8] // page
+        sub     rdx, rax   // offset in page
+        mov     r8 ,rdx
+        shr     rdx, 9     // card offset
+        shr     r8 , 21   // group offset
+
+    // check if concurrent marking is in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard
+
+    // SETTING CARD
+     SetCard:
+        cmp     byte ptr [rax + rdx], 0
+        jne     CardSet
+        mov     byte ptr [rax + rdx], 1
+     SetGroup:
+        cmp     byte ptr [rax + r8 * 2 + 0x80], 0
+        jne     CardSet
+        mov     byte ptr [rax + r8 * 2 + 0x80], 1
+     SetPage:
+        cmp     byte ptr [rax], 0
+        jne     CardSet
+        mov     byte ptr [rax], 1
+
+     CardSet:
+    // check if concurrent marking is still not in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard
+        ret
+
+    // DIRTYING CARD
+     DirtyCard:
+        mov     byte ptr [rax + rdx], 3
+     DirtyGroup:
+        mov     byte ptr [rax + r8 * 2 + 0x80], 3
+     DirtyPage:
+        mov     byte ptr [rax], 3
+
+    Exit:
+        ret
+
+    // this is expected to be rare.
+    RecordEscape:
+
+        // 4) check if the source is escaped
+        mov     rax, rsi
+        and     rax, 0x1FFFFF
+        shr     rax, 3
+        bt      qword ptr [rdx], rax
+        jb      AssignAndMarkCards            // source is already escaped.
+
+        // save rdi, rsi, rdx and r10 (possibly preadjusted rsi)
+        push rdi
+        push rsi
+        push rdx
+        push r10
+
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [rdx + 8]
+
+        pop     r10
+        pop     rdx
+        pop     rsi
+        pop     rdi
+        jmp     AssignAndMarkCards
+LEAF_END RhpAssignRef, _TEXT
+
+//
+// RhpByRefAssignRef simulates movs instruction for object references.
+//
+// On entry:
+//      rdi: address of ref-field (assigned to)
+//      rsi: address of the data (source)
+//
+// On exit:
+//      rdi, rsi are incremented by 8,
+//      rdi, rdx, r9, r10, r11: trashed
+//
+LEAF_ENTRY RhpByRefAssignRef, _TEXT
+        lea     r10, [rsi + 8]
+        mov     rsi, [rsi]
+
+        // See if assignment is into heap
+        mov     rax, rdi
+        shr     rax, 30                    // round to page size ( >> PAGE_BITS )
+        add     rax, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rax], 0
+        jne     C_FUNC(RhpAssignRef)
+
+    .balign 16
+    NotInHeap_RhpByRefAssignRef:
+        mov     [rdi], rsi
+        add     rdi, 8
+        mov     rsi, r10
+        ret
+LEAF_END RhpByRefAssignRef, _TEXT
+
+LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT
+    // Setup rax with the new object for the exchange, that way it will automatically hold the correct result
+    // afterwards and we can leave rsi unaltered ready for the GC write barrier below.
+    mov             rax, rdx
+
+    // check if dst is in heap
+        mov     rdx, rdi
+        shr     rdx, 30                    // round to page size ( >> PAGE_BITS )
+        add     rdx, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rdx], 0
+        je      JustAssign_CmpXchg              // dst not in heap
+
+    // check for escaping assignment
+    // 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rdx, rsi
+        shr     rdx, 30                  // round to page size ( >> PAGE_BITS )
+        add     rdx, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rdx], 0
+        je      JustAssign_CmpXchg              // src not in heap
+#endif
+
+        mov     rdx, rsi
+        and     rdx, 0xFFFFFFFFFFE00000  // source region
+
+#ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign_CmpXchg              // assigning null
+#endif
+
+#ifdef TARGET_OSX
+        mov     r11,  gs:[0]            // thread tag
+#else
+        mov     r11,  fs:[0]            // thread tag
+#endif
+        cmp     qword ptr [rdx], r11     
+        jne     AssignAndMarkCards_CmpXchg      // not local to this thread
+
+    // 2) check if the src and dst are from the same region
+        mov     r11, rsi
+        xor     r11, rdi
+        shr     r11, 21
+        jnz     RecordEscape_CmpXchg            // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        mov     r11, rdi
+        and     r11, 0x1FFFFF
+        shr     r11, 3
+        bt      qword ptr [rdx], r11
+        jb      RecordEscape_CmpXchg            // target is exposed. record an escape.
+
+    JustAssign_CmpXchg:
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocationNotHeap
+        lock cmpxchg    [rdi], rsi     // no card marking, src is not a heap object
+        ret
+
+    AssignAndMarkCards_CmpXchg:
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
+        lock cmpxchg    [rdi], rsi
+        jne             Exit_CmpXchg
+
+        xor     rsi, rdi
+        shr     rsi, 21
+        jz      CheckConcurrent_CmpXchg         // same region, just check if barrier is not concurrent
+
+    // TUNING: nonconcurrent and concurrent barriers could be separate pieces of code, but to switch 
+    //         need to suspend EE, not sure if skipping concurrent check would worth that much.
+
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [rdx + 16], 2
+        jl      MarkCards_CmpXchg
+
+    CheckConcurrent_CmpXchg:
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     MarkCards_CmpXchg
+        ret
+
+    MarkCards_CmpXchg:
+    // fetch card location for rdi
+        mov     r9 , [C_VAR(g_card_table)]     // fetch the page map
+        mov     rdx,  rdi
+        shr     rdi, 30
+        mov     r11, qword ptr [r9 + rdi * 8] // page
+        sub     rdx, r11   // offset in page
+        mov     rsi,rdx
+        shr     rdx, 9     // card offset
+        shr     rsi, 21   // group offset
+
+    // check if concurrent marking is in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard_CmpXchg
+
+    // SETTING CARD FOR rdi
+     SetCard_CmpXchg:
+        cmp     byte ptr [r11 + rdx], 0
+        jne     CardSet_CmpXchg
+        mov     byte ptr [r11 + rdx], 1
+     SetGroup_CmpXchg:
+        cmp     byte ptr [r11 + rsi * 2 + 0x80], 0
+        jne     CardSet_CmpXchg
+        mov     byte ptr [r11 + rsi * 2 + 0x80], 1
+     SetPage_CmpXchg:
+        cmp     byte ptr [r11], 0
+        jne     CardSet_CmpXchg
+        mov     byte ptr [r11], 1
+
+     CardSet_CmpXchg:
+    // check if concurrent marking is still not in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard_CmpXchg
+        ret
+
+    // DIRTYING CARD FOR rdi
+     DirtyCard_CmpXchg:
+        mov     byte ptr [r11 + rdx], 3
+     DirtyGroup_CmpXchg:
+        mov     byte ptr [r11 + rsi * 2 + 0x80], 3
+     DirtyPage_CmpXchg:
+        mov     byte ptr [r11], 3
+
+    Exit_CmpXchg:
+        ret
+
+    // this is expected to be rare.
+    RecordEscape_CmpXchg:
+
+        // 4) check if the source is escaped
+        mov     r11, rsi
+        and     r11, 0x1FFFFF
+        shr     r11, 3
+        bt      qword ptr [rdx], r11
+        jb      AssignAndMarkCards_CmpXchg            // source is already escaped.
+
+        // save rax, rdi, rsi, rdx and have enough stack for the callee
+        push rax
+        push rdi
+        push rsi
+        push rdx
+        sub  rsp, 0x20
+
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [rdx + 8]
+
+        add     rsp, 0x20
+        pop     rdx
+        pop     rsi
+        pop     rdi
+        pop     rax
+        jmp     AssignAndMarkCards_CmpXchg
+LEAF_END RhpCheckedLockCmpXchg, _TEXT
+
+LEAF_ENTRY RhpCheckedXchg, _TEXT
+    // Setup rax with the new object for the exchange, that way it will automatically hold the correct result
+    // afterwards and we can leave rsi unaltered ready for the GC write barrier below.
+    mov             rax, rsi
+
+        // check if dst is in heap
+        mov     rdx, rdi
+        shr     rdx, 30                    // round to page size ( >> PAGE_BITS )
+        add     rdx, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rdx], 0
+        je      JustAssign_Xchg              // dst not in heap
+
+    // check for escaping assignment
+    // 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rdx, rsi
+        shr     rdx, 30                 // round to page size ( >> PAGE_BITS )
+        add     rdx, [C_VAR(g_card_bundle_table)] // fetch the page byte map
+        cmp     byte ptr [rdx], 0
+        je      JustAssign_Xchg              // src not in heap
+#endif
+
+        mov     rdx, rsi
+        and     rdx, 0xFFFFFFFFFFE00000  // source region
+
+#ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign_Xchg              // assigning null
+#endif
+#ifdef TARGET_OSX
+        mov     r11,  gs:[0]            // thread tag
+#else
+        mov     r11,  fs:[0]            // thread tag
+#endif
+        cmp     qword ptr [rdx], r11     
+        jne     AssignAndMarkCards_Xchg      // not local to this thread
+
+    // 2) check if the src and dst are from the same region
+        mov     r11, rsi
+        xor     r11, rdi
+        shr     r11, 21
+        jnz     RecordEscape_Xchg            // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        mov     r11, rdi
+        and     r11, 0x1FFFFF
+        shr     r11, 3
+        bt      qword ptr [rdx], r11
+        jb      RecordEscape_Xchg            // target is exposed. record an escape.
+
+    JustAssign_Xchg:
+ALTERNATE_ENTRY RhpCheckedXchgAVLocationNotHeap
+        xchg    [rdi], rax              // no card marking, src is not a heap object
+        ret
+
+    AssignAndMarkCards_Xchg:
+ALTERNATE_ENTRY RhpCheckedXchgAVLocation
+        xchg    [rdi], rax
+
+        xor     rsi, rdi
+        shr     rsi, 21
+        jz      CheckConcurrent_Xchg         // same region, just check if barrier is not concurrent
+
+    // TUNING: nonconcurrent and concurrent barriers could be separate pieces of code, but to switch 
+    //         need to suspend EE, not sure if skipping concurrent check would worth that much.
+
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [rdx + 16], 2
+        jl      MarkCards_Xchg
+
+    CheckConcurrent_Xchg:
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     MarkCards_Xchg
+        ret
+
+    MarkCards_Xchg:
+    // fetch card location for rdi
+        mov     r9 , [C_VAR(g_card_table)]     // fetch the page map
+        mov     rdx,  rdi
+        shr     rdi, 30
+        mov     r11, qword ptr [r9 + rdi * 8] // page
+        sub     rdx, r11   // offset in page
+        mov     rsi,rdx
+        shr     rdx, 9     // card offset
+        shr     rsi, 21   // group offset
+
+    // check if concurrent marking is in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard_Xchg
+
+    // SETTING CARD FOR rdi
+     SetCard_Xchg:
+        cmp     byte ptr [r11 + rdx], 0
+        jne     CardSet_Xchg
+        mov     byte ptr [r11 + rdx], 1
+     SetGroup_Xchg:
+        cmp     byte ptr [r11 + rsi * 2 + 0x80], 0
+        jne     CardSet_Xchg
+        mov     byte ptr [r11 + rsi * 2 + 0x80], 1
+     SetPage_Xchg:
+        cmp     byte ptr [r11], 0
+        jne     CardSet_Xchg
+        mov     byte ptr [r11], 1
+
+     CardSet_Xchg:
+    // check if concurrent marking is still not in progress
+        cmp     byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0
+        jne     DirtyCard_Xchg
+        ret
+
+    // DIRTYING CARD FOR rdi
+     DirtyCard_Xchg:
+        mov     byte ptr [r11 + rdx], 3
+     DirtyGroup_Xchg:
+        mov     byte ptr [r11 + rsi * 2 + 0x80], 3
+     DirtyPage_Xchg:
+        mov     byte ptr [r11], 3
+
+    Exit_Xchg:
+        ret
+
+    // this is expected to be rare.
+    RecordEscape_Xchg:
+
+        // 4) check if the source is escaped
+        mov     r11, rsi
+        and     r11, 0x1FFFFF
+        shr     r11, 3
+        bt      qword ptr [rdx], r11
+        jb      AssignAndMarkCards_Xchg            // source is already escaped.
+
+        // save rax, rdi, rsi, rdx and have enough stack for the callee
+        push rax
+        push rdi
+        push rsi
+        push rdx
+        sub  rsp, 0x20
+
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [rdx + 8]
+
+        add     rsp, 0x20
+        pop     rdx
+        pop     rsi
+        pop     rdi
+        pop     rax
+        jmp     AssignAndMarkCards_Xchg
+LEAF_END RhpCheckedXchg, _TEXT
+
+
+#endif  // FEATURE_SATORI_GC

src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm

@@ -3,6 +3,8 @@
 
 include AsmMacros.inc
 
+ifndef FEATURE_SATORI_GC
+
 ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
 ;; during garbage collections to verify that object references where never written to the heap without using a
 ;; write barrier. Note that we're potentially racing to update the shadow heap while other threads are writing
@@ -343,4 +345,486 @@ RhpByRefAssignRef_NoBarrierRequired:
     ret
 LEAF_END RhpByRefAssignRef, _TEXT
 
+else  ;FEATURE_SATORI_GC
+
+;
+;   rcx - dest address 
+;   rdx - object
+;
+LEAF_ENTRY RhpCheckedAssignRef, _TEXT
+
+        ; See if this is in GCHeap
+        mov     rax, rcx
+        shr     rax, 30                    ; round to page size ( >> PAGE_BITS )
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
+        jne     RhpAssignRef
+
+NotInHeap:
+ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+    mov     [rcx], rdx
+    ret
+LEAF_END RhpCheckedAssignRef, _TEXT
+
+;
+;   rcx - dest address 
+;   rdx - object
+;
+LEAF_ENTRY RhpAssignRef, _TEXT
+    align 16
+    ; check for escaping assignment
+    ; 1) check if we own the source region
+
+ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rax, rdx
+        shr     rax, 30                 ; round to page size ( >> PAGE_BITS )
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
+        je      JustAssign              ; src not in heap
+endif
+
+        mov     r8, rdx
+        and     r8, 0FFFFFFFFFFE00000h  ; source region
+
+ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign              ; assigning null
+endif
+
+        mov     rax,  gs:[30h]          ; thread tag, TEB on NT
+        cmp     qword ptr [r8], rax     
+        jne     AssignAndMarkCards      ; not local to this thread
+
+    ; 2) check if the src and dst are from the same region
+        mov     rax, rdx
+        xor     rax, rcx
+        shr     rax, 21
+        jnz     RecordEscape            ; cross region assignment. definitely escaping
+
+    ; 3) check if the target is exposed
+        mov     rax, rcx
+        and     rax, 01FFFFFh
+        shr     rax, 3
+        bt      qword ptr [r8], rax
+        jb      RecordEscape            ; target is exposed. record an escape.
+
+    JustAssign:
+ALTERNATE_ENTRY RhpAssignRefAVLocationNotHeap
+        mov     [rcx], rdx              ; threadlocal assignment of unescaped object
+        ret
+
+    AssignAndMarkCards:
+ALTERNATE_ENTRY RhpAssignRefAVLocation
+        mov     [rcx], rdx
+
+        xor     rdx, rcx
+        shr     rdx, 21
+        jz      CheckConcurrent         ; same region, just check if barrier is not concurrent
+
+    ; TUNING: nonconcurrent and concurrent barriers could be separate pieces of code, but to switch 
+    ;         need to suspend EE, not sure if skipping concurrent check would worth that much.
+
+    ; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [r8 + 16], 2
+        jl      MarkCards
+
+    CheckConcurrent:
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     MarkCards
+        ret
+
+    MarkCards:
+    ; fetch card location for rcx
+        mov     r9 , [g_card_table]     ; fetch the page map
+        mov     r8,  rcx
+        shr     rcx, 30
+        mov     rax, qword ptr [r9 + rcx * 8] ; page
+        sub     r8, rax   ; offset in page
+        mov     rdx,r8
+        shr     r8, 9     ; card offset
+        shr     rdx, 21   ; group offset
+
+    ; check if concurrent marking is in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+
+    ; SETTING CARD FOR RCX
+     SetCard:
+        cmp     byte ptr [rax + r8], 0
+        jne     CardSet
+        mov     byte ptr [rax + r8], 1
+     SetGroup:
+        cmp     byte ptr [rax + rdx * 2 + 80h], 0
+        jne     CardSet
+        mov     byte ptr [rax + rdx * 2 + 80h], 1
+     SetPage:
+        cmp     byte ptr [rax], 0
+        jne     CardSet
+        mov     byte ptr [rax], 1
+
+     CardSet:
+    ; check if concurrent marking is still not in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+        ret
+
+    ; DIRTYING CARD FOR RCX
+     DirtyCard:
+        mov     byte ptr [rax + r8], 3
+     DirtyGroup:
+        mov     byte ptr [rax + rdx * 2 + 80h], 3
+     DirtyPage:
+        mov     byte ptr [rax], 3
+
+    Exit:
+        ret
+
+    ; this is expected to be rare.
+    RecordEscape:
+
+        ; 4) check if the source is escaped
+        mov     rax, rdx
+        and     rax, 01FFFFFh
+        shr     rax, 3
+        bt      qword ptr [r8], rax
+        jb      AssignAndMarkCards            ; source is already escaped.
+
+        ; save rcx, rdx, r8 and have enough stack for the callee
+        push rcx
+        push rdx
+        push r8
+        sub  rsp, 20h
+
+        ; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [r8 + 8]
+
+        add     rsp, 20h
+        pop     r8
+        pop     rdx
+        pop     rcx
+        jmp     AssignAndMarkCards
+LEAF_END RhpAssignRef, _TEXT
+
+;;
+;; RhpByRefAssignRef simulates movs instruction for object references.
+;;
+;; On entry:
+;;      rdi: address of ref-field (assigned to)
+;;      rsi: address of the data (source)
+;;
+;; On exit:
+;;      rdi, rsi are incremented by 8,
+;;      rcx, r8, r9, r11: trashed
+;;
+LEAF_ENTRY RhpByRefAssignRef, _TEXT
+        mov     rcx, rdi
+        mov     rdx, [rsi]
+        add     rdi, 8h
+        add     rsi, 8h
+
+        ; See if assignment is into heap
+        mov     rax, rcx
+        shr     rax, 30                    ; round to page size ( >> PAGE_BITS )
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
+        jne     RhpAssignRef
+
+    align 16
+    NotInHeap:
+        mov     [rcx], rdx
+        ret
+LEAF_END RhpByRefAssignRef, _TEXT
+
+LEAF_ENTRY RhpCheckedLockCmpXchg, _TEXT
+    ;; Setup rax with the new object for the exchange, that way it will automatically hold the correct result
+    ;; afterwards and we can leave rdx unaltered ready for the GC write barrier below.
+    mov             rax, r8
+
+    ; check if dst is in heap
+        mov     r8, rcx
+        shr     r8, 30                    ; round to page size ( >> PAGE_BITS )
+        add     r8, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [r8], 0
+        je      JustAssign              ; dst not in heap
+
+    ; check for escaping assignment
+    ; 1) check if we own the source region
+ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     r8, rdx
+        shr     r8, 30                  ; round to page size ( >> PAGE_BITS )
+        add     r8, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [r8], 0
+        je      JustAssign              ; src not in heap
+endif
+
+        mov     r8, rdx
+        and     r8, 0FFFFFFFFFFE00000h  ; source region
+
+ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign              ; assigning null
+endif
+
+        mov     r11,  gs:[30h]          ; thread tag, TEB on NT
+        cmp     qword ptr [r8], r11     
+        jne     AssignAndMarkCards      ; not local to this thread
+
+    ; 2) check if the src and dst are from the same region
+        mov     r11, rdx
+        xor     r11, rcx
+        shr     r11, 21
+        jnz     RecordEscape            ; cross region assignment. definitely escaping
+
+    ; 3) check if the target is exposed
+        mov     r11, rcx
+        and     r11, 01FFFFFh
+        shr     r11, 3
+        bt      qword ptr [r8], r11
+        jb      RecordEscape            ; target is exposed. record an escape.
+
+    JustAssign:
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocationNotHeap
+        lock cmpxchg    [rcx], rdx      ; no card marking, src is not a heap object
+        ret
+
+    AssignAndMarkCards:
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
+        lock cmpxchg    [rcx], rdx
+        jne             Exit
+
+        xor     rdx, rcx
+        shr     rdx, 21
+        jz      CheckConcurrent         ; same region, just check if barrier is not concurrent
+
+    ; TUNING: nonconcurrent and concurrent barriers could be separate pieces of code, but to switch 
+    ;         need to suspend EE, not sure if skipping concurrent check would worth that much.
+
+    ; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [r8 + 16], 2
+        jl      MarkCards
+
+    CheckConcurrent:
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     MarkCards
+        ret
+
+    MarkCards:
+    ; fetch card location for rcx
+        mov     r9 , [g_card_table]     ; fetch the page map
+        mov     r8,  rcx
+        shr     rcx, 30
+        mov     r11, qword ptr [r9 + rcx * 8] ; page
+        sub     r8, r11   ; offset in page
+        mov     rdx,r8
+        shr     r8, 9     ; card offset
+        shr     rdx, 21   ; group offset
+
+    ; check if concurrent marking is in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+
+    ; SETTING CARD FOR RCX
+     SetCard:
+        cmp     byte ptr [r11 + r8], 0
+        jne     CardSet
+        mov     byte ptr [r11 + r8], 1
+     SetGroup:
+        cmp     byte ptr [r11 + rdx * 2 + 80h], 0
+        jne     CardSet
+        mov     byte ptr [r11 + rdx * 2 + 80h], 1
+     SetPage:
+        cmp     byte ptr [r11], 0
+        jne     CardSet
+        mov     byte ptr [r11], 1
+
+     CardSet:
+    ; check if concurrent marking is still not in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+        ret
+
+    ; DIRTYING CARD FOR RCX
+     DirtyCard:
+        mov     byte ptr [r11 + r8], 3
+     DirtyGroup:
+        mov     byte ptr [r11 + rdx * 2 + 80h], 3
+     DirtyPage:
+        mov     byte ptr [r11], 3
+
+    Exit:
+        ret
+
+    ; this is expected to be rare.
+    RecordEscape:
+
+        ; 4) check if the source is escaped
+        mov     r11, rdx
+        and     r11, 01FFFFFh
+        shr     r11, 3
+        bt      qword ptr [r8], r11
+        jb      AssignAndMarkCards            ; source is already escaped.
+
+        ; save rax, rcx, rdx, r8 and have enough stack for the callee
+        push rax
+        push rcx
+        push rdx
+        push r8
+        sub  rsp, 20h
+
+        ; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [r8 + 8]
+
+        add     rsp, 20h
+        pop     r8
+        pop     rdx
+        pop     rcx
+        pop     rax
+        jmp     AssignAndMarkCards
+LEAF_END RhpCheckedLockCmpXchg, _TEXT
+
+LEAF_ENTRY RhpCheckedXchg, _TEXT
+    ;; Setup rax with the new object for the exchange, that way it will automatically hold the correct result
+    ;; afterwards and we can leave rdx unaltered ready for the GC write barrier below.
+    mov             rax, rdx
+
+        ; check if dst is in heap
+        mov     r8, rcx
+        shr     r8, 30                    ; round to page size ( >> PAGE_BITS )
+        add     r8, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [r8], 0
+        je      JustAssign              ; dst not in heap
+
+    ; check for escaping assignment
+    ; 1) check if we own the source region
+ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     r8, rdx
+        shr     r8, 30                 ; round to page size ( >> PAGE_BITS )
+        add     r8, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [r8], 0
+        je      JustAssign              ; src not in heap
+endif
+
+        mov     r8, rdx
+        and     r8, 0FFFFFFFFFFE00000h  ; source region
+
+ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign              ; assigning null
+endif
+        mov     r11,  gs:[30h]          ; thread tag, TEB on NT
+        cmp     qword ptr [r8], r11     
+        jne     AssignAndMarkCards      ; not local to this thread
+
+    ; 2) check if the src and dst are from the same region
+        mov     r11, rdx
+        xor     r11, rcx
+        shr     r11, 21
+        jnz     RecordEscape            ; cross region assignment. definitely escaping
+
+    ; 3) check if the target is exposed
+        mov     r11, rcx
+        and     r11, 01FFFFFh
+        shr     r11, 3
+        bt      qword ptr [r8], r11
+        jb      RecordEscape            ; target is exposed. record an escape.
+
+    JustAssign:
+ALTERNATE_ENTRY RhpCheckedXchgAVLocationNotHeap
+        xchg    [rcx], rax              ; no card marking, src is not a heap object
+        ret
+
+    AssignAndMarkCards:
+ALTERNATE_ENTRY RhpCheckedXchgAVLocation
+        xchg    [rcx], rax
+
+        xor     rdx, rcx
+        shr     rdx, 21
+        jz      CheckConcurrent         ; same region, just check if barrier is not concurrent
+
+    ; TUNING: nonconcurrent and concurrent barriers could be separate pieces of code, but to switch 
+    ;         need to suspend EE, not sure if skipping concurrent check would worth that much.
+
+    ; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        cmp     dword ptr [r8 + 16], 2
+        jl      MarkCards
+
+    CheckConcurrent:
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     MarkCards
+        ret
+
+    MarkCards:
+    ; fetch card location for rcx
+        mov     r9 , [g_card_table]     ; fetch the page map
+        mov     r8,  rcx
+        shr     rcx, 30
+        mov     r11, qword ptr [r9 + rcx * 8] ; page
+        sub     r8, r11   ; offset in page
+        mov     rdx,r8
+        shr     r8, 9     ; card offset
+        shr     rdx, 21   ; group offset
+
+    ; check if concurrent marking is in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+
+    ; SETTING CARD FOR RCX
+     SetCard:
+        cmp     byte ptr [r11 + r8], 0
+        jne     CardSet
+        mov     byte ptr [r11 + r8], 1
+     SetGroup:
+        cmp     byte ptr [r11 + rdx * 2 + 80h], 0
+        jne     CardSet
+        mov     byte ptr [r11 + rdx * 2 + 80h], 1
+     SetPage:
+        cmp     byte ptr [r11], 0
+        jne     CardSet
+        mov     byte ptr [r11], 1
+
+     CardSet:
+    ; check if concurrent marking is still not in progress
+        cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
+        jne     DirtyCard
+        ret
+
+    ; DIRTYING CARD FOR RCX
+     DirtyCard:
+        mov     byte ptr [r11 + r8], 3
+     DirtyGroup:
+        mov     byte ptr [r11 + rdx * 2 + 80h], 3
+     DirtyPage:
+        mov     byte ptr [r11], 3
+
+    Exit:
+        ret
+
+    ; this is expected to be rare.
+    RecordEscape:
+
+        ; 4) check if the source is escaped
+        mov     r11, rdx
+        and     r11, 01FFFFFh
+        shr     r11, 3
+        bt      qword ptr [r8], r11
+        jb      AssignAndMarkCards            ; source is already escaped.
+
+        ; save rax, rcx, rdx, r8 and have enough stack for the callee
+        push rax
+        push rcx
+        push rdx
+        push r8
+        sub  rsp, 20h
+
+        ; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        call    qword ptr [r8 + 8]
+
+        add     rsp, 20h
+        pop     r8
+        pop     rdx
+        pop     rcx
+        pop     rax
+        jmp     AssignAndMarkCards
+LEAF_END RhpCheckedXchg, _TEXT
+
+
+endif  ; FEATURE_SATORI_GC
+
     end

src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S

@@ -3,6 +3,8 @@
 
 #include <unixasmmacros.inc>
 
+#ifndef FEATURE_SATORI_GC
+
 // Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
 // during garbage collections to verify that object references where never written to the heap without using a
 // write barrier. Note that we are potentially racing to update the shadow heap while other threads are writing
@@ -398,3 +400,624 @@ LOCAL_LABEL(NoBarrierXchg):
 #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT
 .arch_extension nolse
 #endif
+    LEAF_END RhpCheckedXchg, _TEXT
+
+#else  //FEATURE_SATORI_GC   ######################################################
+
+// void JIT_ByRefWriteBarrier
+// On entry:
+//   x13  : the source address (points to object reference to write)
+//   x14  : the destination address (object reference written here)
+//
+// On exit:
+//   x13  : incremented by 8
+//   x14  : incremented by 8
+//   x15  : trashed
+//   x12, x17  : trashed
+//
+LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
+
+        ldr     x15, [x13], 8
+        b       C_FUNC(RhpCheckedAssignRefArm64)
+
+LEAF_END RhpByRefAssignRefArm64, _TEXT
+
+// JIT_CheckedWriteBarrier(Object** dst, Object* src)
+//
+// Write barrier for writes to objects that may reside
+// on the managed heap.
+//
+// On entry:
+//   x14 : the destination address (LHS of the assignment).
+//         May not be a heap location (hence the checked).
+//   x15 : the object reference (RHS of the assignment).
+//
+// On exit:
+//   x12, x17 : trashed
+//   x14      : incremented by 8
+LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x14, lsr #30
+        ldrb    w12, [x12]
+        cbnz    x12, C_FUNC(RhpAssignRefArm64)
+
+    LOCAL_LABEL(NotInHeap):
+    ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+        str  x15, [x14], #8
+        ret  lr
+LEAF_END RhpCheckedAssignRefArm64, _TEXT
+
+// JIT_WriteBarrier(Object** dst, Object* src)
+//
+// Write barrier for writes to objects that are known to
+// reside on the managed heap.
+//
+// On entry:
+//  x14 : the destination address (LHS of the assignment).
+//  x15 : the object reference (RHS of the assignment).
+//
+// On exit:
+//  x12, x17 : trashed
+//  x14 : incremented by 8
+LEAF_ENTRY RhpAssignRefArm64, _TEXT
+    // check for escaping assignment
+    // 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x15, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, LOCAL_LABEL(JustAssign)
+#else
+        cbz     x15, LOCAL_LABEL(JustAssign)    // assigning null
+#endif
+        and     x12,  x15, #0xFFFFFFFFFFE00000  // source region
+        ldr     x12, [x12]                      // region tag
+#ifdef TARGET_OSX
+        mrs     x17, TPIDRRO_EL0
+        and     x17, x17, #-8                   // thread tag on OSX
+#else
+        mrs     x17, TPIDR_EL0                  // thread tag on other Unix
+#endif
+        cmp     x12, x17     
+        bne     LOCAL_LABEL(AssignAndMarkCards) // not local to this thread
+
+    // 2) check if the src and dst are from the same region
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbnz    x12, LOCAL_LABEL(RecordEscape)  // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        ubfx    x17, x14,#9,#12                 // word index = (dst >> 9) & 0x1FFFFF
+        and     x12, x15, #0xFFFFFFFFFFE00000   // source region
+        ldr     x17, [x12, x17, lsl #3]         // mark word = [region + index * 8]
+        lsr     x12, x14, #3                    // bit = (dst >> 3) [& 63]
+        lsr     x17, x17, x12
+        tbnz    x17, #0, LOCAL_LABEL(RecordEscape) // target is exposed. record an escape.
+
+        str     x15, [x14], #8                  // UNORDERED assignment of unescaped object
+        ret     lr
+
+    LOCAL_LABEL(JustAssign):
+ALTERNATE_ENTRY RhpAssignRefAVLocationNotHeap
+        stlr    x15, [x14]                      // no card marking, src is not a heap object
+        add     x14, x14, 8
+        ret     lr
+
+    LOCAL_LABEL(AssignAndMarkCards):
+ALTERNATE_ENTRY RhpAssignRefAVLocation
+        stlr    x15, [x14]
+
+    // need couple temps. Save before using.
+        stp     x2,  x3,  [sp, -16]!
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, LOCAL_LABEL(CheckConcurrent) // same region, just check if barrier is not concurrent
+
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     // source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, LOCAL_LABEL(MarkCards)
+
+    LOCAL_LABEL(CheckConcurrent):
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(MarkCards)
+        
+    LOCAL_LABEL(Exit):
+        ldp  x2,  x3, [sp], 16
+        add  x14, x14, 8
+        ret  lr
+
+    LOCAL_LABEL(MarkCards):
+    // fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  // fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              // page
+        sub     x2,  x14, x17   // offset in page
+        lsr     x15, x2,  #21   // group index
+        lsl     x15, x15, #1    // group offset (index * 2)
+        lsr     x2,  x2,  #9    // card offset
+
+    // check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard)
+
+    // SETTING CARD FOR X14
+    LOCAL_LABEL(SetCard):
+        ldrb    w3, [x17, x2]
+        cbnz    w3, LOCAL_LABEL(CardSet)
+        mov     w3, #1
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(SetGroup):
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, LOCAL_LABEL(CardSet)
+        mov     w3, #1
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(SetPage):
+        ldrb    w3, [x17]
+        cbnz    w3, LOCAL_LABEL(CardSet)
+        mov     w3, #1
+        strb    w3, [x17]
+
+    LOCAL_LABEL(CardSet):
+    // check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard)
+        b       LOCAL_LABEL(Exit)
+
+    // DIRTYING CARD FOR X14
+    LOCAL_LABEL(DirtyCard):
+        mov     w3, #3
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(DirtyGroup):
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(DirtyPage):
+        strb    w3, [x17]
+        b       LOCAL_LABEL(Exit)
+
+    // this is expected to be rare.
+    LOCAL_LABEL(RecordEscape):
+
+    // 4) check if the source is escaped
+        and         x12, x15, #0xFFFFFFFFFFE00000  // source region
+        add         x15, x15, #8                   // escape bit is MT + 1
+        ubfx        x17, x15, #9,#12               // word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        // mark word = [region + index * 8]
+        lsr         x12, x15, #3                   // bit = (dst >> 3) [& 63]
+        sub         x15, x15, #8     // undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, LOCAL_LABEL(AssignAndMarkCards)        // source is already escaped.
+
+        // because of the barrier call convention
+        // we need to preserve caller-saved x0 through x18 and x29/x30
+
+        stp     x29,x30, [sp, -16 * 10]!
+        stp     x0, x1,  [sp, 16 * 1]
+        stp     x2, x3,  [sp, 16 * 2]
+        stp     x4, x5,  [sp, 16 * 3]
+        stp     x6, x7,  [sp, 16 * 4]
+        stp     x8, x9,  [sp, 16 * 5]
+        stp     x10,x11, [sp, 16 * 6]
+        stp     x12,x13, [sp, 16 * 7]
+        stp     x14,x15, [sp, 16 * 8]
+        stp     x16,x17, [sp, 16 * 9]
+#ifndef TARGET_OSX
+        str     x18,     [sp, -16]!
+#endif
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        // mov  x0, x14  EscapeFn does not use dst, it is just to avoid arg shuffle on x64
+        mov  x1, x15
+        and  x2, x15, #0xFFFFFFFFFFE00000  // source region
+        ldr  x12, [x2, #8]                 // EscapeFn address
+        blr  x12
+
+#ifndef TARGET_OSX
+        ldr     x18,     [sp], 16
+#endif
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldp     x2, x3,  [sp, 16 * 2]
+        ldp     x4, x5,  [sp, 16 * 3]
+        ldp     x6, x7,  [sp, 16 * 4]
+        ldp     x8, x9,  [sp, 16 * 5]
+        ldp     x10,x11, [sp, 16 * 6]
+        ldp     x12,x13, [sp, 16 * 7]
+        ldp     x14,x15, [sp, 16 * 8]
+        ldp     x16,x17, [sp, 16 * 9]
+        ldp     x29,x30, [sp], 16 * 10
+
+        b       LOCAL_LABEL(AssignAndMarkCards)
+LEAF_END RhpAssignRefArm64, _TEXT
+
+// Same as RhpAssignRefArm64, but with standard ABI.
+LEAF_ENTRY RhpAssignRef, _TEXT
+        mov     x14, x0                     // x14 = dst
+        mov     x15, x1                     // x15 = val
+        b       C_FUNC(RhpAssignRefArm64)
+LEAF_END RhpAssignRef, _TEXT
+
+// RhpCheckedLockCmpXchg(Object** dest, Object* value, Object* comparand)
+//
+// Interlocked compare exchange on objectref.
+//
+// On entry:
+//  x0: pointer to objectref
+//  x1: exchange value
+//  x2: comparand
+//
+// On exit:
+//  x0: original value of objectref
+//  x10, x12, x17: trashed
+//
+LEAF_ENTRY RhpCheckedLockCmpXchg
+    // check if dst is in heap
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x0, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, LOCAL_LABEL(JustAssign_Cmp_Xchg)
+
+    // check for escaping assignment
+    // 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x1, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, LOCAL_LABEL(JustAssign_Cmp_Xchg)
+#else
+        cbz     x1, LOCAL_LABEL(JustAssign_Cmp_Xchg)    // assigning null
+#endif
+        and     x12,  x1, #0xFFFFFFFFFFE00000   // source region
+        ldr     x12, [x12]                      // region tag
+#ifdef TARGET_OSX
+        mrs     x17, TPIDRRO_EL0
+        and     x17, x17, #-8                   // thread tag on OSX
+#else
+        mrs     x17, TPIDR_EL0                  // thread tag on other Unix
+#endif
+        cmp     x12, x17     
+        bne     LOCAL_LABEL(AssignAndMarkCards_Cmp_Xchg) // not local to this thread
+
+    // 2) check if the src and dst are from the same region
+        eor     x12, x0, x1
+        lsr     x12, x12, #21
+        cbnz    x12, LOCAL_LABEL(RecordEscape_Cmp_Xchg)  // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        ubfx        x17, x0,#9,#12              // word index = (dst >> 9) & 0x1FFFFF
+        and         x12, x1, #0xFFFFFFFFFFE00000  // source region
+        ldr         x17, [x12, x17, lsl #3]     // mark word = [region + index * 8]
+        lsr         x12, x0, #3                 // bit = (dst >> 3) [& 63]
+        lsr         x17, x17, x12
+        tbnz        x17, #0, LOCAL_LABEL(RecordEscape_Cmp_Xchg) // target is exposed. record an escape.
+
+    LOCAL_LABEL(JustAssign_Cmp_Xchg):
+#ifdef TARGET_OSX
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocationNotHeap
+        casal  x2, x1, [x0]                  // exchange
+        mov    x0, x2                        // x0 = result
+    LOCAL_LABEL(NoUpdate_Cmp_Xchg):
+        ret    lr
+#else
+        mov     x14, x0
+    LOCAL_LABEL(TryAgain_Cmp_Xchg):
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocationNotHeap
+        ldaxr   x0, [x14]
+        cmp     x0, x2
+        bne     LOCAL_LABEL(NoUpdate_Cmp_Xchg)
+
+        // Current value matches comparand, attempt to update with the new value.
+        stlxr   w12, x1, [x14]
+        cbnz    w12, LOCAL_LABEL(TryAgain_Cmp_Xchg)  // if failed, try again
+
+    LOCAL_LABEL(NoUpdate_Cmp_Xchg):
+        dmb     ish
+        ret     lr
+#endif
+
+    LOCAL_LABEL(AssignAndMarkCards_Cmp_Xchg):
+        mov    x14, x0                       // x14 = dst
+        mov    x15, x1                       // x15 = val
+#ifdef TARGET_OSX
+        mov    x17, x2
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
+        casal  x2, x1, [x0]                  // exchange
+        mov    x0, x2                        // x0 = result
+        cmp    x2, x17
+        bne    LOCAL_LABEL(NoUpdate_Cmp_Xchg)
+#else
+    LOCAL_LABEL(TryAgain1_Cmp_Xchg):
+ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
+        ldaxr   x0, [x14]
+        cmp     x0, x2
+        bne     LOCAL_LABEL(NoUpdate_Cmp_Xchg)
+
+        // Current value matches comparand, attempt to update with the new value.
+        stlxr   w12, x1, [x14]
+        cbnz    w12, LOCAL_LABEL(TryAgain1_Cmp_Xchg)  // if failed, try again
+        dmb     ish
+#endif
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, LOCAL_LABEL(CheckConcurrent_Cmp_Xchg) // same region, just check if barrier is not concurrent
+
+    // we will trash x2 and x3, this is a regular call, so it is ok
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     // source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, LOCAL_LABEL(MarkCards_Cmp_Xchg)
+
+    LOCAL_LABEL(CheckConcurrent_Cmp_Xchg):
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(MarkCards_Cmp_Xchg)
+        
+    LOCAL_LABEL(Exit_Cmp_Xchg):
+        ret  lr
+
+    LOCAL_LABEL(MarkCards_Cmp_Xchg):
+    // fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  // fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              // page
+        sub     x2,  x14, x17   // offset in page
+        lsr     x15, x2,  #21   // group index
+        lsl     x15, x15, #1    // group offset (index * 2)
+        lsr     x2,  x2,  #9    // card offset
+
+    // check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard_Cmp_Xchg)
+
+    // SETTING CARD FOR X14
+    LOCAL_LABEL(SetCard_Cmp_Xchg):
+        ldrb    w3, [x17, x2]
+        cbnz    w3, LOCAL_LABEL(CardSet_Cmp_Xchg)
+        mov     w3, #1
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(SetGroup_Cmp_Xchg):
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, LOCAL_LABEL(CardSet_Cmp_Xchg)
+        mov     w3, #1
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(SetPage_Cmp_Xchg):
+        ldrb    w3, [x17]
+        cbnz    w3, LOCAL_LABEL(CardSet_Cmp_Xchg)
+        mov     w3, #1
+        strb    w3, [x17]
+
+    LOCAL_LABEL(CardSet_Cmp_Xchg):
+    // check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard_Cmp_Xchg)
+        b       LOCAL_LABEL(Exit_Cmp_Xchg)
+
+    // DIRTYING CARD FOR X14
+    LOCAL_LABEL(DirtyCard_Cmp_Xchg):
+        mov     w3, #3
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(DirtyGroup_Cmp_Xchg):
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(DirtyPage_Cmp_Xchg):
+        strb    w3, [x17]
+        b       LOCAL_LABEL(Exit_Cmp_Xchg)
+
+    // this is expected to be rare.
+    LOCAL_LABEL(RecordEscape_Cmp_Xchg):
+
+    // 4) check if the source is escaped
+        and         x12, x1, #0xFFFFFFFFFFE00000  // source region
+        add         x1, x1, #8                   // escape bit is MT + 1
+        ubfx        x17, x1, #9,#12               // word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        // mark word = [region + index * 8]
+        lsr         x12, x1, #3                   // bit = (dst >> 3) [& 63]
+        sub         x1, x1, #8     // undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, LOCAL_LABEL(AssignAndMarkCards_Cmp_Xchg)        // source is already escaped.
+
+        // we need to preserve our parameters x0, x1, x2 and x29/x30
+
+        stp     x29,x30, [sp, -16 * 3]!
+        stp     x0, x1,  [sp, 16 * 1]
+        str     x2,      [sp, 16 * 2]
+
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        and  x2, x1, #0xFFFFFFFFFFE00000  // source region
+        ldr  x12, [x2, #8]                 // EscapeFn address
+        blr  x12
+
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldr     x2,      [sp, 16 * 2]
+        ldp     x29,x30, [sp], 16 * 3
+
+        b       LOCAL_LABEL(AssignAndMarkCards_Cmp_Xchg)
+LEAF_END RhpCheckedLockCmpXchg, _TEXT
+
+// WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
+// - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen within at RhpCheckedXchgAVLocation
+// - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
+
+// RhpCheckedXchg(Object** destination, Object* value)
+//
+// Interlocked exchange on objectref.
+//
+// On entry:
+//  x0: pointer to objectref
+//  x1: exchange value
+//
+// On exit:
+//  x0: original value of objectref
+//  x10: trashed
+//  x12, x17: trashed
+//
+LEAF_ENTRY RhpCheckedXchg, _TEXT
+
+    // check if dst is in heap
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x0, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, LOCAL_LABEL(JustAssign_Xchg)
+
+    // check for escaping assignment
+    // 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x1, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, LOCAL_LABEL(JustAssign_Xchg)
+#else
+        cbz     x1, LOCAL_LABEL(JustAssign_Xchg)    // assigning null
+#endif
+        and     x12,  x1, #0xFFFFFFFFFFE00000  // source region
+        ldr     x12, [x12]                      // region tag
+#ifdef TARGET_OSX
+        mrs     x17, TPIDRRO_EL0
+        and     x17, x17, #-8                   // thread tag on OSX
+#else
+        mrs     x17, TPIDR_EL0                  // thread tag on other Unix
+#endif
+        cmp     x12, x17     
+        bne     LOCAL_LABEL(AssignAndMarkCards_Xchg) // not local to this thread
+
+    // 2) check if the src and dst are from the same region
+        eor     x12, x0, x1
+        lsr     x12, x12, #21
+        cbnz    x12, LOCAL_LABEL(RecordEscape_Xchg)  // cross region assignment. definitely escaping
+
+    // 3) check if the target is exposed
+        ubfx        x17, x0,#9,#12              // word index = (dst >> 9) & 0x1FFFFF
+        and         x12, x1, #0xFFFFFFFFFFE00000  // source region
+        ldr         x17, [x12, x17, lsl #3]      // mark word = [region + index * 8]
+        lsr         x12, x0, #3                 // bit = (dst >> 3) [& 63]
+        lsr         x17, x17, x12
+        tbnz        x17, #0, LOCAL_LABEL(RecordEscape_Xchg) // target is exposed. record an escape.
+
+    LOCAL_LABEL(JustAssign_Xchg):
+#ifdef TARGET_OSX
+ALTERNATE_ENTRY RhpCheckedXchgAVLocationNotHeap
+        swpal  x1, x0, [x0]                   // exchange
+#else
+    LOCAL_LABEL(TryAgain_Xchg):
+ALTERNATE_ENTRY RhpCheckedXchgAVLocationNotHeap
+        ldaxr   x17, [x0]
+        stlxr   w12, x1, [x0]
+        cbnz    w12, LOCAL_LABEL(TryAgain_Xchg)
+        mov     x0, x17
+        dmb     ish
+#endif
+        ret    lr
+
+    LOCAL_LABEL(AssignAndMarkCards_Xchg):
+        mov    x14, x0                        // x14 = dst
+        mov    x15, x1                        // x15 = val    // TODO: VS not needed, can use x1
+#ifdef TARGET_OSX
+ALTERNATE_ENTRY RhpCheckedXchgAVLocation
+        swpal  x1, x0, [x0]                   // exchange
+#else
+    LOCAL_LABEL(TryAgain1_Xchg):
+ALTERNATE_ENTRY RhpCheckedXchgAVLocation
+        ldaxr   x17, [x0]
+        stlxr   w12, x1, [x0]
+        cbnz    w12, LOCAL_LABEL(TryAgain1_Xchg)
+        mov     x0, x17
+        dmb     ish
+#endif
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, LOCAL_LABEL(CheckConcurrent_Xchg) // same region, just check if barrier is not concurrent
+
+    // we will trash x2 and x3, this is a regular call, so it is ok
+    // if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     // source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, LOCAL_LABEL(MarkCards_Xchg)
+
+    LOCAL_LABEL(CheckConcurrent_Xchg):
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(MarkCards_Xchg)
+        
+    LOCAL_LABEL(Exit_Xchg):
+        ret  lr
+
+    LOCAL_LABEL(MarkCards_Xchg):
+    // fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  // fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              // page
+        sub     x2,  x14, x17   // offset in page
+        lsr     x15, x2,  #21   // group index
+        lsl     x15, x15, #1    // group offset (index * 2)
+        lsr     x2,  x2,  #9    // card offset
+
+    // check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard_Xchg)
+
+    // SETTING CARD FOR X14
+    LOCAL_LABEL(SetCard_Xchg):
+        ldrb    w3, [x17, x2]
+        cbnz    w3, LOCAL_LABEL(CardSet_Xchg)
+        mov     w3, #1
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(SetGroup_Xchg):
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, LOCAL_LABEL(CardSet_Xchg)
+        mov     w3, #1
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(SetPage_Xchg):
+        ldrb    w3, [x17]
+        cbnz    w3, LOCAL_LABEL(CardSet_Xchg)
+        mov     w3, #1
+        strb    w3, [x17]
+
+    LOCAL_LABEL(CardSet_Xchg):
+    // check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  // !g_write_watch_table -> !concurrent
+        cbnz    x12, LOCAL_LABEL(DirtyCard_Xchg)
+        b       LOCAL_LABEL(Exit_Xchg)
+
+    // DIRTYING CARD FOR X14
+    LOCAL_LABEL(DirtyCard_Xchg):
+        mov     w3, #3
+        strb    w3, [x17, x2]
+    LOCAL_LABEL(DirtyGroup_Xchg):
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+    LOCAL_LABEL(DirtyPage_Xchg):
+        strb    w3, [x17]
+        b       LOCAL_LABEL(Exit_Xchg)
+
+    // this is expected to be rare.
+    LOCAL_LABEL(RecordEscape_Xchg):
+
+    // 4) check if the source is escaped
+        and         x12, x1, #0xFFFFFFFFFFE00000  // source region
+        add         x1, x1, #8                   // escape bit is MT + 1
+        ubfx        x17, x1, #9,#12               // word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        // mark word = [region + index * 8]
+        lsr         x12, x1, #3                   // bit = (dst >> 3) [& 63]
+        sub         x1, x1, #8     // undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, LOCAL_LABEL(AssignAndMarkCards_Xchg)        // source is already escaped.
+
+        // we need to preserve our parameters x0, x1 and x29/x30
+        stp     x29,x30, [sp, -16 * 2]!
+        stp     x0, x1,  [sp, 16 * 1]
+
+        // void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        and  x2, x1, #0xFFFFFFFFFFE00000  // source region
+        ldr  x12, [x2, #8]                 // EscapeFn address
+        blr  x12
+
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldp     x29,x30, [sp], 16 * 2
+
+        b       LOCAL_LABEL(AssignAndMarkCards_Xchg)
+LEAF_END RhpCheckedXchg, _TEXT
+
+
+#endif //FEATURE_SATORI_GC

src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm

@@ -12,6 +12,8 @@
 
     TEXTAREA
 
+#ifndef FEATURE_SATORI_GC
+
 ;; Macro used to copy contents of newly updated GC heap locations to a shadow copy of the heap. This is used
 ;; during garbage collections to verify that object references where never written to the heap without using a
 ;; write barrier. Note that we're potentially racing to update the shadow heap while other threads are writing
@@ -396,4 +398,573 @@ NoBarrierXchg
 
     LEAF_END RhpCheckedXchg
 
+#else  ;;FEATURE_SATORI_GC   ######################################################
+
+;; void JIT_ByRefWriteBarrier
+;; On entry:
+;;   x13  : the source address (points to object reference to write)
+;;   x14  : the destination address (object reference written here)
+;;
+;; On exit:
+;;   x13  : incremented by 8
+;;   x14  : incremented by 8
+;;   x15  : trashed
+;;   x12, x17  : trashed
+;;
+    LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
+
+        ldr     x15, [x13], 8
+        b       RhpCheckedAssignRefArm64
+
+    LEAF_END RhpByRefAssignRefArm64
+
+;; JIT_CheckedWriteBarrier(Object** dst, Object* src)
+;;
+;; Write barrier for writes to objects that may reside
+;; on the managed heap.
+;;
+;; On entry:
+;;   x14 : the destination address (LHS of the assignment).
+;;         May not be a heap location (hence the checked).
+;;   x15 : the object reference (RHS of the assignment).
+;;
+;; On exit:
+;;   x12, x17 : trashed
+;;   x14      : incremented by 8
+    LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x14, lsr #30
+        ldrb    w12, [x12]
+        cbnz    x12, RhpAssignRefArm64
+
+NotInHeap
+    ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+        str  x15, [x14], #8
+        ret  lr
+    LEAF_END RhpCheckedAssignRefArm64
+
+;; JIT_WriteBarrier(Object** dst, Object* src)
+;;
+;; Write barrier for writes to objects that are known to
+;; reside on the managed heap.
+;;
+;; On entry:
+;;  x14 : the destination address (LHS of the assignment).
+;;  x15 : the object reference (RHS of the assignment).
+;;
+;; On exit:
+;;  x12, x17 : trashed
+;;  x14 : incremented by 8
+    LEAF_ENTRY RhpAssignRefArm64, _TEXT
+    ;; check for escaping assignment
+    ;; 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x15, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, JustAssign
+#else
+        cbz     x15, JustAssign    ;; assigning null
+#endif
+        and     x12,  x15, #0xFFFFFFFFFFE00000  ;; source region
+        ldr     x12, [x12]                      ; region tag
+        cmp     x12, x18                        ; x18 - TEB
+        bne     AssignAndMarkCards              ; not local to this thread
+
+    ;; 2) check if the src and dst are from the same region
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbnz    x12, RecordEscape  ;; cross region assignment. definitely escaping
+
+    ;; 3) check if the target is exposed
+        ubfx    x17, x14,#9,#12                 ;; word index = (dst >> 9) & 0x1FFFFF
+        and     x12, x15, #0xFFFFFFFFFFE00000   ;; source region
+        ldr     x17, [x12, x17, lsl #3]         ;; mark word = [region + index * 8]
+        lsr     x12, x14, #3                    ;; bit = (dst >> 3) [& 63]
+        lsr     x17, x17, x12
+        tbnz    x17, #0, RecordEscape ;; target is exposed. record an escape.
+
+        str     x15, [x14], #8                  ;; UNORDERED assignment of unescaped object
+        ret     lr
+
+JustAssign
+    ALTERNATE_ENTRY RhpAssignRefAVLocationNotHeap
+        stlr    x15, [x14]                      ;; no card marking, src is not a heap object
+        add     x14, x14, 8
+        ret     lr
+
+AssignAndMarkCards
+    ALTERNATE_ENTRY RhpAssignRefAVLocation
+        stlr    x15, [x14]
+
+    ;; need couple temps. Save before using.
+        stp     x2,  x3,  [sp, -16]!
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, CheckConcurrent ;; same region, just check if barrier is not concurrent
+
+    ;; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     ;; source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, MarkCards
+
+CheckConcurrent
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, MarkCards
+        
+Exit
+        ldp  x2,  x3, [sp], 16
+        add  x14, x14, 8
+        ret  lr
+
+MarkCards
+    ;; fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  ;; fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              ;; page
+        sub     x2,  x14, x17   ;; offset in page
+        lsr     x15, x2,  #21   ;; group index
+        lsl     x15, x15, #1    ;; group offset (index * 2)
+        lsr     x2,  x2,  #9    ;; card offset
+
+    ;; check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard
+
+    ;; SETTING CARD FOR X14
+SetCard
+        ldrb    w3, [x17, x2]
+        cbnz    w3, CardSet
+        mov     w3, #1
+        strb    w3, [x17, x2]
+SetGroup
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, CardSet
+        mov     w3, #1
+        strb    w3, [x12, x15]
+SetPage
+        ldrb    w3, [x17]
+        cbnz    w3, CardSet
+        mov     w3, #1
+        strb    w3, [x17]
+
+CardSet
+    ;; check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard
+        b       Exit
+
+    ;; DIRTYING CARD FOR X14
+DirtyCard
+        mov     w3, #3
+        strb    w3, [x17, x2]
+DirtyGroup
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+DirtyPage
+        strb    w3, [x17]
+        b       Exit
+
+    ;; this is expected to be rare.
+RecordEscape
+
+    ;; 4) check if the source is escaped
+        and         x12, x15, #0xFFFFFFFFFFE00000  ;; source region
+        add         x15, x15, #8                   ;; escape bit is MT + 1
+        ubfx        x17, x15, #9,#12               ;; word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        ;; mark word = [region + index * 8]
+        lsr         x12, x15, #3                   ;; bit = (dst >> 3) [& 63]
+        sub         x15, x15, #8     ;; undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, AssignAndMarkCards        ;; source is already escaped.
+
+        ;; because of the barrier call convention
+        ;; we need to preserve caller-saved x0 through x18 and x29/x30
+
+        stp     x29,x30, [sp, -16 * 10]!
+        stp     x0, x1,  [sp, 16 * 1]
+        stp     x2, x3,  [sp, 16 * 2]
+        stp     x4, x5,  [sp, 16 * 3]
+        stp     x6, x7,  [sp, 16 * 4]
+        stp     x8, x9,  [sp, 16 * 5]
+        stp     x10,x11, [sp, 16 * 6]
+        stp     x12,x13, [sp, 16 * 7]
+        stp     x14,x15, [sp, 16 * 8]
+        stp     x16,x17, [sp, 16 * 9]
+
+        ;; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        ;; mov  x0, x14  EscapeFn does not use dst, it is just to avoid arg shuffle on x64
+        mov  x1, x15
+        and  x2, x15, #0xFFFFFFFFFFE00000  ;; source region
+        ldr  x12, [x2, #8]                 ;; EscapeFn address
+        blr  x12
+
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldp     x2, x3,  [sp, 16 * 2]
+        ldp     x4, x5,  [sp, 16 * 3]
+        ldp     x6, x7,  [sp, 16 * 4]
+        ldp     x8, x9,  [sp, 16 * 5]
+        ldp     x10,x11, [sp, 16 * 6]
+        ldp     x12,x13, [sp, 16 * 7]
+        ldp     x14,x15, [sp, 16 * 8]
+        ldp     x16,x17, [sp, 16 * 9]
+        ldp     x29,x30, [sp], 16 * 10
+
+        b       AssignAndMarkCards
+    LEAF_END RhpAssignRefArm64
+
+;; Same as RhpAssignRefArm64, but with standard ABI.
+    LEAF_ENTRY RhpAssignRef, _TEXT
+        mov     x14, x0                     ;; x14 = dst
+        mov     x15, x1                     ;; x15 = val
+        b       RhpAssignRefArm64
+    LEAF_END RhpAssignRef
+
+;; RhpCheckedLockCmpXchg(Object** dest, Object* value, Object* comparand)
+;;
+;; Interlocked compare exchange on objectref.
+;;
+;; On entry:
+;;  x0: pointer to objectref
+;;  x1: exchange value
+;;  x2: comparand
+;;
+;; On exit:
+;;  x0: original value of objectref
+;;  x10, x12, x17: trashed
+;;
+    LEAF_ENTRY RhpCheckedLockCmpXchg
+    ;; check if dst is in heap
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x0, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, JustAssign_Cmp_Xchg
+
+    ;; check for escaping assignment
+    ;; 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x1, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, JustAssign_Cmp_Xchg
+#else
+        cbz     x1, JustAssign_Cmp_Xchg    ;; assigning null
+#endif
+        and     x12,  x1, #0xFFFFFFFFFFE00000   ; source region
+        ldr     x12, [x12]                      ; region tag
+        cmp     x12, x18                        ; x18 - TEB
+        bne     AssignAndMarkCards_Cmp_Xchg     ; not local to this thread
+
+    ;; 2) check if the src and dst are from the same region
+        eor     x12, x0, x1
+        lsr     x12, x12, #21
+        cbnz    x12, RecordEscape_Cmp_Xchg  ;; cross region assignment. definitely escaping
+
+    ;; 3) check if the target is exposed
+        ubfx        x17, x0,#9,#12              ;; word index = (dst >> 9) & 0x1FFFFF
+        and         x12, x1, #0xFFFFFFFFFFE00000  ;; source region
+        ldr         x17, [x12, x17, lsl #3]     ;; mark word = [region + index * 8]
+        lsr         x12, x0, #3                 ;; bit = (dst >> 3) [& 63]
+        lsr         x17, x17, x12
+        tbnz        x17, #0, RecordEscape_Cmp_Xchg ;; target is exposed. record an escape.
+
+JustAssign_Cmp_Xchg
+        mov     x14, x0
+TryAgain_Cmp_Xchg
+    ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocationNotHeap
+        ldaxr   x0, [x14]
+        cmp     x0, x2
+        bne     NoUpdate_Cmp_Xchg
+
+        ;; Current value matches comparand, attempt to update with the new value.
+        stlxr   w12, x1, [x14]
+        cbnz    w12, TryAgain_Cmp_Xchg  ;; if failed, try again
+
+NoUpdate_Cmp_Xchg
+        dmb     ish
+        ret     lr
+
+AssignAndMarkCards_Cmp_Xchg
+        mov    x14, x0                       ;; x14 = dst
+        mov    x15, x1                       ;; x15 = val
+TryAgain1_Cmp_Xchg
+    ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
+        ldaxr   x0, [x14]
+        cmp     x0, x2
+        bne     NoUpdate_Cmp_Xchg
+
+        ;; Current value matches comparand, attempt to update with the new value.
+        stlxr   w12, x1, [x14]
+        cbnz    w12, TryAgain1_Cmp_Xchg  ;; if failed, try again
+        dmb     ish
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, CheckConcurrent_Cmp_Xchg ;; same region, just check if barrier is not concurrent
+
+    ;; we will trash x2 and x3, this is a regular call, so it is ok
+    ;; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     ;; source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, MarkCards_Cmp_Xchg
+
+CheckConcurrent_Cmp_Xchg
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, MarkCards_Cmp_Xchg
+        
+Exit_Cmp_Xchg
+        ret  lr
+
+MarkCards_Cmp_Xchg
+    ;; fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  ;; fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              ;; page
+        sub     x2,  x14, x17   ;; offset in page
+        lsr     x15, x2,  #21   ;; group index
+        lsl     x15, x15, #1    ;; group offset (index * 2)
+        lsr     x2,  x2,  #9    ;; card offset
+
+    ;; check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard_Cmp_Xchg
+
+    ;; SETTING CARD FOR X14
+SetCard_Cmp_Xchg
+        ldrb    w3, [x17, x2]
+        cbnz    w3, CardSet_Cmp_Xchg
+        mov     w3, #1
+        strb    w3, [x17, x2]
+SetGroup_Cmp_Xchg
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, CardSet_Cmp_Xchg
+        mov     w3, #1
+        strb    w3, [x12, x15]
+SetPage_Cmp_Xchg
+        ldrb    w3, [x17]
+        cbnz    w3, CardSet_Cmp_Xchg
+        mov     w3, #1
+        strb    w3, [x17]
+
+CardSet_Cmp_Xchg
+    ;; check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard_Cmp_Xchg
+        b       Exit_Cmp_Xchg
+
+    ;; DIRTYING CARD FOR X14
+DirtyCard_Cmp_Xchg
+        mov     w3, #3
+        strb    w3, [x17, x2]
+DirtyGroup_Cmp_Xchg
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+DirtyPage_Cmp_Xchg
+        strb    w3, [x17]
+        b       Exit_Cmp_Xchg
+
+    ;; this is expected to be rare.
+RecordEscape_Cmp_Xchg
+
+    ;; 4) check if the source is escaped
+        and         x12, x1, #0xFFFFFFFFFFE00000  ;; source region
+        add         x1, x1, #8                   ;; escape bit is MT + 1
+        ubfx        x17, x1, #9,#12               ;; word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        ;; mark word = [region + index * 8]
+        lsr         x12, x1, #3                   ;; bit = (dst >> 3) [& 63]
+        sub         x1, x1, #8     ;; undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, AssignAndMarkCards_Cmp_Xchg        ;; source is already escaped.
+
+        ;; we need to preserve our parameters x0, x1, x2 and x29/x30
+
+        stp     x29,x30, [sp, -16 * 3]!
+        stp     x0, x1,  [sp, 16 * 1]
+        str     x2,      [sp, 16 * 2]
+
+        ;; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        and  x2, x1, #0xFFFFFFFFFFE00000  ;; source region
+        ldr  x12, [x2, #8]                 ;; EscapeFn address
+        blr  x12
+
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldr     x2,      [sp, 16 * 2]
+        ldp     x29,x30, [sp], 16 * 3
+
+        b       AssignAndMarkCards_Cmp_Xchg
+    LEAF_END RhpCheckedLockCmpXchg
+
+;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular:
+;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen within at RhpCheckedXchgAVLocation
+;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address
+
+;; RhpCheckedXchg(Object** destination, Object* value)
+;;
+;; Interlocked exchange on objectref.
+;;
+;; On entry:
+;;  x0: pointer to objectref
+;;  x1: exchange value
+;;
+;; On exit:
+;;  x0: original value of objectref
+;;  x10: trashed
+;;  x12, x17: trashed
+;;
+    LEAF_ENTRY RhpCheckedXchg, _TEXT
+
+    ;; check if dst is in heap
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x0, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, JustAssign_Xchg
+
+    ;; check for escaping assignment
+    ;; 1) check if we own the source region
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+        add     x12, x12, x1, lsr #30
+        ldrb    w12, [x12]
+        cbz     x12, JustAssign_Xchg
+#else
+        cbz     x1, JustAssign_Xchg    ;; assigning null
+#endif
+        and     x12,  x1, #0xFFFFFFFFFFE00000   ; source region
+        ldr     x12, [x12]                      ; region tag
+        cmp     x12, x18                        ; x18 - TEB
+        bne     AssignAndMarkCards_Xchg         ; not local to this thread
+
+    ;; 2) check if the src and dst are from the same region
+        eor     x12, x0, x1
+        lsr     x12, x12, #21
+        cbnz    x12, RecordEscape_Xchg  ;; cross region assignment. definitely escaping
+
+    ;; 3) check if the target is exposed
+        ubfx        x17, x0,#9,#12              ;; word index = (dst >> 9) & 0x1FFFFF
+        and         x12, x1, #0xFFFFFFFFFFE00000  ;; source region
+        ldr         x17, [x12, x17, lsl #3]      ;; mark word = [region + index * 8]
+        lsr         x12, x0, #3                 ;; bit = (dst >> 3) [& 63]
+        lsr         x17, x17, x12
+        tbnz        x17, #0, RecordEscape_Xchg ;; target is exposed. record an escape.
+
+JustAssign_Xchg
+TryAgain_Xchg
+    ALTERNATE_ENTRY RhpCheckedXchgAVLocationNotHeap
+        ldaxr   x17, [x0]
+        stlxr   w12, x1, [x0]
+        cbnz    w12, TryAgain_Xchg
+        mov     x0, x17
+        dmb     ish
+        ret    lr
+
+AssignAndMarkCards_Xchg
+        mov    x14, x0                        ;; x14 = dst
+        mov    x15, x1                        ;; x15 = val    ;; TODO: VS not needed, can use x1
+TryAgain1_Xchg
+    ALTERNATE_ENTRY RhpCheckedXchgAVLocation
+        ldaxr   x17, [x0]
+        stlxr   w12, x1, [x0]
+        cbnz    w12, TryAgain1_Xchg
+        mov     x0, x17
+        dmb     ish
+
+        eor     x12, x14, x15
+        lsr     x12, x12, #21
+        cbz     x12, CheckConcurrent_Xchg ;; same region, just check if barrier is not concurrent
+
+    ;; we will trash x2 and x3, this is a regular call, so it is ok
+    ;; if src is in gen2/3 and the barrier is not concurrent we do not need to mark cards
+        and     x2,  x15, #0xFFFFFFFFFFE00000     ;; source region
+        ldr     w12, [x2, 16]
+        tbz     x12, #1, MarkCards_Xchg
+
+CheckConcurrent_Xchg
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, MarkCards_Xchg
+        
+Exit_Xchg
+        ret  lr
+
+MarkCards_Xchg
+    ;; fetch card location for x14
+        PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12  ;; fetch the page map
+        lsr     x17, x14, #30
+        ldr     x17, [x12, x17, lsl #3]              ;; page
+        sub     x2,  x14, x17   ;; offset in page
+        lsr     x15, x2,  #21   ;; group index
+        lsl     x15, x15, #1    ;; group offset (index * 2)
+        lsr     x2,  x2,  #9    ;; card offset
+
+    ;; check if concurrent marking is in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard_Xchg
+
+    ;; SETTING CARD FOR X14
+SetCard_Xchg
+        ldrb    w3, [x17, x2]
+        cbnz    w3, CardSet_Xchg
+        mov     w3, #1
+        strb    w3, [x17, x2]
+SetGroup_Xchg
+        add     x12, x17, #0x80
+        ldrb    w3, [x12, x15]
+        cbnz    w3, CardSet_Xchg
+        mov     w3, #1
+        strb    w3, [x12, x15]
+SetPage_Xchg
+        ldrb    w3, [x17]
+        cbnz    w3, CardSet_Xchg
+        mov     w3, #1
+        strb    w3, [x17]
+
+CardSet_Xchg
+    ;; check if concurrent marking is still not in progress
+        PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12  ;; !g_write_watch_table -> !concurrent
+        cbnz    x12, DirtyCard_Xchg
+        b       Exit_Xchg
+
+    ;; DIRTYING CARD FOR X14
+DirtyCard_Xchg
+        mov     w3, #3
+        strb    w3, [x17, x2]
+DirtyGroup_Xchg
+        add     x12, x17, #0x80
+        strb    w3, [x12, x15]
+DirtyPage_Xchg
+        strb    w3, [x17]
+        b       Exit_Xchg
+
+    ;; this is expected to be rare.
+RecordEscape_Xchg
+
+    ;; 4) check if the source is escaped
+        and         x12, x1, #0xFFFFFFFFFFE00000  ;; source region
+        add         x1, x1, #8                   ;; escape bit is MT + 1
+        ubfx        x17, x1, #9,#12               ;; word index = (dst >> 9) & 0x1FFFFF
+        ldr         x17, [x12, x17, lsl #3]        ;; mark word = [region + index * 8]
+        lsr         x12, x1, #3                   ;; bit = (dst >> 3) [& 63]
+        sub         x1, x1, #8     ;; undo MT + 1
+        lsr         x17, x17, x12
+        tbnz        x17, #0, AssignAndMarkCards_Xchg        ;; source is already escaped.
+
+        ;; we need to preserve our parameters x0, x1 and x29/x30
+        stp     x29,x30, [sp, -16 * 2]!
+        stp     x0, x1,  [sp, 16 * 1]
+
+        ;; void SatoriRegion::EscapeFn(SatoriObject** dst, SatoriObject* src, SatoriRegion* region)
+        and  x2, x1, #0xFFFFFFFFFFE00000  ;; source region
+        ldr  x12, [x2, #8]                 ;; EscapeFn address
+        blr  x12
+
+        ldp     x0, x1,  [sp, 16 * 1]
+        ldp     x29,x30, [sp], 16 * 2
+
+        b       AssignAndMarkCards_Xchg
+    LEAF_END RhpCheckedXchg
+
+#endif
+
     end

src/coreclr/nativeaot/Runtime/gcenv.ee.cpp

@@ -465,6 +465,27 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         assert(!"should never be called without FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP");
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         return;
+
+#if FEATURE_SATORI_GC
+    case WriteBarrierOp::StartConcurrentMarkingSatori:
+        g_write_watch_table = (uint8_t*)1;
+        g_sw_ww_enabled_for_gc_heap = true;
+        if (!is_runtime_suspended)
+        {
+            // If runtime is not suspended, force all threads to see the changed state before
+            // observing future allocations.
+            FlushProcessWriteBuffers();
+        }
+
+        return;
+
+    case WriteBarrierOp::StopConcurrentMarkingSatori:
+        assert(args->is_runtime_suspended && "the runtime must be suspended here!");
+        g_write_watch_table = (uint8_t*)0;
+        g_sw_ww_enabled_for_gc_heap = false;
+        return;
+#endif
+
     default:
         assert(!"Unknokwn WriteBarrierOp enum");
         return;
@@ -500,7 +521,7 @@ bool GCToEEInterface::EagerFinalized(Object* obj)
     // after marking strongly reachable and prior to marking dependent and long weak handles.
     // Managed code should not be running.
 
-    // TODO: VS threadlocal GC is also ok
+    // threadlocal GC is also ok
     // ASSERT(GCHeapUtilities::GetGCHeap()->IsGCInProgressHelper());
 
     // the lowermost 2 bits are reserved for storing additional info about the handle
@@ -538,6 +559,35 @@ struct ThreadStubArguments
     CLREventStatic m_ThreadStartedEvent;
 };
 
+static bool CreateUnsuspendableThread(void (*threadStart)(void*), void* arg, const char* name)
+{
+    UNREFERENCED_PARAMETER(name);
+
+    ThreadStubArguments* threadStubArgs = new (nothrow) ThreadStubArguments();
+    if (!threadStubArgs)
+        return false;
+
+    threadStubArgs->m_pRealStartRoutine = threadStart;
+    threadStubArgs->m_pRealContext = arg;
+
+    // Helper used to wrap the start routine of background GC threads so we can do things like initialize the
+    // Redhawk thread state which requires running in the new thread's context.
+    auto threadStub = [](void* argument) -> DWORD
+    {
+        ThreadStore::RawGetCurrentThread()->SetGCSpecial();
+
+        ThreadStubArguments* pStartContext = (ThreadStubArguments*)argument;
+        auto realStartRoutine = pStartContext->m_pRealStartRoutine;
+        void* realContext = pStartContext->m_pRealContext;
+        delete pStartContext;
+
+        STRESS_LOG_RESERVE_MEM(GC_STRESSLOG_MULTIPLY);
+
+        realStartRoutine(realContext);
+
+        return 0;
+    };
+
 static bool CreateNonSuspendableThread(void (*threadStart)(void*), void* arg, const char* name)
 {
     UNREFERENCED_PARAMETER(name);

src/coreclr/nativeaot/Runtime/gcheaputilities.h

@@ -82,7 +82,9 @@ public:
 
 #ifdef FEATURE_SVR_GC
         _ASSERTE(g_heap_type != GC_HEAP_INVALID);
-        return (g_heap_type == GC_HEAP_SVR);
+
+        // TODO: Satori is closer to SVR than WKS, since there is threading. Need a better API though.
+        return g_heap_type > GC_HEAP_WKS;
 #else
         return false;
 #endif // FEATURE_SVR_GC

src/coreclr/nativeaot/Runtime/inc/MethodTable.h

@@ -293,6 +293,7 @@ public:
     // Methods expected by the GC
     uint32_t ContainsGCPointers() { return HasReferenceFields(); }
     uint32_t ContainsGCPointersOrCollectible() { return HasReferenceFields(); }
+    bool Collectible() { return false; }
     UInt32_BOOL SanityCheck() { return Validate(); }
 };
 

src/coreclr/nativeaot/Runtime/thread.cpp

@@ -37,7 +37,8 @@ static Thread* g_RuntimeInitializingThread;
 
 PInvokeTransitionFrame* Thread::GetTransitionFrame()
 {
-    if (ThreadStore::GetSuspendingThread() == this)
+    if (ThreadStore::GetSuspendingThread() == this ||
+        ThreadStore::GetCurrentThread() == this)
     {
         // This thread is in cooperative mode, so we grab the deferred frame
         // which is the frame from the most

src/coreclr/vm/amd64/JitHelpers_Fast.asm

@@ -307,7 +307,7 @@ LEAF_END_MARKED JIT_CheckedWriteBarrier, _TEXT
 
 
 
-else  ;FEATURE_SATORI_GC
+else  ;FEATURE_SATORI_GC     ##########################################################################
 
 Section segment para 'DATA'
 
@@ -336,13 +336,10 @@ LEAF_END JIT_PatchedCodeStart, _TEXT
 LEAF_ENTRY JIT_CheckedWriteBarrier, _TEXT
 
         ; See if this is in GCHeap
-        cmp     rcx, [g_highest_address]
-        ja      NotInHeap
-
-        mov     r9 , [g_card_table]     ; fetch the page map
         mov     rax, rcx
         shr     rax, 30                 ; round to page size ( >> PAGE_BITS )
-        cmp     qword ptr [r9 + rax * 8], 0
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
         jne     JIT_WriteBarrier
 
     NotInHeap:
@@ -359,9 +356,22 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
     align 16
     ; check for escaping assignment
     ; 1) check if we own the source region
+
+ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rax, rdx
+        shr     rax, 30                 ; round to page size ( >> PAGE_BITS )
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
+        je      JustAssign              ; src not in heap
+endif
+
         mov     r8, rdx
         and     r8, 0FFFFFFFFFFE00000h  ; source region
-        jz      JustAssign              ; assigning null   
+
+ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
+        jz      JustAssign              ; assigning null
+endif
+
         mov     rax,  gs:[30h]          ; thread tag, TEB on NT
         cmp     qword ptr [r8], rax     
         jne     AssignAndMarkCards      ; not local to this thread
@@ -380,7 +390,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
         jb      RecordEscape            ; target is exposed. record an escape.
 
     JustAssign:
-        mov     [rcx], rdx              ; threadlocal assignment of unescaped object
+        mov     [rcx], rdx              ; no card marking, src is not a heap object
         ret
 
     AssignAndMarkCards:
@@ -400,7 +410,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
     CheckConcurrent:
         cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
         jne     MarkCards
-        REPRET
+        ret
 
     MarkCards:
     ; fetch card location for rcx
@@ -435,7 +445,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
     ; check if concurrent marking is still not in progress
         cmp     byte ptr [g_sw_ww_enabled_for_gc_heap], 0h
         jne     DirtyCard
-        REPRET
+        ret
 
     ; DIRTYING CARD FOR RCX
      DirtyCard:
@@ -492,13 +502,10 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT
         add     rsi, 8h
 
         ; See if assignment is into heap
-        cmp     rcx, [g_highest_address]
-        ja      NotInHeap               ; not in heap
-
-        mov     r9 , [g_card_table]     ; fetch the page map
         mov     rax, rcx
-        shr     rax, 30                 ; round to page size ( >> PAGE_BITS )
+        shr     rax, 30                    ; round to page size ( >> PAGE_BITS )
-        cmp     qword ptr [r9 + rax * 8], 0
+        add     rax, [g_card_bundle_table] ; fetch the page byte map
+        cmp     byte ptr [rax], 0
         jne     JIT_WriteBarrier
 
     align 16

src/coreclr/vm/amd64/jithelpers_fast.S

@@ -213,7 +213,7 @@ LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier):
 LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT
 
 
-#else  //FEATURE_SATORI_GC
+#else  //FEATURE_SATORI_GC      ##############################################################################
 
 // Mark start of the code region that we patch at runtime
 LEAF_ENTRY JIT_PatchedCodeStart, _TEXT
@@ -225,15 +225,11 @@ LEAF_END JIT_PatchedCodeStart, _TEXT
 LEAF_ENTRY JIT_CheckedWriteBarrier, _TEXT
 
         // See if this is in GCHeap
-        PREPARE_EXTERNAL_VAR g_highest_address, rax
-        cmp     rdi, [rax]
-        ja      NotInHeap
-
-        PREPARE_EXTERNAL_VAR g_card_table, r8
-        mov     r8,  [r8]               // fetch the page map
         mov     rax, rdi
         shr     rax, 30                 // round to page size ( >> PAGE_BITS )
-        cmp     qword ptr [r8 + rax * 8], 0
+        PREPARE_EXTERNAL_VAR g_card_bundle_table, r8
+        add     rax, [r8]               // fetch the page byte map
+        cmp     byte ptr [rax], 0
         jne     C_FUNC(JIT_WriteBarrier)
 
     NotInHeap:
@@ -249,11 +245,21 @@ LEAF_END_MARKED JIT_CheckedWriteBarrier, _TEXT
 //
 .balign 16
 LEAF_ENTRY JIT_WriteBarrier, _TEXT
+#ifdef FEATURE_SATORI_EXTERNAL_OBJECTS
+        mov     rax, rsi
+        shr     rax, 30                 // round to page size ( >> PAGE_BITS )
+        PREPARE_EXTERNAL_VAR g_card_bundle_table, r8
+        add     rax, [r8]               // fetch the page byte map
+        cmp     byte ptr [rax], 0
+        je      JustAssign              // src not in heap
+#endif
     // check for escaping assignment
     // 1) check if we own the source region
         mov     rdx, rsi
         and     rdx, 0xFFFFFFFFFFE00000  // source region
+#ifndef FEATURE_SATORI_EXTERNAL_OBJECTS
         jz      JustAssign              // assigning null
+#endif
 #ifdef TARGET_OSX
         mov     rax,  gs:[0]            // thread tag
 #else
@@ -276,7 +282,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
         jb      RecordEscape            // target is exposed. record an escape.
 
     JustAssign:
-        mov     [rdi], rsi              // threadlocal assignment of unescaped object
+        mov     [rdi], rsi              // no card marking, src is not a heap object
 
         // set rdi, rsi per contract with JIT_ByRefWriteBarrier
         add     rdi, 8
@@ -307,7 +313,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
     CheckConcurrent:
         cmp     byte ptr [r11], 0
         jne     MarkCards
-        REPRET
+        ret
 
     MarkCards:
     // fetch card location for rax  (saved rdi)
@@ -343,7 +349,7 @@ LEAF_ENTRY JIT_WriteBarrier, _TEXT
      // check if concurrent marking is still not in progress
         cmp     byte ptr [r11], 0
         jne     DirtyCard
-        REPRET
+        ret
 
      // DIRTYING CARD
      DirtyCard:

src/coreclr/vm/frozenobjectheap.cpp

@@ -45,13 +45,19 @@ Object* FrozenObjectHeapManager::TryAllocateObject(PTR_MethodTable type, size_t
       return nullptr;
     }
 
-    obj = AllocateImmortalObject(type, objectSize);
+#if defined(_DEBUG) && defined(FEATURE_SATORI_EXTERNAL_OBJECTS)
-    if (initFunc != nullptr)
+    // in debug use external objects once in a while - for coverage
+    if (objectSize % 16 != 0)
+#endif
     {
-      initFunc(obj, pParam);
+      obj = AllocateImmortalObject(type, objectSize);
-    }
+      if (initFunc != nullptr)
+      {
+        initFunc(obj, pParam);
+      }
 
-    return obj;
+      return obj;
+    }
 #endif
 
 

src/coreclr/vm/gcenv.ee.cpp

@@ -1152,6 +1152,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
         break;
 
+#if FEATURE_SATORI_GC
     case WriteBarrierOp::StartConcurrentMarkingSatori:
         g_sw_ww_table = (uint8_t*)1;
         g_sw_ww_enabled_for_gc_heap = true;
@@ -1170,6 +1171,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
         g_sw_ww_enabled_for_gc_heap = false;
         stompWBCompleteActions |= ::SwitchToNonWriteWatchBarrier(true);
         return;
+#endif
 
     default:
         assert(!"unknown WriteBarrierOp enum");

src/coreclr/vm/gchelpers.cpp

@@ -1438,10 +1438,13 @@ bool IsInHeapSatori(void* ptr)
 void CheckEscapeSatori(Object** dst, Object* ref)
 {
     SatoriObject* obj = (SatoriObject*)ref;
-    SatoriRegion* region = obj->ContainingRegion();
+    // TODO: no nullcheck when external
+    if (!obj || obj->IsExternal())
+        return;
 
     // we should be the owner of the region to care about escapes
-    if (region && region->IsEscapeTrackedByCurrentThread())
+    SatoriRegion* region = obj->ContainingRegion();
+    if (region->IsEscapeTrackedByCurrentThread())
     {
         if ((((size_t)dst ^ (size_t)ref) >> 21) == 0 &&
             !(region->IsExposed((SatoriObject**)dst)))
@@ -1455,88 +1458,6 @@ void CheckEscapeSatori(Object** dst, Object* ref)
     }
 }
 
-bool CheckEscapeSatoriRange(size_t dst, size_t src, size_t len)
-{
-    SatoriRegion* curRegion = (SatoriRegion*)GCToEEInterface::GetAllocContext()->gc_reserved_1;
-    if (!curRegion || !curRegion->IsEscapeTracking())
-    {
-        // not tracking escapes, not a local assignment.
-        return false;
-    }
-
-    _ASSERTE(curRegion->IsEscapeTrackedByCurrentThread());
-
-    // if dst is within the curRegion and is not exposed, we are done
-    if (((dst ^ curRegion->Start()) >> 21) == 0)
-    {
-        if (!curRegion->AnyExposed(dst, len))
-        {
-            // thread-local assignment
-            return true;
-        }
-    }
-
-    if (!PageForAddressCheckedSatori((void*)dst))
-    {
-        // dest not in heap, must be stack, so, local
-        return true;
-    }
-
-    if (((src ^ curRegion->Start()) >> 21) == 0)
-    {
-        // if src is in current region, the elements could be escaping
-        if (!curRegion->AnyExposed(src, len))
-        {
-            // one-element array copy is embarrasingly common. specialcase that.
-            if (len == sizeof(size_t))
-            {
-                SatoriObject* obj = *(SatoriObject**)src;
-                if (obj->ContainingRegion() == curRegion)
-                {
-                    curRegion->EscapeRecursively(obj);
-                }
-            }
-            else
-            {
-                SatoriObject* containingSrcObj = curRegion->FindObject(src);
-                containingSrcObj->ForEachObjectRef(
-                    [&](SatoriObject** ref)
-                    {
-                        SatoriObject* child = *ref;
-                        if (child->ContainingRegion() == curRegion)
-                        {
-                            curRegion->EscapeRecursively(child);
-                        }
-                    },
-                    src,
-                    src + len
-                );
-            }
-        }
-
-        return false;
-    }
-
-    if (PageForAddressCheckedSatori((void*)src))
-    {
-        // src is not in current region but in heap,
-        // it can't escape anything that belong to current thread, but it is not local.
-        return false;
-    }
-
-    // This is a very rare case where we are copying refs out of non-heap area like stack or native heap.
-    // We do not have a containing type and that is somewhat inconvenient.
-    // 
-    // There are not many scenarios that lead here. In particular, boxing uses a newly
-    // allocated and not yet escaped target, so it does not end up here.
-    // One possible way to get here is a copy-back after a reflection call with a boxed nullable
-    // argument that happen to escape.
-    // 
-    // We could handle this is by concervatively escaping any value that matches an unescaped pointer in curRegion.
-    // However, considering how uncommon this is, we will just give up tracking.
-    curRegion->StopEscapeTracking();
-    return false;
-}
 #endif
 
 // This function sets the card table with the granularity of 1 byte, to avoid ghost updates
@@ -1550,7 +1471,7 @@ void ErectWriteBarrier(OBJECTREF *dst, OBJECTREF ref)
 #if FEATURE_SATORI_GC
 
     SatoriObject* obj = (SatoriObject*)OBJECTREFToObject(ref);
-    if (!obj)
+    if (!obj || obj->IsExternal())
         return;
 
     // check for obj in the same region or in gen2

src/coreclr/vm/gchelpers.h

@@ -93,7 +93,6 @@ void ErectWriteBarrier(OBJECTREF* dst, OBJECTREF ref);
 #if FEATURE_SATORI_GC
 bool IsInHeapSatori(void* ptr);
 void CheckEscapeSatori(Object** dst, Object* ref);
-bool CheckEscapeSatoriRange(size_t dst, size_t src, size_t len);
 #endif
 
 void PublishFrozenObject(Object*& orObject);

src/coreclr/vm/gchelpers.inl

@@ -32,6 +32,7 @@ static const int card_byte_shift = 10;
 const static int REGION_BITS = 21;
 const static size_t REGION_SIZE_GRANULARITY = 1 << REGION_BITS;
 
+#if !FEATURE_SATORI_GC
 FORCEINLINE void InlinedSetCardsAfterBulkCopyHelper(Object** start, size_t len)
 {
     // Check whether the writes were even into the heap. If not there's no card update required.
@@ -107,5 +108,6 @@ FORCEINLINE void InlinedSetCardsAfterBulkCopyHelper(Object** start, size_t len)
     while (bundleByteCount != 0);
 #endif
 }
+#endif // !FEATURE_SATORI_GC
 
 #endif // !_GCHELPERS_INL_

src/coreclr/vm/weakreferencenative.cpp

@@ -27,7 +27,7 @@ void FinalizeWeakReference(Object* obj)
     // after marking strongly reachable and prior to marking dependent and long weak handles.
     // Managed code should not be running.
 
-    // TODO: VS threadlocal GC is also ok
+    // threadlocal GC is also ok
     //_ASSERTE(GCHeapUtilities::IsGCInProgress());
 
     // the lowermost 2 bits are reserved for storing additional info about the handle