diff --git a/src/coreclr/gc/CMakeLists.txt b/src/coreclr/gc/CMakeLists.txt index 9bab1a21857..b0505210999 100644 --- a/src/coreclr/gc/CMakeLists.txt +++ b/src/coreclr/gc/CMakeLists.txt @@ -26,6 +26,15 @@ set( GC_SOURCES gcload.cpp handletablecache.cpp satori/SatoriGCHeap.cpp + satori/SatoriHeap.cpp + satori/SatoriPage.cpp + satori/SatoriRegion.cpp + satori/SatoriRegionQueue.cpp + satori/SatoriAllocator.cpp + satori/SatoriRecycler.cpp + satori/SatoriAllocationContext.cpp + satori/SatoriUtil.cpp + satori/SatoriLock.cpp ) if(CLR_CMAKE_HOST_UNIX) @@ -94,6 +103,15 @@ if (CLR_CMAKE_TARGET_WIN32) softwarewritewatch.h vxsort/do_vxsort.h satori/SatoriGCHeap.h + satori/SatoriHeap.h + satori/SatoriPage.h + satori/SatoriRegion.h + satori/SatoriRegionQueue.h + satori/SatoriAllocator.h + satori/SatoriRecycler.h + satori/SatoriAllocationContext.h + satori/SatoriUtil.h + satori/SatoriLock.h ) endif(CLR_CMAKE_TARGET_WIN32) diff --git a/src/coreclr/gc/env/gcenv.os.h b/src/coreclr/gc/env/gcenv.os.h index f8724fc8687..ec8bc8f98d4 100644 --- a/src/coreclr/gc/env/gcenv.os.h +++ b/src/coreclr/gc/env/gcenv.os.h @@ -269,6 +269,8 @@ public: // granularity. static void* VirtualReserve(size_t size, size_t alignment, uint32_t flags, uint16_t node = NUMA_NODE_UNDEFINED); + static void* VirtualReserve(void* location, size_t size); + // Release virtual memory range previously reserved using VirtualReserve // Parameters: // address - starting virtual address diff --git a/src/coreclr/gc/windows/gcenv.windows.cpp b/src/coreclr/gc/windows/gcenv.windows.cpp index 53b868df682..aa150967252 100644 --- a/src/coreclr/gc/windows/gcenv.windows.cpp +++ b/src/coreclr/gc/windows/gcenv.windows.cpp @@ -703,6 +703,12 @@ void* GCToOSInterface::VirtualReserve(size_t size, size_t alignment, uint32_t fl } } +void* GCToOSInterface::VirtualReserve(void* location, size_t size) +{ + DWORD memFlags = MEM_RESERVE; + return ::VirtualAlloc(location, size, memFlags, PAGE_READWRITE); +} + // Release virtual memory range previously reserved using VirtualReserve // Parameters: // address - starting virtual address diff --git a/src/coreclr/src/gc/satori/SatoriAllocationContext.cpp b/src/coreclr/src/gc/satori/SatoriAllocationContext.cpp new file mode 100644 index 00000000000..39e58ffcfea --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriAllocationContext.cpp @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" +#include "gcenv.h" +#include "../env/gcenv.os.h" + +#include "SatoriAllocationContext.h" +#include "SatoriRegion.h" + +class SatoriHeap; + +void SatoriAllocationContext::OnTerminateThread(SatoriHeap* heap) +{ + if (RegularRegion() != nullptr) + { + RegularRegion()->Deactivate(heap); + RegularRegion() = nullptr; + } + + if (LargeRegion() != nullptr) + { + LargeRegion()->Deactivate(heap); + LargeRegion() = nullptr; + } + //TODO: VS also maybe allocated bytes accounting? + } diff --git a/src/coreclr/src/gc/satori/SatoriAllocationContext.h b/src/coreclr/src/gc/satori/SatoriAllocationContext.h new file mode 100644 index 00000000000..fe5ce981529 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriAllocationContext.h @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_ALLOCATION_CONTEXT_H__ +#define __SATORI_ALLOCATION_CONTEXT_H__ + +#include "common.h" +#include "../gc.h" + +class SatoriRegion; +class SatoriHeap; + +class SatoriAllocationContext : public gc_alloc_context +{ +public: + SatoriRegion*& RegularRegion() + { + return (SatoriRegion*&)this->gc_reserved_1; + } + + SatoriRegion*& LargeRegion() + { + return (SatoriRegion*&)this->gc_reserved_2; + } + + void OnTerminateThread(SatoriHeap* heap); + +private: + +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriAllocator.cpp b/src/coreclr/src/gc/satori/SatoriAllocator.cpp new file mode 100644 index 00000000000..c9d9b31e03d --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriAllocator.cpp @@ -0,0 +1,228 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriUtil.h" + +#include "SatoriHeap.h" +#include "SatoriPage.h" +#include "SatoriAllocator.h" +#include "SatoriRegion.h" +#include "SatoriRegionQueue.h" +#include "SatoriAllocationContext.h" + +void SatoriAllocator::Initialize(SatoriHeap* heap) +{ + m_heap = heap; + + for (int i = 0; i < Satori::BUCKET_COUNT; i++) + { + m_queues[i] = new SatoriRegionQueue(); + } +} + +SatoriRegion* SatoriAllocator::GetRegion(size_t regionSize) +{ +tryAgain: + + SatoriRegion* putBack = nullptr; + + int bucket = SizeToBucket(regionSize); + SatoriRegion* region = m_queues[bucket]->TryRemove(regionSize, putBack); + if (region) + { + if (putBack) + { + ReturnRegion(putBack); + } + + return region; + } + + while (++bucket < Satori::BUCKET_COUNT) + { + region = m_queues[bucket]->TryPop(regionSize, putBack); + if (region) + { + if (putBack) + { + ReturnRegion(putBack); + } + + return region; + } + } + + if (regionSize < Satori::PAGE_SIZE_GRANULARITY / 2) + { + SatoriPage* page = nullptr; + if (m_heap->TryAddRegularPage(page)) + { + if (page) + { + region = page->MakeInitialRegion(); + putBack = region->Split(region->Size() - regionSize); + AddRegion(putBack); + return region; + } + + // Someone added a page, but not us. There could be more regions. + goto tryAgain; + } + } + else + { + SatoriPage* page = m_heap->AddLargePage(regionSize); + if (page) + { + putBack = page->MakeInitialRegion(); + region = putBack->Split(regionSize); + AddRegion(putBack); + return region; + } + } + + // most likely OOM + return nullptr; +} + +void SatoriAllocator::ReturnRegion(SatoriRegion* region) +{ + // TODO: Push or enqueue? + // TODO: VS select by current core + m_queues[SizeToBucket(region->Size())]->Push(region); +} + +void SatoriAllocator::AddRegion(SatoriRegion* region) +{ + // TODO: VS select by current core + m_queues[SizeToBucket(region->Size())]->Enqueue(region); +} + +Object* SatoriAllocator::Alloc(SatoriAllocationContext* context, size_t size, uint32_t flags) +{ + size = ALIGN_UP(size, sizeof(size_t)); + + if (context->alloc_ptr + size <= context->alloc_limit) + { + Object* result = (Object*)context->alloc_ptr; + context->alloc_ptr += size; + return result; + } + + if (size < Satori::LARGE_OBJECT_THRESHOLD) + { + return AllocSmall(context, size, flags); + } + + return AllocLarge(context, size, flags); +} + +Object* SatoriAllocator::AllocSmall(SatoriAllocationContext* context, size_t size, uint32_t flags) +{ + SatoriRegion* region = context->RegularRegion(); + + while (true) + { + if (region != nullptr && + region->AllocEnd() - (size_t)context->alloc_ptr > size + Satori::MIN_FREE_SIZE) + { + size_t alloc = context->alloc_ptr + size - context->alloc_limit; + bool isZeroing = true; + if (isZeroing && alloc < Satori::MIN_REGULAR_ALLOC) + { + alloc = min(region->AllocEnd() - Satori::MIN_FREE_SIZE - (size_t)context->alloc_limit, alloc + Satori::MIN_REGULAR_ALLOC); + } + + if (region->Allocate(alloc, isZeroing)) + { + context->alloc_bytes += alloc; + context->alloc_limit += alloc; + + Object* result = (Object*)context->alloc_ptr; + context->alloc_ptr += size; + // clean syncblock. + ((size_t*)result)[-1] = 0; + return result; + } + } + + if (region != nullptr) + { + // unclaim unused. + context->alloc_bytes -= context->alloc_limit - context->alloc_ptr; + + // make parsable? + + // TODO: VS try compact current + + m_heap->Recycler()->AddRegion(region); + context->alloc_ptr = context->alloc_limit = nullptr; + } + + region = GetRegion(Satori::REGION_SIZE_GRANULARITY); + if (region == nullptr) + { + return nullptr; + } + + context->alloc_ptr = context->alloc_limit = (uint8_t*)region->AllocStart(); + context->RegularRegion() = region; + } +} + +Object* SatoriAllocator::AllocLarge(SatoriAllocationContext* context, size_t size, uint32_t flags) +{ + size_t result = 0; + + // try large region first, if present + SatoriRegion* region = context->LargeRegion(); + if (region) + { + result = region->Allocate(size, true); + if (result) + { + goto done; + } + } + + size_t regionSize = ALIGN_UP(size + sizeof(SatoriRegion), Satori::REGION_SIZE_GRANULARITY); + region = GetRegion(regionSize); + + if (regionSize == Satori::REGION_SIZE_GRANULARITY) + { + result = region->Allocate(size, true); + } + else + { + result = region->AllocateHuge(size, true); + } + + if (context->LargeRegion() == nullptr) + { + context->LargeRegion() = region; + } + else + { + if (context->LargeRegion()->AllocSize() < region->AllocSize()) + { + SatoriRegion* tmp = context->LargeRegion(); + context->LargeRegion() = region; + region = tmp; + } + + m_heap->Recycler()->AddRegion(region); + } + +done: + // clean syncblock. + ((size_t*)result)[-1] = 0; + context->alloc_bytes_uoh += size; + return (Object*)result; +} diff --git a/src/coreclr/src/gc/satori/SatoriAllocator.h b/src/coreclr/src/gc/satori/SatoriAllocator.h new file mode 100644 index 00000000000..b3f070d8c0d --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriAllocator.h @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_ALLOCATOR_H__ +#define __SATORI_ALLOCATOR_H__ + +#include "common.h" +#include "../gc.h" +#include "SatoriUtil.h" + +#include "SatoriRegionQueue.h" + +class SatoriHeap; +class SatoriRegion; +class SatoriAllocationContext; + +class SatoriAllocator +{ +public: + void Initialize(SatoriHeap* heap); + SatoriRegion* GetRegion(size_t minSize); + + void ReturnRegion(SatoriRegion* region); + void AddRegion(SatoriRegion* region); + Object* Alloc(SatoriAllocationContext* context, size_t size, uint32_t flags); + +private: + SatoriHeap* m_heap; + //TODO: VS embed + SatoriRegionQueue* m_queues[Satori::BUCKET_COUNT]; + + Object* AllocLarge(SatoriAllocationContext* context, size_t size, uint32_t flags); + Object* AllocSmall(SatoriAllocationContext* context, size_t size, uint32_t flags); + + static int SizeToBucket(size_t size) + { + _ASSERTE(size >= Satori::REGION_SIZE_GRANULARITY); + + DWORD highestBit; +#ifdef HOST_64BIT + BitScanReverse64(&highestBit, size); +#else + BitScanReverse(&highestBit, value); +#endif + return min(highestBit - Satori::REGION_BITS, Satori::BUCKET_COUNT - 1); + } +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriGCHeap.cpp b/src/coreclr/src/gc/satori/SatoriGCHeap.cpp index cb41452ab9b..a75c3fff8e7 100644 --- a/src/coreclr/src/gc/satori/SatoriGCHeap.cpp +++ b/src/coreclr/src/gc/satori/SatoriGCHeap.cpp @@ -10,15 +10,11 @@ #include "gcenv.h" #include "../env/gcenv.os.h" -//#include "../env/gcenv.structs.h" -//#include "../env/gcenv.base.h" -//#include "../env/gcenv.os.h" -//#include "../env/gcenv.interlocked.h" -//#include "../env/gcenv.interlocked.inl" -//#include "../env/gcenv.object.h" -//#include "../env/gcenv.sync.h" +#include "SatoriUtil.h" #include "SatoriGCHeap.h" +#include "SatoriAllocationContext.h" +#include "SatoriHeap.h" bool SatoriGCHeap::IsValidSegmentSize(size_t size) { @@ -180,7 +176,7 @@ uint64_t SatoriGCHeap::GetTotalAllocatedBytes() HRESULT SatoriGCHeap::GarbageCollect(int generation, bool low_memory_p, int mode) { - __UNREACHABLE(); + // TODO: Satori return S_OK; } @@ -234,8 +230,15 @@ void InitWriteBarrier() HRESULT SatoriGCHeap::Initialize() { - m_perfCounterFrquency = GCToOSInterface::QueryPerformanceFrequency(); + m_perfCounterFrequency = GCToOSInterface::QueryPerformanceFrequency(); InitWriteBarrier(); + SatoriUtil::Initialize(); + m_heap = SatoriHeap::Create(); + if (m_heap == nullptr) + { + return E_OUTOFMEMORY; + } + return S_OK; } @@ -247,8 +250,9 @@ bool SatoriGCHeap::IsPromoted(Object* object) bool SatoriGCHeap::IsHeapPointer(void* object, bool small_heap_only) { - //TODO: Satori Alloc - return false; + return m_heap->IsHeapAddress((uint8_t*)object); + + //TODO: Satori small_heap_only ? } unsigned SatoriGCHeap::GetCondemnedGeneration() @@ -287,6 +291,8 @@ uint32_t SatoriGCHeap::WaitUntilGCComplete(bool bConsiderGCStart) void SatoriGCHeap::FixAllocContext(gc_alloc_context* acontext, void* arg, void* heap) { + // this is only called when thread is terminating and about to clear its context. + ((SatoriAllocationContext*)acontext)->OnTerminateThread(m_heap); } size_t SatoriGCHeap::GetCurrentObjSize() @@ -331,22 +337,12 @@ size_t SatoriGCHeap::GetLastGCDuration(int generation) size_t SatoriGCHeap::GetNow() { int64_t t = GCToOSInterface::QueryPerformanceCounter(); - return (size_t)(t / (m_perfCounterFrquency / 1000)); -} - -inline size_t Align(size_t nbytes) -{ - return (nbytes + 7) & ~7; + return (size_t)(t / (m_perfCounterFrequency / 1000)); } Object* SatoriGCHeap::Alloc(gc_alloc_context* acontext, size_t size, uint32_t flags) { - size_t objectSize = Align(size); - uint8_t* data = new uint8_t[objectSize]; - memset(data, 0, objectSize); - - // skip syncblock word - return (Object*)(data + sizeof(size_t)); + return m_heap->Allocator()->Alloc((SatoriAllocationContext*)acontext, size, flags); } void SatoriGCHeap::PublishObject(uint8_t* obj) diff --git a/src/coreclr/src/gc/satori/SatoriGCHeap.h b/src/coreclr/src/gc/satori/SatoriGCHeap.h index 3f87ca0c678..32f5f475d39 100644 --- a/src/coreclr/src/gc/satori/SatoriGCHeap.h +++ b/src/coreclr/src/gc/satori/SatoriGCHeap.h @@ -11,10 +11,17 @@ #include "common.h" #include "../gc.h" +class SatoriHeap; +class SatoriAllocator; +class SatoriRecycler; + class SatoriGCHeap : public IGCHeapInternal { private: - int64_t m_perfCounterFrquency; + int64_t m_perfCounterFrequency; + SatoriHeap* m_heap; + SatoriAllocator* m_allocator; + SatoriRecycler* m_recycler; // what is the difference between these two? // should these be volatile diff --git a/src/coreclr/src/gc/satori/SatoriHeap.cpp b/src/coreclr/src/gc/satori/SatoriHeap.cpp new file mode 100644 index 00000000000..809587c5af2 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriHeap.cpp @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriUtil.h" + +#include "SatoriHeap.h" + + +SatoriHeap* SatoriHeap::Create() +{ + // we need to cover the whole possible address space (48bit, 52 on rare ARM64). + // Half-byte per 4Gb + const int availableAddressSpaceBits = 48; + const int pageCountBits = availableAddressSpaceBits - Satori::PAGE_BITS; + const int mapSize = 1 << pageCountBits; + size_t rezerveSize = mapSize + sizeof(SatoriHeap); + + void* reserved = GCToOSInterface::VirtualReserve(rezerveSize, 0, VirtualReserveFlags::None); + + size_t commitSize = min(GCToOSInterface::GetPageSize(), rezerveSize); + if (!GCToOSInterface::VirtualCommit(reserved, commitSize)) + { + // failure + GCToOSInterface::VirtualRelease(reserved, rezerveSize); + return nullptr; + } + + SatoriHeap* result = (SatoriHeap*)reserved; + result->m_reservedMapSize = mapSize; + result->m_committedMapSize = (int)(commitSize - ((size_t)&result->m_pageMap - (size_t)result)); + result->m_mapLock.Initialize(); + result->m_nextPageIndex = 1; + + result->m_allocator.Initialize(result); + result->m_recycler.Initialize(result); + + return result; +} + +bool SatoriHeap::CommitMoreMap(int currentlyCommitted) +{ + void* commitFrom = &m_pageMap[currentlyCommitted]; + size_t commitSize = GCToOSInterface::GetPageSize(); + + SatoriLockHolder holder(&m_mapLock); + if (currentlyCommitted < m_committedMapSize) + { + if (GCToOSInterface::VirtualCommit(commitFrom, commitSize)) + { + // we did the commit + m_committedMapSize = min(currentlyCommitted + (int)commitSize, m_reservedMapSize); + } + } + + // either we did commit or someone else did, otherwise this is a failure. + return m_committedMapSize > currentlyCommitted; +} + +bool SatoriHeap::TryAddRegularPage(SatoriPage*& newPage) +{ + int nextPageIndex = m_nextPageIndex; + for (int i = nextPageIndex; i < m_reservedMapSize; i++) + { + int currentMapSize = m_committedMapSize; + if (i >= m_committedMapSize && !CommitMoreMap(currentMapSize)) + { + break; + } + + size_t pageAddress = (size_t)i << Satori::PAGE_BITS; + newPage = SatoriPage::InitializeAt(pageAddress, Satori::PAGE_SIZE_GRANULARITY); + if (newPage) + { + // mark the map, before an object can be allocated in the new page and + // may be seen in a GC barrier + m_pageMap[i] = 1; + + // we also need to ensure that the other thread doing the barrier, + // reads the object before reading the updated map. + // on ARM this would require load fence in the barrier. We will do a processwide here instead. +#if defined(HOST_ARM64) || defined(HOST_ARM) + GCToOSInterface::FlushProcessWriteBuffers(); +#endif + + // ensure the next is advanced to at least i + 1 + while ((nextPageIndex = m_nextPageIndex) < i + 1 && + Interlocked::CompareExchange(&m_nextPageIndex, i + 1, nextPageIndex) != nextPageIndex); + + return true; + } + + // check if someone else added a page + if (m_nextPageIndex != nextPageIndex) + { + return true; + } + } + + // most likely OOM + return false; +} + +// it is harder to find a contiguous space for a large page +// also unlikely that we cross-use one if just comitted +// we scan ahead and claim, but do not move m_nextPageIndex +// unless we claimed contiguously. +SatoriPage* SatoriHeap::AddLargePage(size_t minSize) +{ + //TODO: VS adjust minSize to include overhead. Can do rough %% + // for now 1/16, about 6% + minSize += minSize / 16; + + minSize = ALIGN_UP(minSize, Satori::PAGE_SIZE_GRANULARITY); + int mapMarkCount = (int)(minSize >> Satori::PAGE_BITS) + 1; + + for (int i = m_nextPageIndex; i < m_reservedMapSize; i++) + { + int currentMapSize = m_committedMapSize; + while (i + mapMarkCount >= m_committedMapSize) + { + if (!CommitMoreMap(currentMapSize)) + { + return nullptr; + } + } + + size_t pageAddress = (size_t)i << Satori::PAGE_BITS; + SatoriPage* newPage = SatoriPage::InitializeAt(pageAddress, minSize); + if (newPage) + { + // mark the map, before an object can be allocated in the new page and + // may be seen in a GC barrier + m_pageMap[i] = 1; + for (int j = 1; j < mapMarkCount; j++) + { + // TODO: VS skip-marks + m_pageMap[i + j] = 2; + } + + // we also need to ensure that the other thread doing the barrier, + // reads the object before reading the updated map. + // on ARM this would require load fence in the barrier. We will do a processwide here instead. +#if defined(HOST_ARM64) || defined(HOST_ARM) + GCToOSInterface::FlushProcessWriteBuffers(); +#endif + + + // advance next if contiguous + // Note: it may become contiguous after we check, but it should be rare. + // advancing is not needed for correctness, so we do not care. + if (i == m_nextPageIndex) + { + Interlocked::CompareExchange(&m_nextPageIndex, i + mapMarkCount, i); + } + + return newPage; + } + } + + return nullptr; +} + +SatoriPage* SatoriHeap::PageForAddress(uint8_t* address) +{ + return nullptr; +} diff --git a/src/coreclr/src/gc/satori/SatoriHeap.h b/src/coreclr/src/gc/satori/SatoriHeap.h new file mode 100644 index 00000000000..82971f7f078 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriHeap.h @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_HEAP_H__ +#define __SATORI_HEAP_H__ + +#include "common.h" +#include "../gc.h" +#include "SatoriUtil.h" + +#include "SatoriLock.h" +#include "SatoriPage.h" +#include "SatoriAllocator.h" +#include "SatoriRecycler.h" + +class SatoriHeap +{ +public: + SatoriHeap() = delete; + ~SatoriHeap() = delete; + + static SatoriHeap* Create(); + + bool TryAddRegularPage(SatoriPage*& newPage); + SatoriPage* AddLargePage(size_t size); + + SatoriAllocator* Allocator() + { + return &m_allocator; + } + + SatoriRecycler* Recycler() + { + return &m_recycler; + } + + SatoriPage* PageForAddress(uint8_t* address); + + bool IsHeapAddress(uint8_t* address) + { + size_t mapIndex = (size_t)address >> Satori::PAGE_BITS; + return (unsigned int)mapIndex < (unsigned int)m_committedMapSize && + m_pageMap[mapIndex] != 0; + } + +private: + SatoriAllocator m_allocator; + SatoriRecycler m_recycler; + + int m_reservedMapSize; + int m_committedMapSize; + int m_nextPageIndex; + SatoriLock m_mapLock; + uint8_t m_pageMap[1]; + + bool CommitMoreMap(int currentlyCommitted); +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriLock.cpp b/src/coreclr/src/gc/satori/SatoriLock.cpp new file mode 100644 index 00000000000..1ce8fdfcabf --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriLock.cpp @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriLock.h" + diff --git a/src/coreclr/src/gc/satori/SatoriLock.h b/src/coreclr/src/gc/satori/SatoriLock.h new file mode 100644 index 00000000000..3f9e3e847ac --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriLock.h @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_LOCK_H__ +#define __SATORI_LOCK_H__ + +#include "common.h" +#include "../gc.h" + +class SatoriLock +{ +private: + CLRCriticalSection m_cs; + +public: + void Initialize() + { + m_cs.Initialize(); + } + + void Destroy() + { + m_cs.Destroy(); + } + + void Enter() + { + m_cs.Enter(); + } + + void Leave() + { + m_cs.Leave(); + } +}; + +class SatoriSpinLock +{ +private: + +public: + +}; + +// TODO: VS move to util +class StackOnly { +private: + void* operator new(size_t size) throw() = delete; + void* operator new[](size_t size) throw() = delete; +}; + + +template +class SatoriLockHolder : public StackOnly { +private: + T* const m_lock; + +public: + // Disallow copying + SatoriLockHolder& operator=(const SatoriLockHolder&) = delete; + SatoriLockHolder(const SatoriLockHolder&) = delete; + + SatoriLockHolder(T* lock) + : m_lock(lock) + { + m_lock->Enter(); + } + + ~SatoriLockHolder() + { + m_lock->Leave(); + } +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriPage.cpp b/src/coreclr/src/gc/satori/SatoriPage.cpp new file mode 100644 index 00000000000..b3cf654fc90 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriPage.cpp @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriPage.h" +#include "SatoriRegion.h" + +SatoriPage* SatoriPage::InitializeAt(size_t address, size_t pageSize) +{ + _ASSERTE(pageSize % Satori::PAGE_SIZE_GRANULARITY == 0); + + SatoriPage* result = (SatoriPage*)GCToOSInterface::VirtualReserve((void*)address, pageSize); + if (result == nullptr) + { + return result; + } + + size_t headerSize = ALIGN_UP(sizeof(SatoriPage), Satori::REGION_SIZE_GRANULARITY); + size_t commitSize = headerSize; + if (!GCToOSInterface::VirtualCommit((void*)address, commitSize)) + { + GCToOSInterface::VirtualRelease((void*)address, pageSize); + return nullptr; + } + + result->m_end = address + pageSize; + result->m_firstRegion = address + headerSize; + result->m_initialCommit = address + commitSize; + return result; +} + +SatoriRegion* SatoriPage::MakeInitialRegion() +{ + return SatoriRegion::InitializeAt(this, m_firstRegion, m_end - m_firstRegion, m_initialCommit, m_firstRegion); +} + +void SatoriPage::RegionAdded(size_t address) +{ +} + +void SatoriPage::RegionDestroyed(size_t address) +{ +} diff --git a/src/coreclr/src/gc/satori/SatoriPage.h b/src/coreclr/src/gc/satori/SatoriPage.h new file mode 100644 index 00000000000..18cd6c4b270 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriPage.h @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_PAGE_H__ +#define __SATORI_PAGE_H__ + +#include "common.h" +#include "../gc.h" + +class SatoriRegion; + +class SatoriPage +{ +public: + SatoriPage() = delete; + ~SatoriPage() = delete; + + static SatoriPage* InitializeAt(size_t address, size_t pageSize); + SatoriRegion* MakeInitialRegion(); + + void RegionAdded(size_t address); + void RegionDestroyed(size_t address); + + size_t End() + { + return m_end; + } + + size_t RegionsStart() + { + return m_firstRegion; + } + +private: + size_t m_end; + size_t m_initialCommit; + size_t m_firstRegion; +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriRecycler.cpp b/src/coreclr/src/gc/satori/SatoriRecycler.cpp new file mode 100644 index 00000000000..27a9b2a4dde --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRecycler.cpp @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" + +#include "SatoriHeap.h" +#include "SatoriRecycler.h" + +void SatoriRecycler::Initialize(SatoriHeap* heap) +{ + m_heap = heap; +} + +void SatoriRecycler::AddRegion(SatoriRegion* region) +{ + // TODO: VS make end parsable? + + // TODO: VS leak the region for now +} diff --git a/src/coreclr/src/gc/satori/SatoriRecycler.h b/src/coreclr/src/gc/satori/SatoriRecycler.h new file mode 100644 index 00000000000..53ae55e179e --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRecycler.h @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_RECYCLER_H__ +#define __SATORI_RECYCLER_H__ + +#include "common.h" +#include "../gc.h" + +class SatoriHeap; +class SatoriRegion; + +class SatoriRecycler +{ +public: + void Initialize(SatoriHeap* heap); + void AddRegion(SatoriRegion* region); + +private: + SatoriHeap* m_heap; +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriRegion.cpp b/src/coreclr/src/gc/satori/SatoriRegion.cpp new file mode 100644 index 00000000000..b24c29cc330 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRegion.cpp @@ -0,0 +1,296 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriGCHeap.h" +#include "SatoriAllocator.h" +#include "SatoriRecycler.h" +#include "SatoriUtil.h" +#include "SatoriRegion.h" + +SatoriRegion* SatoriRegion::InitializeAt(SatoriPage* containingPage, size_t address, size_t regionSize, size_t committed, size_t zeroInitedAfter) +{ + _ASSERTE(zeroInitedAfter <= committed); + _ASSERTE(regionSize % Satori::REGION_SIZE_GRANULARITY == 0); + + committed = max(address, committed); + zeroInitedAfter = max(address, zeroInitedAfter); + + SatoriRegion* result = (SatoriRegion*)address; + + ptrdiff_t toCommit = address + sizeof(SatoriRegion) - committed; + if (toCommit > 0) + { + _ASSERTE(GCToOSInterface::GetPageSize() < Satori::REGION_SIZE_GRANULARITY); + toCommit = ALIGN_UP(toCommit, GCToOSInterface::GetPageSize()); + + if (!GCToOSInterface::VirtualCommit((void*)committed, toCommit)) + { + return nullptr; + } + committed += toCommit; + } + + result->m_end = address + regionSize; + result->m_committed = min(committed, address + regionSize); + result->m_zeroInitedAfter = min(max(zeroInitedAfter, address + sizeof(SatoriRegion)), result->End()); + result->m_containingPage = containingPage; + + result->m_next = result->m_prev = nullptr; + result->m_containingQueue = nullptr; + + result->m_allocStart = (size_t)&result->m_firstObject; + // +1 for syncblock + result->m_allocEnd = result->m_end + 1; + + if (zeroInitedAfter > (size_t)&result->m_index) + { + ZeroMemory(&result->m_index, offsetof(SatoriRegion, m_syncBlock) - offsetof(SatoriRegion, m_index)); + } + + result->m_containingPage->RegionAdded(address); + return result; +} + +void SatoriRegion::MakeBlank() +{ + m_allocStart = (size_t)&m_firstObject; + // +1 for syncblock + m_allocEnd = m_end + 1; + + if (m_zeroInitedAfter > (size_t)&m_index) + { + ZeroMemory(&m_index, offsetof(SatoriRegion, m_syncBlock) - offsetof(SatoriRegion, m_index)); + } +} + +bool SatoriRegion::ValidateBlank() +{ + if (!IsAllocating()) + { + return false; + } + + if (Start() < m_containingPage->RegionsStart() || End() > m_containingPage->End()) + { + return false; + } + + if (Size() % Satori::REGION_SIZE_GRANULARITY != 0) + { + return false; + } + + if (m_committed > End() || m_committed < Start() + sizeof(SatoriRegion)) + { + return false; + } + + if (m_zeroInitedAfter > m_committed || m_zeroInitedAfter < Start() + sizeof(SatoriRegion)) + { + return false; + } + + if (m_allocStart != (size_t)&m_firstObject || m_allocEnd != m_end + 1) + { + return false; + } + + for (int i = 0; i < Satori::INDEX_ITEMS; i += Satori::INDEX_ITEMS / 4) + { + if (m_index[i] != nullptr) + { + return false; + } + } + + if (m_syncBlock != 0) + { + return false; + } + + return true; +} + +void SatoriRegion::StopAllocating() +{ + _ASSERTE(IsAllocating()); + + SatoriUtil::MakeFreeObject((Object*)m_allocStart, AllocSize()); + // TODO: VS update index + m_allocEnd = 0; +} + +void SatoriRegion::Deactivate(SatoriHeap* heap) +{ + StopAllocating(); + + // local collect (leaves it parsable \w updated index) + + if (IsEmpty()) + { + this->MakeBlank(); + heap->Allocator()->ReturnRegion(this); + return; + } + + // TODO: VS: if can be splitd, split and return tail + heap->Recycler()->AddRegion(this); +} + +bool SatoriRegion::IsEmpty() +{ + _ASSERTE(!IsAllocating()); + return (size_t)SatoriUtil::Next(&m_firstObject) > m_end; +} + +void SatoriRegion::SplitCore(size_t regionSize, size_t& nextStart, size_t& nextCommitted, size_t& nextZeroInitedAfter) +{ + _ASSERTE(regionSize % Satori::REGION_SIZE_GRANULARITY == 0); + _ASSERTE(regionSize < this->Size()); + _ASSERTE(m_allocEnd == m_end + 1); + _ASSERTE((size_t)m_allocStart < m_end - regionSize - Satori::MIN_FREE_SIZE); + + size_t newEnd = m_end - regionSize; + nextStart = newEnd; + nextCommitted = m_committed; + nextZeroInitedAfter = m_zeroInitedAfter; + + m_end = newEnd; + m_committed = min(newEnd, m_committed); + m_zeroInitedAfter = min(newEnd, m_zeroInitedAfter); + m_allocEnd = min(newEnd + 1, m_allocEnd); + + _ASSERTE(Size() >= Satori::REGION_SIZE_GRANULARITY); + _ASSERTE(IsAllocating()); +} + +SatoriRegion* SatoriRegion::Split(size_t regionSize) +{ + size_t nextStart, nextCommitted, nextZeroInitedAfter; + SplitCore(regionSize, nextStart, nextCommitted, nextZeroInitedAfter); + + // format the rest as a new region + SatoriRegion* result = InitializeAt(m_containingPage, nextStart, regionSize, nextCommitted, nextZeroInitedAfter); + _ASSERTE(result->ValidateBlank()); + return result; +} + +bool SatoriRegion::CanCoalesce(SatoriRegion* other) +{ + return m_committed == other->Start(); +} + +void SatoriRegion::Coalesce(SatoriRegion* next) +{ + _ASSERTE(ValidateBlank()); + _ASSERTE(next->ValidateBlank()); + _ASSERTE(next->m_prev == next->m_next); + _ASSERTE(next->m_containingQueue == nullptr); + _ASSERTE(next->m_containingPage == m_containingPage); + _ASSERTE(CanCoalesce(next)); + + m_containingPage->RegionDestroyed((size_t)next); + + m_end = next->m_end; + m_committed = next->m_committed; + m_zeroInitedAfter = next->m_zeroInitedAfter; + m_allocEnd = next->m_allocEnd; +} + +size_t SatoriRegion::Allocate(size_t size, bool ensureZeroInited) +{ + _ASSERTE(m_containingQueue == nullptr); + size_t allocStart = m_allocStart; + size_t allocEnd = m_allocStart + size; + if (allocEnd <= m_allocEnd - Satori::MIN_FREE_SIZE || allocEnd == m_allocEnd) + { + if (allocEnd > m_committed) + { + // TODO: VS commit quantum should be a static (ensure it is power of 2 and < 2Mb, otherwise committ whole pages) + size_t newComitted = ALIGN_UP(allocEnd, GCToOSInterface::GetPageSize()); + if (!GCToOSInterface::VirtualCommit((void*)m_committed, newComitted - m_committed)) + { + return 0; + } + + m_committed = newComitted; + } + + if (ensureZeroInited) + { + size_t zeroInitStart = allocStart; + size_t zeroInitEnd = min(m_zeroInitedAfter, allocEnd); + ptrdiff_t zeroInitCount = zeroInitEnd - zeroInitStart; + if (zeroInitCount > 0) + { + ZeroMemory((void*)zeroInitStart, zeroInitCount); + } + } + + size_t result = m_allocStart; + m_allocStart = allocEnd; + m_zeroInitedAfter = max(m_zeroInitedAfter, allocEnd); + return result; + } + + return 0; +} + +size_t SatoriRegion::AllocateHuge(size_t size, bool ensureZeroInited) +{ + _ASSERTE(ValidateBlank()); + _ASSERTE(AllocSize() >= size); + + size_t allocStart; + size_t allocEnd; + if (AllocSize() - size > Satori::LARGE_OBJECT_THRESHOLD) + { + allocEnd = max(m_allocStart + size, m_committed); + allocStart = allocEnd - size; + } + else + { + allocStart = m_allocStart; + allocEnd = m_allocStart + size; + } + + _ASSERTE(allocEnd > Start() + Satori::REGION_SIZE_GRANULARITY); + + if (allocEnd > m_committed) + { + // TODO: VS commit quantum should be a static (ensure it is power of 2 and < 2Mb, otherwise commit whole pages) + size_t newComitted = ALIGN_UP(allocEnd, GCToOSInterface::GetPageSize()); + if (!GCToOSInterface::VirtualCommit((void*)m_committed, newComitted - m_committed)) + { + return 0; + } + + m_committed = newComitted; + } + + // if did not allocate from start, there could be some fillable space before the obj. + m_allocEnd = allocStart; + if (ensureZeroInited) + { + size_t zeroInitStart = allocStart; + size_t zeroInitEnd = min(m_zeroInitedAfter, allocEnd); + ptrdiff_t zeroInitCount = zeroInitEnd - zeroInitStart; + if (zeroInitCount > 0) + { + ZeroMemory((void*)zeroInitStart, zeroInitCount); + } + } + + //TODO: VS update index? + size_t result = m_allocStart; + m_zeroInitedAfter = max(m_zeroInitedAfter, allocEnd); + return result; +} diff --git a/src/coreclr/src/gc/satori/SatoriRegion.h b/src/coreclr/src/gc/satori/SatoriRegion.h new file mode 100644 index 00000000000..69132956cdd --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRegion.h @@ -0,0 +1,106 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_REGION_H__ +#define __SATORI_REGION_H__ + +#include "common.h" +#include "../gc.h" +#include "SatoriHeap.h" +#include "SatoriUtil.h" + +class SatoriRegion +{ + friend class SatoriRegionQueue; + + +public: + SatoriRegion() = delete; + ~SatoriRegion() = delete; + + static SatoriRegion* InitializeAt(SatoriPage* containingPage, size_t address, size_t regionSize, size_t committed, size_t zeroInitedAfter); + void MakeBlank(); + bool ValidateBlank(); + void StopAllocating(); + + SatoriRegion* Split(size_t regionSize); + bool CanCoalesce(SatoriRegion* other); + void Coalesce(SatoriRegion* next); + + void Deactivate(SatoriHeap* heap); + + size_t Allocate(size_t size, bool ensureZeroInited); + size_t AllocateHuge(size_t size, bool ensureZeroInited); + + bool IsAllocating() + { + return m_allocEnd != 0; + } + + size_t Start() + { + return (size_t)&m_end; + } + + size_t End() + { + return m_end; + } + + size_t Size() + { + return End() - Start(); + } + + size_t AllocStart() + { + return m_allocStart; + } + + size_t AllocEnd() + { + return m_allocEnd; + } + + size_t AllocSize() + { + return m_allocEnd - m_allocStart; + } + + static SatoriRegion* RegionForObject(Object* obj) + { + return (SatoriRegion*)(((size_t)obj) >> Satori::REGION_BITS); + } + +private: + // end is edge exclusive + size_t m_end; + size_t m_committed; + size_t m_zeroInitedAfter; + SatoriPage* m_containingPage; + + SatoriRegion* m_prev; + SatoriRegion* m_next; + SatoriRegionQueue* m_containingQueue; + + // active allocation may happen in the following range. + // the range may not be parseable as sequence of objects + // NB: it is in terms of objects, if converting to size_t beware of sync blocks + size_t m_allocStart; + size_t m_allocEnd; + + Object* m_index[Satori::INDEX_ITEMS]; + + size_t m_syncBlock; + Object m_firstObject; + +private: + void SplitCore(size_t regionSize, size_t& newStart, size_t& newCommitted, size_t& newZeroInitedAfter); + bool IsEmpty(); +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriRegionQueue.cpp b/src/coreclr/src/gc/satori/SatoriRegionQueue.cpp new file mode 100644 index 00000000000..58e5999fd9b --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRegionQueue.cpp @@ -0,0 +1,260 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriRegionQueue.h" + +#include "SatoriLock.h" +#include "SatoriUtil.h" + +#include "SatoriRegion.h" + +void SatoriRegionQueue::Push(SatoriRegion* region) +{ + _ASSERTE(region->ValidateBlank()); + + SatoriLockHolder holder(&m_lock); + region->m_containingQueue = this; + if (m_head == nullptr) + { + _ASSERTE(m_tail == nullptr); + m_head = m_tail = region; + } + else + { + region->m_next = m_head; + m_head->m_prev = region; + m_head = region; + } +} + +void SatoriRegionQueue::Enqueue(SatoriRegion* region) +{ + _ASSERTE(region->ValidateBlank()); + + SatoriLockHolder holder(&m_lock); + region->m_containingQueue = this; + if (m_tail == nullptr) + { + _ASSERTE(m_head == nullptr); + m_head = m_tail = region; + } + else + { + region->m_prev = m_tail; + m_tail->m_next = region; + m_tail = region; + } +} + +SatoriRegion* SatoriRegionQueue::TryPop() +{ + SatoriRegion* result; + { + SatoriLockHolder holder(&m_lock); + result = m_head; + if (result == nullptr) + { + return nullptr; + } + + result->m_containingQueue = nullptr; + m_head = result->m_next; + if (m_head == nullptr) + { + m_tail = nullptr; + } + else + { + m_head->m_prev = nullptr; + } + } + + _ASSERTE(result->m_prev == nullptr); + result->m_next = nullptr; + + _ASSERTE(result->ValidateBlank()); + return result; +} + +SatoriRegion* SatoriRegionQueue::TryPop(size_t regionSize, SatoriRegion*& putBack) +{ + m_lock.Enter(); + + SatoriRegion* result = m_head; + if (result == nullptr) + { + m_lock.Leave(); + return nullptr; + } + + _ASSERTE(result->ValidateBlank()); + + // TODO: VS put back the assert when bucketized. + // _ASSERTE(result->Size() >= regionSize); + if (result->Size() < regionSize) + { + m_lock.Leave(); + return nullptr; + } + + if (result->Size() - SatoriUtil::RoundDownPwr2(result->Size()) > regionSize) + { + // inplace case + // split "size" and return as a new region + size_t nextStart, nextCommitted, nextZeroInitedAfter; + result->SplitCore(regionSize, nextStart, nextCommitted, nextZeroInitedAfter); + _ASSERTE(result->ValidateBlank()); + m_lock.Leave(); + + result = SatoriRegion::InitializeAt(result->m_containingPage, nextStart, regionSize, nextCommitted, nextZeroInitedAfter); + _ASSERTE(result->ValidateBlank()); + putBack = nullptr; + } + else + { + // take out the result + result->m_containingQueue = nullptr; + m_head = result->m_next; + if (m_head == nullptr) + { + m_tail = nullptr; + } + else + { + m_head->m_prev = nullptr; + } + + m_lock.Leave(); + + if (result->Size() > regionSize) + { + // if there is a diff split it off and put back to appropriate queue. + putBack = result->Split(result->Size() - regionSize); + } + + _ASSERTE(result->m_prev == nullptr); + result->m_next = nullptr; + } + + return result; +} + +SatoriRegion* SatoriRegionQueue::TryRemove(size_t regionSize, SatoriRegion*& putBack) +{ + m_lock.Enter(); + + SatoriRegion* result = m_head; + while (true) + { + if (result == nullptr) + { + m_lock.Leave(); + return nullptr; + } + + if (result->Size() >= regionSize) + { + break; + } + + result = result->m_next; + } + + _ASSERTE(result->ValidateBlank()); + + if (result->Size() - SatoriUtil::RoundDownPwr2(result->Size()) > regionSize) + { + // inplace case + // split "size" and return as a new region + size_t nextStart, nextCommitted, nextZeroInitedAfter; + result->SplitCore(regionSize, nextStart, nextCommitted, nextZeroInitedAfter); + _ASSERTE(result->ValidateBlank()); + m_lock.Leave(); + + result = SatoriRegion::InitializeAt(result->m_containingPage, nextStart, regionSize, nextCommitted, nextZeroInitedAfter); + _ASSERTE(result->ValidateBlank()); + putBack = nullptr; + } + else + { + // take out the result + result->m_containingQueue = nullptr; + if (result->m_prev == nullptr) + { + m_head = result->m_next; + } + else + { + result->m_prev->m_next = result->m_next; + } + + if (result->m_next == nullptr) + { + m_tail = result->m_prev; + } + else + { + result->m_next->m_prev = result->m_prev; + } + + m_lock.Leave(); + + if (result->Size() > regionSize) + { + // if there is a diff split it off and put back to appropriate queue. + putBack = result->Split(result->Size() - regionSize); + } + + result->m_prev = nullptr; + result->m_next = nullptr; + } + + return result; +} + +bool SatoriRegionQueue::Contains(SatoriRegion* region) +{ + return region->m_containingQueue == this; +} + +bool SatoriRegionQueue::TryRemove(SatoriRegion* region) +{ + { + SatoriLockHolder holder(&m_lock); + if (!Contains(region)) + { + return false; + } + + region->m_containingQueue = nullptr; + if (region->m_prev == nullptr) + { + m_head = region->m_next; + } + else + { + region->m_prev->m_next = region->m_next; + } + + if (region->m_next == nullptr) + { + m_tail = region->m_prev; + } + else + { + region->m_next->m_prev = region->m_prev; + } + } + + region->m_next = nullptr; + region->m_prev = nullptr; + return true; +} diff --git a/src/coreclr/src/gc/satori/SatoriRegionQueue.h b/src/coreclr/src/gc/satori/SatoriRegionQueue.h new file mode 100644 index 00000000000..861cb2a2ca2 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriRegionQueue.h @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_REGION_QUEUE_H__ +#define __SATORI_REGION_QUEUE_H__ + +#include "common.h" +#include "../gc.h" +#include "SatoriLock.h" + +class SatoriRegion; + +class SatoriRegionQueue +{ +public: + SatoriRegion* TryPop(); + SatoriRegion* TryPop(size_t regionSize, SatoriRegion* &putBack); + + bool TryRemove(SatoriRegion* region); + SatoriRegion* TryRemove(size_t regionSize, SatoriRegion*& putBack); + + void Push(SatoriRegion* region); + void Enqueue(SatoriRegion* region); + bool Contains(SatoriRegion* region); + + SatoriRegionQueue() : + m_lock(), m_head(), m_tail() + { + m_lock.Initialize(); + }; + + bool CanPop() + { + return m_head != nullptr; + } + + bool CanDequeue() + { + return m_tail != nullptr; + } + +private: + SatoriLock m_lock; + SatoriRegion* m_head; + SatoriRegion* m_tail; +}; + +#endif diff --git a/src/coreclr/src/gc/satori/SatoriUtil.cpp b/src/coreclr/src/gc/satori/SatoriUtil.cpp new file mode 100644 index 00000000000..c5db052f8c5 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriUtil.cpp @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.cpp +// + +#include "common.h" + +#include "gcenv.h" +#include "../env/gcenv.os.h" +#include "SatoriUtil.h" + +MethodTable* SatoriUtil::s_emptyObjectMt; + +void SatoriUtil::Initialize() +{ + s_emptyObjectMt = GCToEEInterface::GetFreeObjectMethodTable(); +} + +void SatoriUtil::MakeFreeObject(Object* obj, size_t size) +{ + _ASSERTE(size >= sizeof(Object) + sizeof(size_t)); + + // sync block + ((size_t*)obj)[-1] = 0; + + // mt + obj->RawSetMethodTable(s_emptyObjectMt); + + // size + // Note: we allow empty objects to be more than 4Gb on 64bit and use the "pad" for higher bits. +#if BIGENDIAN +#error "This won't work on big endian platforms" +#endif + ((size_t*)obj)[ArrayBase::GetOffsetOfNumComponents() / sizeof(size_t)] = size - sizeof(ArrayBase); +} + diff --git a/src/coreclr/src/gc/satori/SatoriUtil.h b/src/coreclr/src/gc/satori/SatoriUtil.h new file mode 100644 index 00000000000..e5e3e02ce04 --- /dev/null +++ b/src/coreclr/src/gc/satori/SatoriUtil.h @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +// +// SatoriGCHeap.h +// + +#ifndef __SATORI_UTIL_H__ +#define __SATORI_UTIL_H__ + +#include "common.h" +#include "../gc.h" + + +//TODO: VS rename move somwhere +namespace Satori +{ + // page granularity is 1 Gb, but they can be bigger + // (on 32 bit we will have smaller sizes) + static const int PAGE_BITS = 30; + static const size_t PAGE_SIZE_GRANULARITY = (size_t)1 << PAGE_BITS; + + // regions are aligned at 2 Mb + // (TODO: VS our commit unit must be powerof 2. must be <= 2 Mb?, otherwise large mode?) + // objects can be larger than that and straddle multiple region tiles. + // the additional "tail" tiles cannot have object starts though (makes finding region for obj easier). + const static int REGION_BITS = 21; + const static size_t REGION_SIZE_GRANULARITY = 1 << REGION_BITS; + + const static int INDEX_GRANULARITY = 4096; + const static int INDEX_ITEMS = REGION_SIZE_GRANULARITY / INDEX_GRANULARITY; + + static const int BUCKET_COUNT = PAGE_BITS - REGION_BITS; + + // objects smaller than this go into regular region. A random big number. + static const int LARGE_OBJECT_THRESHOLD = 85000; + + // TODO: VS consider: in theory could be 2, but that would complicate walking. + static const size_t MIN_FREE_SIZE = 3; + + static const size_t MIN_REGULAR_ALLOC = 1 << 12; +} + +class SatoriUtil +{ +private: + static MethodTable* s_emptyObjectMt; + +public: + static void Initialize(); + + //TODO: VS Free vs. Empty + static void MakeFreeObject(Object* obj, size_t size); + + static size_t RoundDownPwr2(size_t value) + { + _ASSERTE(value > 0); + DWORD highestBit; +#ifdef HOST_64BIT + BitScanReverse64(&highestBit, value); +#else + BitScanReverse(&highestBit, value); +#endif + return (size_t)1 << highestBit; + } + + static size_t Size(Object* obj) + { + MethodTable* mt = obj->RawGetMethodTable(); + + size_t size = mt->GetBaseSize(); + if (mt->HasComponentSize()) + { + size += (size_t)((ArrayBase*)mt)->GetNumComponents() * mt->RawGetComponentSize(); + } + + return size; + } + + static Object* Next(Object* obj) + { + return (Object*)((size_t)obj + Size(obj)); + } + + static bool SatoriUtil::IsFreeObject(Object* obj) + { + return obj->RawGetMethodTable() == s_emptyObjectMt; + } +}; + +#endif diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index f3872e8148c..9a802057b35 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -505,7 +505,17 @@ set(GC_SOURCES_WKS ../gc/gcload.cpp ../gc/softwarewritewatch.cpp ../gc/handletablecache.cpp - ../gc/satori/SatoriGCHeap.cpp) + ../gc/satori/SatoriGCHeap.cpp + ../gc/satori/SatoriHeap.cpp + ../gc/satori/SatoriPage.cpp + ../gc/satori/SatoriRegion.cpp + ../gc/satori/SatoriRegionQueue.cpp + ../gc/satori/SatoriAllocator.cpp + ../gc/satori/SatoriRecycler.cpp + ../gc/satori/SatoriAllocationContext.cpp + ../gc/satori/SatoriUtil.cpp + ../gc/satori/SatoriLock.cpp + ) if (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32) set ( GC_SOURCES_WKS @@ -529,7 +539,17 @@ set(GC_HEADERS_WKS ../gc/gcscan.h ../gc/gchandletableimpl.h ../gc/softwarewritewatch.h - ../gc/satori/SatoriGCHeap.h) + ../gc/satori/SatoriGCHeap.h + ../gc/satori/SatoriHeap.h + ../gc/satori/SatoriPage.h + ../gc/satori/SatoriRegion.h + ../gc/satori/SatoriRegionQueue.h + ../gc/satori/SatoriAllocator.h + ../gc/satori/SatoriRecycler.h + ../gc/satori/SatoriAllocationContext.h + ../gc/satori/SatoriUtil.h + ../gc/satori/SatoriLock.h + ) if(FEATURE_EVENT_TRACE) list(APPEND VM_SOURCES_WKS diff --git a/src/coreclr/vm/gcenv.os.cpp b/src/coreclr/vm/gcenv.os.cpp index 2567e0fb107..46ee941bd1e 100644 --- a/src/coreclr/vm/gcenv.os.cpp +++ b/src/coreclr/vm/gcenv.os.cpp @@ -353,6 +353,12 @@ void* GCToOSInterface::VirtualReserve(size_t size, size_t alignment, uint32_t fl } } +void* GCToOSInterface::VirtualReserve(void* location, size_t size) +{ + DWORD memFlags = MEM_RESERVE; + return ::ClrVirtualAlloc(location, size, memFlags, PAGE_READWRITE); +} + // Release virtual memory range previously reserved using VirtualReserve // Parameters: // address - starting virtual address