VSadov/Satori
1
0
Fork 0
mirror of https://github.com/VSadov/Satori.git synced 2025-06-09 09:34:49 +09:00
Code Activity

Card table functional.

Browse source
This commit is contained in:
vsadov 2020-10-01 17:02:19 -07:00
parent 0dd725d5c1
commit ab1d752c48
15 changed files with 403 additions and 207 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/coreclr/gc/env/gcenv.interlocked.inl vendored
View file

@ -109,10 +109,10 @@ __forceinline size_t Interlocked::CompareExchange<size_t>(size_t volatile * dest
}
template <>
__forceinline uint8_t Interlocked::CompareExchange<uint8_t>(uint8_t volatile* destination, uint8_t exchange, uint8_t comparand)
__forceinline int8_t Interlocked::CompareExchange<int8_t>(int8_t volatile* destination, int8_t exchange, int8_t comparand)
{
#ifdef _MSC_VER
return _InterlockedCompareExchange8((char*)destination, exchange, comparand);
return (int8_t)_InterlockedCompareExchange8((char*)destination, exchange, comparand);
#else
T result = __sync_val_compare_and_swap(destination, comparand, exchange);
ArmInterlockedOperationBarrier();

2
src/coreclr/src/gc/satori/SatoriAllocator.cpp
View file

@ -339,6 +339,8 @@ SatoriMarkChunk* SatoriAllocator::TryGetMarkChunk()
bool SatoriAllocator::AddMoreMarkChunks()
{
//TODO: VS if committing by OS page get one region at the beginning and dispense by a piece (take a lock),
// then just do whole regions.
SatoriRegion* region = GetRegion(Satori::REGION_SIZE_GRANULARITY);
if (!region)
{

37
src/coreclr/src/gc/satori/SatoriGC.cpp
View file

@ -133,8 +133,8 @@ unsigned SatoriGC::WhichGeneration(Object* obj)
int SatoriGC::CollectionCount(int generation, int get_bgc_fgc_coutn)
{
//TODO: VS this is implementable. We can just count blocking GCs.
return m_heap->Recycler()->GetScanCount();
//TODO: VS get_bgc_fgc_coutn ?.
return (int)m_heap->Recycler()->GetCollectionCount(generation);
}
int SatoriGC::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
@ -168,13 +168,15 @@ uint64_t SatoriGC::GetTotalAllocatedBytes()
HRESULT SatoriGC::GarbageCollect(int generation, bool low_memory_p, int mode)
{
// TODO: VS we do full GC for now.
m_heap->Recycler()->Collect(/*force*/ true);
generation = max(1, generation);
generation = min(2, generation);
m_heap->Recycler()->Collect(generation, /*force*/ true);
return S_OK;
}
unsigned SatoriGC::GetMaxGeneration()
{
// TODO: VS we will probably have only 0, 1 and 2.
return 2;
}
@ -237,26 +239,29 @@ HRESULT SatoriGC::Initialize()
// makes sense only during marking phases.
bool SatoriGC::IsPromoted(Object* object)
{
_ASSERTE(object == nullptr || m_heap->IsHeapAddress((size_t)object));
SatoriObject* o = (SatoriObject*)object;
// objects outside of the collected generation (including null) are considered marked.
// (existing behavior)
_ASSERTE(object == nullptr || m_heap->IsHeapAddress((size_t)object));
// TODO: VS, will need to adjust for Gen1
SatoriObject* o = (SatoriObject*)object;
return o == nullptr || o->IsMarked();
return o == nullptr ||
o->IsMarked()
// TODO: VS enable when truly generational
// || o->ContainingRegion()->Generation() > m_heap->Recycler()->CondemnedGeneration()
;
}
bool SatoriGC::IsHeapPointer(void* object, bool small_heap_only)
{
return m_heap->IsHeapAddress((size_t)object);
//TODO: Satori small_heap_only ?
return m_heap->IsHeapAddress((size_t)object);
}
unsigned SatoriGC::GetCondemnedGeneration()
{
// TODO: VS, will need to adjust for Gen1
return 2;
// TODO: VS enable when truly generational
// return m_heap->Recycler()->CondemnedGeneration();
}
bool SatoriGC::IsGCInProgressHelper(bool bConsiderGCStart)
@ -266,8 +271,12 @@ bool SatoriGC::IsGCInProgressHelper(bool bConsiderGCStart)
unsigned SatoriGC::GetGcCount()
{
//TODO: Satori Collect
return 0;
if (!m_heap)
{
return 0;
}
return (unsigned)m_heap->Recycler()->GetScanCount();
}
bool SatoriGC::IsThreadUsingAllocationContextHeap(gc_alloc_context* acontext, int thread_number)

10
src/coreclr/src/gc/satori/SatoriGC.h
View file

@ -17,6 +17,16 @@ class SatoriRecycler;
class SatoriGC : public IGCHeapInternal
{
public:
SatoriGC()
{
m_perfCounterFrequency = 0;
m_heap = nullptr;
m_gcInProgress = false;
m_suspensionPending = false;
m_waitForGCEvent = nullptr;
}
private:
int64_t m_perfCounterFrequency;
SatoriHeap* m_heap;

11
src/coreclr/src/gc/satori/SatoriObject.cpp
View file

@ -82,19 +82,14 @@ SatoriObject* SatoriObject::FormatAsFreeAfterHuge(size_t location, size_t size)
return obj;
}
void SatoriObject::SetCardsForContent()
void SatoriObject::DirtyCardsForContent()
{
_ASSERTE(IsMarked());
MethodTable* mt = RawGetMethodTable();
if (mt->ContainsPointers())
{
SatoriPage* page = ContainingRegion()->m_containingPage;
// TODO: VS SetCardsForRange
for (size_t i = Start(); i < End(); i++)
{
page->SetCardForAddress(i);
}
page->DirtyCardsForRange(Start(), End());
}
if (mt->Collectible())
@ -103,7 +98,7 @@ void SatoriObject::SetCardsForContent()
if (!o->IsMarked())
{
o->SetMarked();
o->SetCardsForContent();
o->DirtyCardsForContent();
}
}
}

2
src/coreclr/src/gc/satori/SatoriObject.h
View file

@ -44,7 +44,7 @@ public:
bool IsEscapedOrPinned();
int MarkBitOffset(size_t* bitmapIndex);
void SetCardsForContent();
void DirtyCardsForContent();
void EscapeCheck();

93
src/coreclr/src/gc/satori/SatoriPage.cpp
View file

@ -25,13 +25,14 @@ SatoriPage* SatoriPage::InitializeAt(size_t address, size_t pageSize, SatoriHeap
return result;
}
size_t cardTableBytes = pageSize / Satori::BYTES_PER_CARD_BYTE;
size_t cardTableSize = pageSize / Satori::BYTES_PER_CARD_BYTE;
// TODO: VS we do not need to commit the whole card table, we can wait until regions are used
// and track commit via card groups.
// TODO: VS if commit granularity is OS pagem do not commit the whole card table, we can wait until regions are committed
// and track card table commit via groups.
//
// commit size is the same as header size. We could commit more in the future.
// or commit on demand as regions are committed.
size_t commitSize = ALIGN_UP(cardTableBytes, Satori::CommitGranularity());
size_t commitSize = ALIGN_UP(cardTableSize, Satori::CommitGranularity());
if (!GCToOSInterface::VirtualCommit((void*)address, commitSize))
{
GCToOSInterface::VirtualRelease((void*)address, pageSize);
@ -39,16 +40,16 @@ SatoriPage* SatoriPage::InitializeAt(size_t address, size_t pageSize, SatoriHeap
}
result->m_end = address + pageSize;
result->m_firstRegion = address + ALIGN_UP(cardTableBytes, Satori::REGION_SIZE_GRANULARITY);
result->m_firstRegion = address + ALIGN_UP(cardTableSize, Satori::REGION_SIZE_GRANULARITY);
result->m_initialCommit = address + commitSize;
result->m_cardTableSize = (int)cardTableBytes / sizeof(size_t);
result->m_cardTableSize = cardTableSize;
// conservatively assume the first useful card word to cover the start of the first region.
size_t cardTableStart = (result->m_firstRegion - address) / Satori::BYTES_PER_CARD_WORD;
size_t cardTableStart = (result->m_firstRegion - address) / Satori::BYTES_PER_CARD_BYTE;
size_t regionMapSize = pageSize >> Satori::REGION_BITS;
result->m_cardTableStart = (int)cardTableStart;
result->m_cardTableStart = cardTableStart;
result->m_heap = heap;
// make sure offset of m_cardsStatus is 128.
@ -56,7 +57,7 @@ SatoriPage* SatoriPage::InitializeAt(size_t address, size_t pageSize, SatoriHeap
result->m_regionMap = (uint8_t*)(address + 128 + (pageSize >> Satori::REGION_BITS));
// make sure the first useful card word is beyond the header.
_ASSERTE(result->Start() + cardTableStart * sizeof(size_t) > (size_t)(result->m_regionMap) + regionMapSize);
_ASSERTE(result->Start() + cardTableStart > (size_t)(result->m_regionMap) + regionMapSize);
return result;
}
@ -105,3 +106,77 @@ SatoriRegion* SatoriPage::RegionForAddress(size_t address)
return (SatoriRegion*)((mapIndex << Satori::REGION_BITS) + Start());
}
SatoriRegion* SatoriPage::RegionForCardGroup(size_t group)
{
size_t mapIndex = group;
while (RegionMap()[mapIndex] > 1)
{
mapIndex -= ((size_t)1 << (RegionMap()[mapIndex] - 2));
}
return (SatoriRegion*)((mapIndex << Satori::REGION_BITS) + Start());
}
void SatoriPage::DirtyCardForAddress(size_t address)
{
size_t offset = address - Start();
size_t cardByteOffset = offset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(cardByteOffset >= m_cardTableStart);
_ASSERTE(cardByteOffset < m_cardTableSize);
m_cardTable[cardByteOffset] = Satori::CARD_DIRTY;
size_t cardGroupOffset = offset / Satori::REGION_SIZE_GRANULARITY;
this->m_cardGroups[cardGroupOffset] = Satori::CARD_DIRTY;
this->m_cardState = Satori::CARD_DIRTY;
}
void SatoriPage::DirtyCardsForRange(size_t start, size_t end)
{
size_t firstByteOffset = start - Start();
size_t lastByteOffset = end - Start() - 1;
size_t firstCard = firstByteOffset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(firstCard >= m_cardTableStart);
_ASSERTE(firstCard < m_cardTableSize);
size_t lastCard = lastByteOffset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(lastCard >= m_cardTableStart);
_ASSERTE(lastCard < m_cardTableSize);
for (size_t i = firstCard; i <= lastCard; i++)
{
m_cardTable[i] = Satori::CARD_DIRTY;
}
size_t firstGroup = firstByteOffset / Satori::REGION_SIZE_GRANULARITY;
size_t lastGroup = lastByteOffset / Satori::REGION_SIZE_GRANULARITY;
for (size_t i = firstGroup; i <= lastGroup; i++)
{
this->m_cardGroups[i] = Satori::CARD_DIRTY;
}
this->m_cardState = Satori::CARD_DIRTY;
}
void SatoriPage::WipeCardsForRange(size_t start, size_t end)
{
size_t firstByteOffset = start - Start();
size_t lastByteOffset = end - Start() - 1;
size_t firstCard = firstByteOffset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(firstCard >= m_cardTableStart);
_ASSERTE(firstCard < m_cardTableSize);
size_t lastCard = lastByteOffset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(lastCard >= m_cardTableStart);
_ASSERTE(lastCard < m_cardTableSize);
memset((void*)(m_cardTable + firstCard), 0, lastCard - firstCard + 1);
size_t firstGroup = firstByteOffset / Satori::REGION_SIZE_GRANULARITY;
size_t lastGroup = lastByteOffset / Satori::REGION_SIZE_GRANULARITY;
memset((void*)(m_cardGroups + firstGroup), 0, lastGroup - firstGroup + 1);
}

49
src/coreclr/src/gc/satori/SatoriPage.h
View file

@ -29,28 +29,35 @@ public:
SatoriRegion* RegionForAddress(size_t address);
SatoriRegion* RegionForCardGroup(size_t group);
size_t Start();
size_t End();
size_t RegionsStart();
uint8_t* RegionMap();
SatoriHeap* Heap();
void SetCardForAddress(size_t address);
void DirtyCardForAddress(size_t address);
bool IsClean()
void DirtyCardsForRange(size_t start, size_t length);
void WipeCardsForRange(size_t start, size_t end);
int8_t CardState()
{
return VolatileLoadWithoutBarrier(&m_cardState) == 0;
// TODO: VS should this be VolatileLoad when we have concurrency?
return m_cardState;
}
void SetProcessing()
{
VolatileStoreWithoutBarrier(&m_cardState, 2);
// TODO: VS should this be VolatileStore when we have concurrency? (same for the groups)
m_cardState = Satori::CARD_PROCESSING;
}
bool TrySetClean()
{
_ASSERTE((m_cardState >= 0) && (m_cardState <= 2));
return Interlocked::CompareExchange(&m_cardState, 0, 2) != 1;
return Interlocked::CompareExchange(&m_cardState, Satori::CARD_HAS_REFERENCES, Satori::CARD_PROCESSING) != Satori::CARD_DIRTY;
}
size_t CardGroupCount()
@ -58,27 +65,17 @@ public:
return (End() - Start()) >> Satori::REGION_BITS;
}
Satori::CardGroupState CardGroupState(size_t i)
int8_t& CardGroup(size_t i)
{
return (Satori::CardGroupState)m_cardGroups[i];
return m_cardGroups[i];
}
void CardGroupSetProcessing(size_t i)
int8_t* CardsForGroup(size_t i)
{
m_cardGroups[i] = Satori::CardGroupState::processing;
return &m_cardTable[i * Satori::CARD_BYTES_IN_CARD_GROUP];
}
void CardGroupTrySetClean(size_t i)
{
Interlocked::CompareExchange<uint8_t>(&m_cardGroups[i], Satori::CardGroupState::clean, Satori::CardGroupState::processing);
}
size_t* CardsForGroup(size_t i)
{
return &m_cardTable[i * Satori::CARD_WORDS_IN_CARD_GROUP];
}
size_t LocationForCard(void* cardPtr)
size_t LocationForCard(int8_t* cardPtr)
{
return Start() + ((size_t)cardPtr - Start()) * Satori::BYTES_PER_CARD_BYTE;
}
@ -93,12 +90,12 @@ private:
// 2Mb - 1Gb (region granule can store cards for 1Gb page)
// We can start card table at the beginning of the page for simplicity
// The first 4K cover the card itself, so that space will be unused and we can use it for other metadata.
size_t m_cardTable[1];
int8_t m_cardTable[1];
// header (can be up to 2Kb for 1Gb page)
struct
{
int32_t m_cardState;
int8_t m_cardState;
size_t m_end;
size_t m_initialCommit;
size_t m_firstRegion;
@ -107,8 +104,8 @@ private:
// the following is useful when scanning/clearing cards
// it can be computed from the page size, but we have space, so we will store.
int m_cardTableSize;
int m_cardTableStart;
size_t m_cardTableSize;
size_t m_cardTableStart;
// ----- we can have a few more fields above as long as m_cardsStatus starts at offset 128.
// that can be adjusted if needed
@ -122,7 +119,7 @@ private:
// 1byte per region
// 512 bytes per 1Gb
DECLSPEC_ALIGN(128)
uint8_t m_cardGroups[1];
int8_t m_cardGroups[1];
};
};
};

16
src/coreclr/src/gc/satori/SatoriPage.inl
View file

@ -36,20 +36,4 @@ inline SatoriHeap* SatoriPage::Heap()
return m_heap;
}
inline void SatoriPage::SetCardForAddress(size_t address)
{
size_t offset = address - Start();
size_t cardByteOffset = offset / Satori::BYTES_PER_CARD_BYTE;
_ASSERTE(cardByteOffset / 8 > m_cardTableStart);
_ASSERTE(cardByteOffset / 8 < m_cardTableSize);
((uint8_t*)this)[cardByteOffset] = 0xFF;
size_t cardGroupOffset = offset / Satori::REGION_SIZE_GRANULARITY;
this->m_cardGroups[cardGroupOffset] = Satori::CardGroupState::dirty;
this->m_cardState = 1;
}
#endif

279
src/coreclr/src/gc/satori/SatoriRecycler.cpp
View file

@ -38,8 +38,39 @@ void SatoriRecycler::Initialize(SatoriHeap* heap)
m_workList = new SatoriMarkChunkQueue();
m_gcInProgress = 0;
m_gen1Count = m_gen2Count = 0;
m_condemnedGeneration = 0;
}
// TODO: VS interlocked?
void SatoriRecycler::IncrementScanCount()
{
m_scanCount++;
}
// TODO: VS volatile?
int SatoriRecycler::GetScanCount()
{
return m_scanCount;
}
int64_t SatoriRecycler::GetCollectionCount(int gen)
{
switch (gen)
{
case 0:
case 1:
return m_gen1Count;
default:
return m_gen2Count;
}
}
int SatoriRecycler::CondemnedGeneration()
{
return m_condemnedGeneration;
}
void SatoriRecycler::AddRegion(SatoriRegion* region)
{
_ASSERTE(region->AllocStart() == 0);
@ -47,9 +78,9 @@ void SatoriRecycler::AddRegion(SatoriRegion* region)
region->Verify();
// TODO: VS volatile?
region->Publish();
region->ResetOwningThread();
region->CleanMarks();
region->SetGeneration(1);
if (region->HasFinalizables())
{
@ -75,13 +106,22 @@ void SatoriRecycler::MaybeTriggerGC()
}
else if (Interlocked::CompareExchange(&m_gcInProgress, 1, 0) == 0)
{
Collect(/*force*/ false);
// for now just do 1 , 2, 1, 2, ...
int generation = m_scanCount % 2 == 0 ? 1 : 2;
Collect(generation, /*force*/ false);
}
}
}
// TODO: VS gen1
// clean all cards after gen2
// Volatile for dirty (or put a comment)
// pass generation to EE helpers
// bariers
// do not mark or trace into gen2 regions when in gen1
// GcPromotionsGranted
void SatoriRecycler::Collect(bool force)
void SatoriRecycler::Collect(int generation, bool force)
{
bool wasCoop = GCToEEInterface::EnablePreemptiveGC();
_ASSERTE(wasCoop);
@ -95,28 +135,27 @@ void SatoriRecycler::Collect(bool force)
int count = m_finalizationTrackingRegions->Count() + m_regularRegions->Count();
if (count - m_prevRegionCount > 10 || force)
{
m_condemnedGeneration = generation;
// deactivate all stacks
DeactivateAllStacks();
// mark own stack into work queues
IncrementScanCount();
MarkOwnStack();
// TODO: VS perhaps drain queues as a part of MarkOwnStack? - to give other threads chance to self-mark?
// thread marking is fast though, so it may not help a lot.
MarkOtherStacks();
// drain queues
DrainMarkQueues();
// mark handles
MarkHandles();
while (m_workList->Count() > 0)
// mark through all cards that has interesting refs (remembered set).
bool revisitCards = generation == 1 ?
MarkThroughCards(/* minState */ Satori::CARD_HAS_REFERENCES) :
false;
while (m_workList->Count() > 0 || revisitCards)
{
DrainMarkQueues();
revisitCards = MarkThroughCards(/* minState */ Satori::CARD_DIRTY);
}
// all strongly reachable objects are marked here
@ -125,7 +164,12 @@ void SatoriRecycler::Collect(bool force)
DependentHandlesInitialScan();
while (m_workList->Count() > 0)
{
DrainMarkQueues();
do
{
DrainMarkQueues();
revisitCards = MarkThroughCards(/* minState */ Satori::CARD_DIRTY);
} while (m_workList->Count() > 0 || revisitCards);
DependentHandlesRescan();
}
@ -138,7 +182,12 @@ void SatoriRecycler::Collect(bool force)
// TODO: VS why no sync before?
while (m_workList->Count() > 0)
{
DrainMarkQueues();
do
{
DrainMarkQueues();
revisitCards = MarkThroughCards(/* minState */ Satori::CARD_DIRTY);
} while (m_workList->Count() > 0 || revisitCards);
DependentHandlesRescan();
}
@ -161,7 +210,7 @@ void SatoriRecycler::Collect(bool force)
// once no more regs in queue
// go through sources and relocate to destinations,
// grab empties if no space, add to stayers and use as if gotten from free buckets.
// if no space at all, put the reg to stayers.
// if no space at all, put the region to stayers.
// go through roots and update refs
@ -171,35 +220,58 @@ void SatoriRecycler::Collect(bool force)
// TODO: VS do trivial sweep for now
auto SweepRegions = [&](SatoriRegionQueue* regions)
{
SatoriRegion* curReg;
while (curReg = regions->TryPop())
SatoriRegion* curRegion;
while (curRegion = regions->TryPop())
{
if (curReg->NothingMarked())
// we must sweep in gen2, since unloadable types may invalidate method tables and make
// unreachable objects unwalkable.
// we do not sweep gen1 though. without compaction there is no benefit, just forcing index rebuilding.
bool nothingMarked = generation == 2 ?
curRegion->Sweep() :
curRegion->NothingMarked();
if (curRegion->Generation() <= generation && nothingMarked)
{
curReg->MakeBlank();
m_heap->Allocator()->AddRegion(curReg);
// TODO: VS wipe cards should be a part of return and MakeBlank too, but make it minimal
curRegion->WipeCards();
curRegion->MakeBlank();
m_heap->Allocator()->AddRegion(curRegion);
}
else
{
m_stayingRegions->Push(curReg);
if (generation == 2)
{
// everything is Gen2 now.
curRegion->SetGeneration(2);
curRegion->WipeCards();
}
m_stayingRegions->Push(curRegion);
}
}
while (curReg = m_stayingRegions->TryPop())
while (curRegion = m_stayingRegions->TryPop())
{
curReg->CleanMarks();
regions->Push(curReg);
curRegion->CleanMarks();
regions->Push(curRegion);
}
};
SweepRegions(m_regularRegions);
SweepRegions(m_finalizationTrackingRegions);
m_prevRegionCount = m_finalizationTrackingRegions->Count() + m_regularRegions->Count();
m_gen1Count++;
if (generation == 2)
{
m_gen2Count++;
}
}
m_condemnedGeneration = 0;
// restart VM
GCToEEInterface::RestartEE(true);
m_gcInProgress = false;
}
@ -210,7 +282,7 @@ void SatoriRecycler::AssertNoWork()
m_heap->ForEachPage(
[&](SatoriPage* page)
{
_ASSERTE(page->IsClean());
_ASSERTE(page->CardState() < Satori::CARD_PROCESSING);
}
);
}
@ -260,9 +332,12 @@ void SatoriRecycler::PushToMarkQueuesSlow(SatoriMarkChunk* &currentMarkChunk, Sa
m_workList->Push(currentMarkChunk);
}
//TODO: VS HACK HACK HACK, this is just to force overflows.(do this always in debug?)
#ifdef _DEBUG
// Limit worklist to one item in debug/chk.
// This is just to force more overflows. Otherwise they are rather rare.
currentMarkChunk = nullptr;
if (m_workList->Count() == 0)
#endif
{
currentMarkChunk = m_heap->Allocator()->TryGetMarkChunk();
}
@ -273,7 +348,7 @@ void SatoriRecycler::PushToMarkQueuesSlow(SatoriMarkChunk* &currentMarkChunk, Sa
}
else
{
o->SetCardsForContent();
o->DirtyCardsForContent();
}
}
@ -385,18 +460,6 @@ void SatoriRecycler::MarkOtherStacks()
}
}
// TODO: VS interlocked?
void SatoriRecycler::IncrementScanCount()
{
m_scanCount++;
}
// TODO: VS volatile?
int SatoriRecycler::GetScanCount()
{
return m_scanCount;
}
void SatoriRecycler::DrainMarkQueues()
{
SatoriMarkChunk* srcChunk = m_workList->TryPop();
@ -447,91 +510,97 @@ void SatoriRecycler::DrainMarkQueues()
_ASSERTE(dstChunk->Count() == 0);
m_heap->Allocator()->ReturnMarkChunk(dstChunk);
}
CleanCards();
}
void SatoriRecycler::CleanCards()
bool SatoriRecycler::MarkThroughCards(int8_t minState)
{
SatoriMarkChunk* dstChunk = nullptr;
bool revisit = false;
do
{
m_heap->ForEachPage(
[&](SatoriPage* page)
m_heap->ForEachPage(
[&](SatoriPage* page)
{
if (page->CardState() >= minState)
{
if (!page->IsClean())
page->SetProcessing();
size_t groupCount = page->CardGroupCount();
for (size_t i = 0; i < groupCount; i++)
{
page->SetProcessing();
size_t groupCount = page->CardGroupCount();
for (size_t i = 0; i < groupCount; i++)
// TODO: VS when stealing is implemented we should start from a random location
if (page->CardGroup(i) >= minState)
{
// TODO: VS when stealing is implemented we should start from a random location
if (page->CardGroupState(i) == Satori::CardGroupState::dirty)
int8_t* cards = page->CardsForGroup(i);
SatoriRegion* region = page->RegionForCardGroup(i);
int8_t resetValue = region->Generation() == 2 ? Satori::CARD_HAS_REFERENCES : Satori::CARD_BLANK;
//TODO: VS enable when truly generational.
bool considerAllMarked = false; // region->Generation() > m_condemnedGeneration;
page->CardGroup(i) = resetValue;
for (int j = 0; j < Satori::CARD_BYTES_IN_CARD_GROUP; j++)
{
page->CardGroupSetProcessing(i);
size_t* cards = page->CardsForGroup(i);
// TODO: VS page->RegionForCardGroup
SatoriRegion* region = page->RegionForAddress(page->LocationForCard(cards));
for (int j = 0; j < Satori::CARD_WORDS_IN_CARD_GROUP; j++)
//TODO: VS size_t steps, maybe, at least when skipping?
if (cards[j] < minState)
{
// TODO: VS do one word at a time for now, scan for contiguous ranges later.
size_t card = cards[j];
if (!card)
{
continue;
}
cards[j] = 0;
size_t start = page->LocationForCard(&cards[j]);
size_t end = start + Satori::BYTES_PER_CARD_WORD;
SatoriObject* o = region->FindObject(start);
do
{
if (o->IsMarked())
{
o->ForEachObjectRef(
[&](SatoriObject** ref)
{
// TODO: VS check that the ref is dirty in the card.
SatoriObject* child = *ref;
if (child && !child->IsMarked())
{
child->SetMarked();
child->Validate();
if (!dstChunk || !dstChunk->TryPush(child))
{
this->PushToMarkQueuesSlow(dstChunk, child);
}
}
},
start,
end
);
}
o = o->Next();
} while (o->Start() < end);
continue;
}
page->CardGroupTrySetClean(i);
size_t start = page->LocationForCard(&cards[j]);
do
{
cards[j++] = resetValue;
} while (j < Satori::CARD_BYTES_IN_CARD_GROUP && cards[j] >= minState);
size_t end = page->LocationForCard(&cards[j]);
SatoriObject* o = region->FindObject(start);
do
{
// we trace only through marked objects.
// things to consider:
// 1) tracing into dead objects is dangerous. marked objects will not have that.
// 2) tracing from unmarked retains too much, in particular blocking gen2 must be precise.
// 3) overflow will mark before dirtying, so always ok.
// 4) for concurrent dirtying - asignment happening before marking will be traced (careful with HW order!!)
// 5) gen2 objects are all considered marked in partial GC, but full GC is precise.
// 6) gen2 should not have dead objects and must sweep (the other reason is unloadable types)
//
if (considerAllMarked || o->IsMarked())
{
o->ForEachObjectRef(
[&](SatoriObject** ref)
{
SatoriObject* child = *ref;
if (child && !child->IsMarked())
{
child->SetMarked();
child->Validate();
if (!dstChunk || !dstChunk->TryPush(child))
{
this->PushToMarkQueuesSlow(dstChunk, child);
}
}
}, start, end);
}
o = o->Next();
} while (o->Start() < end);
}
}
// record missed clean to revisit the whole deal.
revisit = !page->TrySetClean();
}
// record a missed clean to revisit the whole deal.
revisit = !page->TrySetClean();
}
);
} while (revisit);
}
);
if (dstChunk)
{
m_workList->Push(dstChunk);
}
return revisit;
}
void SatoriRecycler::MarkHandles()

13
src/coreclr/src/gc/satori/SatoriRecycler.h
View file

@ -27,20 +27,23 @@ public:
void MaybeTriggerGC();
int GetScanCount();
int64_t GetCollectionCount(int gen);
int CondemnedGeneration();
void Collect(bool force);
void Collect(int generation, bool force);
void AssertNoWork();
private:
SatoriHeap* m_heap;
// this is to ensure that only one thread suspends VM and to block all others on it.
SatoriLock m_suspensionLock;
// used to ensure each thread is scanned once per scan round.
int m_scanCount;
int64_t m_gen1Count;
int64_t m_gen2Count;
int m_condemnedGeneration;
// region count at the end of last GC, used in a crude GC triggering heuristic.
int m_prevRegionCount;
int m_gcInProgress;
@ -67,7 +70,7 @@ private:
void MarkOtherStacks();
void IncrementScanCount();
void DrainMarkQueues();
void CleanCards();
bool MarkThroughCards(int8_t minState);
void MarkHandles();
void WeakPtrScan(bool isShort);
void WeakPtrScanBySingleThread();

59
src/coreclr/src/gc/satori/SatoriRegion.cpp
View file

@ -80,6 +80,11 @@ SatoriAllocator* SatoriRegion::Allocator()
return m_containingPage->Heap()->Allocator();
}
void SatoriRegion::WipeCards()
{
m_containingPage->WipeCardsForRange(Start(), End());
}
void SatoriRegion::MakeBlank()
{
m_ownerThreadTag = 0;
@ -191,7 +196,7 @@ void SatoriRegion::SplitCore(size_t regionSize, size_t& nextStart, size_t& nextC
{
_ASSERTE(regionSize % Satori::REGION_SIZE_GRANULARITY == 0);
_ASSERTE(regionSize < this->Size());
_ASSERTE(m_allocEnd == m_end);
_ASSERTE(m_allocEnd == 0 || m_allocEnd == m_end);
_ASSERTE((size_t)m_allocStart < m_end - regionSize - Satori::MIN_FREE_SIZE);
size_t newEnd = m_end - regionSize;
@ -202,10 +207,9 @@ void SatoriRegion::SplitCore(size_t regionSize, size_t& nextStart, size_t& nextC
m_end = newEnd;
m_committed = min(newEnd, m_committed);
m_used = min(newEnd, m_used);
m_allocEnd = newEnd;
m_allocEnd = min(m_allocEnd, newEnd);
_ASSERTE(Size() >= Satori::REGION_SIZE_GRANULARITY);
_ASSERTE(IsAllocating());
}
SatoriRegion* SatoriRegion::Split(size_t regionSize)
@ -977,6 +981,55 @@ SatoriObject* SatoriRegion::SkipUnmarked(SatoriObject* from, size_t upTo)
return result;
}
bool SatoriRegion::Sweep()
{
size_t limit = Start() + Satori::REGION_SIZE_GRANULARITY;
if (End() > limit)
{
SatoriObject* last = FindObject(limit - 1);
if (!last->IsMarked())
{
SatoriObject::FormatAsFree(last->Start(), limit - last->Start());
SatoriRegion* other = this->Split(Size() - Satori::REGION_SIZE_GRANULARITY);
other->WipeCards();
other->MakeBlank();
Allocator()->ReturnRegion(other);
}
}
bool sawMarked = false;
SatoriObject* cur = FirstObject();
while(true)
{
if (cur->Start() >= limit)
{
break;
}
if (cur->IsMarked())
{
sawMarked = true;
cur = cur->Next();
continue;
}
size_t lastMarkedEnd = cur->Start();
cur = SkipUnmarked(cur);
size_t skipped = cur->Start() - lastMarkedEnd;
if (skipped)
{
SatoriObject::FormatAsFree(lastMarkedEnd, skipped);
}
}
// clean index
memset(&m_index, 0, sizeof(m_index));
return !sawMarked;
}
bool SatoriRegion::NothingMarked()
{
for (size_t bitmapIndex = BITMAP_START; bitmapIndex < BITMAP_LENGTH; bitmapIndex++)

11
src/coreclr/src/gc/satori/SatoriRegion.h
View file

@ -35,6 +35,7 @@ public:
static SatoriRegion* InitializeAt(SatoriPage* containingPage, size_t address, size_t regionSize, size_t committed, size_t used);
void MakeBlank();
bool ValidateBlank();
void WipeCards();
SatoriRegion* Split(size_t regionSize);
bool CanCoalesce(SatoriRegion* other);
@ -49,12 +50,13 @@ public:
void StopAllocating(size_t allocPtr);
bool IsAllocating();
void Publish();
void ResetOwningThread();
bool IsThreadLocal();
bool OwnedByCurrentThread();
size_t Generation();
void SetGeneration(size_t generation);
int Generation();
void SetGeneration(int generation);
size_t Start();
size_t End();
@ -69,6 +71,7 @@ public:
void ThreadLocalUpdatePointers();
SatoriObject* SkipUnmarked(SatoriObject* from);
SatoriObject* SkipUnmarked(SatoriObject* from, size_t upTo);
bool Sweep();
bool NothingMarked();
bool ThreadLocalCompact(size_t desiredFreeSpace);
@ -108,7 +111,7 @@ private:
// TEB address could be used on Windows, for example
size_t m_ownerThreadTag;
void (*m_escapeFunc)(SatoriObject**, SatoriObject*, SatoriRegion*);
size_t m_generation;
int m_generation;
size_t m_end;
size_t m_committed;

8
src/coreclr/src/gc/satori/SatoriRegion.inl
View file

@ -26,12 +26,12 @@ inline bool SatoriRegion::OwnedByCurrentThread()
return m_ownerThreadTag == SatoriUtil::GetCurrentThreadTag();
}
inline size_t SatoriRegion::Generation()
inline int SatoriRegion::Generation()
{
return m_generation;
}
inline void SatoriRegion::SetGeneration(size_t generation)
inline void SatoriRegion::SetGeneration(int generation)
{
m_generation = generation;
}
@ -78,11 +78,9 @@ inline size_t SatoriRegion::Occupancy()
return m_occupancy;
}
//TODO: VS why do we have this?
inline void SatoriRegion::Publish()
inline void SatoriRegion::ResetOwningThread()
{
m_ownerThreadTag = 0;
m_generation = 1;
}
template<typename F>

16
src/coreclr/src/gc/satori/SatoriUtil.h
View file

@ -63,16 +63,14 @@ namespace Satori
}
static const int BYTES_PER_CARD_BYTE = 512;
static const int BYTES_PER_CARD_WORD = BYTES_PER_CARD_BYTE * sizeof(size_t);
static const int CARD_WORDS_IN_CARD_GROUP = Satori::REGION_SIZE_GRANULARITY / BYTES_PER_CARD_WORD;
static const int CARD_BYTES_IN_CARD_GROUP = Satori::REGION_SIZE_GRANULARITY / BYTES_PER_CARD_BYTE;
enum CardGroupState : uint8_t
{
uncommitted = 0xFF,
clean = 0,
dirty = 1,
processing = 2
};
static const int8_t CARD_UNCOMMITTED = (int8_t)-1;
static const int8_t CARD_BLANK = 0;
static const int8_t CARD_HAS_REFERENCES = 1;
static const int8_t CARD_PROCESSING = 2;
// TODO: VS setting and reading dirty should be Volatile Store/Load, matters only on ARM though
static const int8_t CARD_DIRTY = 3;
}
class SatoriUtil