Memory Diagnostics: GC Bookkeeping (#84454)

* Add ISOSMemoryEnum and HandleTable enum

* Update comment

* Add GC Bookkeeping data to the dac

- bookkeeping_covered_start is now compiled into all versions of the GC, instead of just with USE_REGIONS.  This allows us to find the base address of the allocated memory for GC Bookkeeping.
- Added dac enumeration of GC Bookkeeping.

* Add assert

* Add support for free gc regions

* Don't require card_table_element_layout

* Fix issue with naming in request

* Remote DEFINE_MISSING_FIELD from dac

We should never have missing fields in the dac.  This is a leftover from
previous code.

* Whitespace fix

* Break if we loop

* Fix static_assert

* Segment fixes

When using segments, update bookkeeping_covered_start and card_table_info.size when we update the card table.

* Defensive coding

Check how many times we loop through memory to guard against heap corruption.

* free_region fixes

* Remove hardcoding of region list

* Fix gcc warnings

* Remove unneeded #define

* Add freeable_soh_segment/freeable_uoh_segment

* Add more card table checks

* Fix field definition issue

* Fix pointer issue

* Fix compile issue

* Rename bookkeeping_covered_start to bookkeeping_start

* Update src/coreclr/debug/daccess/daccess.cpp

Co-authored-by: Andrew Au <cshung@gmail.com>

* Code review feedback

* Remove unused parameter

* Add heap number

---------

Co-authored-by: Andrew Au <andrewau@microsoft.com>
Co-authored-by: Andrew Au <cshung@gmail.com>

src/coreclr/debug/daccess/daccess.cpp

@@ -7654,7 +7654,6 @@ void DacHandleWalker::WalkHandles()
                         {
                             if (mask & 1)
                             {
-                                dac_handle_table *pTable = hTable;
                                 PTR_AppDomain pDomain = AppDomain::GetCurrentDomain();
                                 param.AppDomain = TO_CDADDR(pDomain.GetAddr());
                                 param.Type = handleType;
@@ -8285,3 +8284,234 @@ HRESULT DacStackReferenceErrorEnum::Next(unsigned int count, SOSStackRefError re
     *pFetched = i;
     return i < count ? S_FALSE : S_OK;
 }
+
+
+HRESULT DacMemoryEnumerator::Skip(unsigned int count)
+{
+    mIteratorIndex += count;
+    return S_OK;
+}
+
+HRESULT DacMemoryEnumerator::Reset()
+{
+    mIteratorIndex = 0;
+    return S_OK;
+}
+
+HRESULT DacMemoryEnumerator::GetCount(unsigned int* pCount)
+{
+    if (!pCount)
+        return E_POINTER;
+
+    mRegions.GetCount();
+    return S_OK;
+}
+
+HRESULT DacMemoryEnumerator::Next(unsigned int count, SOSMemoryRegion regions[], unsigned int* pFetched)
+{
+    if (!pFetched)
+        return E_POINTER;
+
+    if (!regions)
+        return E_POINTER;
+
+    unsigned int i = 0;
+    while (i < count && mIteratorIndex < mRegions.GetCount())
+    {
+        regions[i++] = mRegions.Get(mIteratorIndex++);
+    }
+
+    *pFetched = i;
+    return i < count ? S_FALSE : S_OK;
+}
+
+
+HRESULT DacGCBookkeepingEnumerator::Init()
+{
+    if (g_gcDacGlobals->bookkeeping_start == nullptr)
+        return E_FAIL;
+
+    TADDR ctiAddr = TO_TADDR(*g_gcDacGlobals->bookkeeping_start);
+    if (ctiAddr == 0)
+        return E_FAIL;
+
+    DPTR(dac_card_table_info) card_table_info(ctiAddr);
+
+    SOSMemoryRegion mem = {0};
+    if (card_table_info->recount && card_table_info->size)
+    {
+        mem.Start = card_table_info.GetAddr();
+        mem.Size = card_table_info->size;
+        mRegions.Add(mem);
+    }
+
+    size_t card_table_info_size = g_gcDacGlobals->card_table_info_size;
+    TADDR next = card_table_info->next_card_table;
+
+    // Cap the number of regions we will walk in case we have run into some kind of
+    // memory corruption.  We shouldn't have more than a few linked card tables anyway.
+    int maxRegions = 32;
+
+    // This loop is effectively "while (next != 0)" but with an added check to make
+    // sure we don't underflow next when subtracting card_table_info_size if we encounter
+    // a bad pointer.
+    while (next > card_table_info_size)
+    {
+        DPTR(dac_card_table_info) ct(next - card_table_info_size);
+
+        if (ct->recount && ct->size)
+        {
+            mem = {0};
+            mem.Start = ct.GetAddr();
+            mem.Size = ct->size;
+            mRegions.Add(mem);
+        }
+
+        next = ct->next_card_table;
+        if (next == card_table_info->next_card_table)
+            break;
+
+        if (--maxRegions <= 0)
+            break;
+    }
+
+    return S_OK;
+}
+
+
+HRESULT DacHandleTableMemoryEnumerator::Init()
+{
+    int max_slots = 1;
+
+#ifdef FEATURE_SVR_GC
+    if (GCHeapUtilities::IsServerHeap())
+        max_slots = GCHeapCount();
+#endif // FEATURE_SVR_GC
+
+    // Cap the number of regions we will walk in case we hit an infinite loop due
+    // to memory corruption
+    int maxRegions = 8192;
+
+    for (dac_handle_table_map *map = g_gcDacGlobals->handle_table_map; map && maxRegions >= 0; map = map->pNext, maxRegions--)
+    {
+        for (int i = 0; i < INITIAL_HANDLE_TABLE_ARRAY_SIZE; ++i)
+        {
+            if (map->pBuckets[i] != NULL)
+            {
+                for (int j = 0; j < max_slots ; ++j)
+                {
+                    DPTR(dac_handle_table) pTable = map->pBuckets[i]->pTable[j];
+                    DPTR(dac_handle_table_segment) pFirstSegment = pTable->pSegmentList;
+                    DPTR(dac_handle_table_segment) curr = pFirstSegment;
+
+                    do
+                    {                        
+                        SOSMemoryRegion mem = {0};
+                        mem.Start = curr.GetAddr();
+                        mem.Size = HANDLE_SEGMENT_SIZE;
+                        mem.Heap = j; // heap number
+
+                        mRegions.Add(mem);
+
+                        curr = curr->pNextSegment;
+                    } while (curr != nullptr && curr != pFirstSegment);
+                }
+            }
+        }
+    }
+
+    return S_OK;
+}
+
+void DacFreeRegionEnumerator::AddSingleSegment(const dac_heap_segment &curr, FreeRegionKind kind, int heap)
+{
+    SOSMemoryRegion mem = {0};
+    mem.Start = TO_CDADDR(curr.mem);
+    mem.ExtraData = (CLRDATA_ADDRESS)kind;
+    mem.Heap = heap;
+
+    if (curr.mem < curr.committed)
+        mem.Size = TO_CDADDR(curr.committed) - mem.Start;
+
+    if (mem.Start)
+        mRegions.Add(mem);
+}
+
+void DacFreeRegionEnumerator::AddSegmentList(DPTR(dac_heap_segment) start, FreeRegionKind kind, int heap)
+{
+    int iterationMax = 2048;
+
+    DPTR(dac_heap_segment) curr = start;
+    while (curr != nullptr)
+    {
+        AddSingleSegment(*curr, kind, heap);
+
+        curr = curr->next;
+        if (curr == start)
+            break;
+
+        if (iterationMax-- <= 0)
+            break;
+    }
+}
+
+void DacFreeRegionEnumerator::AddFreeList(DPTR(dac_region_free_list) free_list, FreeRegionKind kind)
+{
+    if (free_list != nullptr)
+    {
+        AddSegmentList(free_list->head_free_region, kind);
+    }
+}
+
+HRESULT DacFreeRegionEnumerator::Init()
+{
+    // Cap the number of free regions we will walk at a sensible number.  This is to protect against
+    // memory corruption, un-initialized data, or just a bug.
+    int count_free_region_kinds = g_gcDacGlobals->count_free_region_kinds;
+    count_free_region_kinds = min(count_free_region_kinds, 16);
+
+    unsigned int index = 0;
+    if (g_gcDacGlobals->global_free_huge_regions != nullptr)
+    {
+        DPTR(dac_region_free_list) global_free_huge_regions(g_gcDacGlobals->global_free_huge_regions);
+        AddFreeList(global_free_huge_regions, FreeRegionKind::FreeGlobalHugeRegion);
+    }
+
+    if (g_gcDacGlobals->global_regions_to_decommit != nullptr)
+    {
+        DPTR(dac_region_free_list) regionList(g_gcDacGlobals->global_regions_to_decommit);
+        if (regionList != nullptr)
+            for (int i = 0; i < count_free_region_kinds; i++, regionList++)
+                AddFreeList(regionList, FreeRegionKind::FreeGlobalRegion);
+    }
+
+#if defined(FEATURE_SVR_GC)
+    if (GCHeapUtilities::IsServerHeap())
+    {
+        AddServerRegions();
+    }
+    else
+#endif //FEATURE_SVR_GC
+    {
+        DPTR(dac_region_free_list) regionList(g_gcDacGlobals->free_regions);
+        if (regionList != nullptr)
+            for (int i = 0; i < count_free_region_kinds; i++, regionList++)
+                AddFreeList(regionList, FreeRegionKind::FreeRegion);
+
+        if (g_gcDacGlobals->freeable_soh_segment != nullptr)
+        {
+            DPTR(DPTR(dac_heap_segment)) freeable_soh_segment_ptr(g_gcDacGlobals->freeable_soh_segment);
+            if (freeable_soh_segment_ptr != nullptr)
+                AddSegmentList(*freeable_soh_segment_ptr, FreeRegionKind::FreeSohSegment);
+        }
+
+        if (g_gcDacGlobals->freeable_uoh_segment != nullptr)
+        {
+            DPTR(DPTR(dac_heap_segment)) freeable_uoh_segment_ptr(g_gcDacGlobals->freeable_uoh_segment);
+            if (freeable_uoh_segment_ptr != nullptr)
+                AddSegmentList(*freeable_uoh_segment_ptr, FreeRegionKind::FreeUohSegment);
+        }
+    }
+
+    return S_OK;
+}

src/coreclr/debug/daccess/dacimpl.h

@@ -1211,6 +1211,10 @@ class ClrDataAccess
     virtual HRESULT STDMETHODCALLTYPE GetDomainLoaderAllocator(CLRDATA_ADDRESS domainAddress, CLRDATA_ADDRESS *pLoaderAllocator);
     virtual HRESULT STDMETHODCALLTYPE GetLoaderAllocatorHeapNames(int count, const char **ppNames, int *pNeeded);
     virtual HRESULT STDMETHODCALLTYPE GetLoaderAllocatorHeaps(CLRDATA_ADDRESS loaderAllocator, int count, CLRDATA_ADDRESS *pLoaderHeaps, LoaderHeapKind *pKinds, int *pNeeded);
+    virtual HRESULT STDMETHODCALLTYPE GetHandleTableMemoryRegions(ISOSMemoryEnum **ppEnum);
+    virtual HRESULT STDMETHODCALLTYPE GetGCBookkeepingMemoryRegions(ISOSMemoryEnum **ppEnum);
+    virtual HRESULT STDMETHODCALLTYPE GetGCFreeRegions(ISOSMemoryEnum **ppEnum);
+    virtual HRESULT STDMETHODCALLTYPE LockedFlush();
 
     //
     // ClrDataAccess.
@@ -1954,6 +1958,56 @@ class DacReferenceList
         unsigned int _capacity;
 };
 
+
+class DacMemoryEnumerator : public DefaultCOMImpl<ISOSMemoryEnum, IID_ISOSMemoryEnum>
+{
+public:
+    DacMemoryEnumerator()
+        : mIteratorIndex(0)
+    {
+    }
+
+    virtual ~DacMemoryEnumerator() {}
+    virtual HRESULT Init() = 0;
+
+    HRESULT STDMETHODCALLTYPE Skip(unsigned int count);
+    HRESULT STDMETHODCALLTYPE Reset();
+    HRESULT STDMETHODCALLTYPE GetCount(unsigned int *pCount);
+    HRESULT STDMETHODCALLTYPE Next(unsigned int count,
+                                   SOSMemoryRegion regions[],
+                                   unsigned int *pFetched);
+
+protected:
+    DacReferenceList<SOSMemoryRegion> mRegions;
+
+private:
+    unsigned int mIteratorIndex;
+};
+
+class DacHandleTableMemoryEnumerator : public DacMemoryEnumerator
+{
+public:
+    virtual HRESULT Init();
+};
+
+class DacGCBookkeepingEnumerator : public DacMemoryEnumerator
+{
+public:
+    virtual HRESULT Init();
+};
+
+class DacFreeRegionEnumerator : public DacMemoryEnumerator
+{
+public:
+    virtual HRESULT Init();
+
+private:
+    void AddSingleSegment(const dac_heap_segment &seg, FreeRegionKind kind, int heap);
+    void AddSegmentList(DPTR(dac_heap_segment) seg, FreeRegionKind kind, int heap = 0);
+    void AddFreeList(DPTR(dac_region_free_list) freeList, FreeRegionKind kind);
+    void AddServerRegions();
+};
+
 struct DacGcReference;
  /* DacStackReferenceWalker.
  */