gh-159137: Restore known-root marking for the GIL garbage collector

Author: pablogsal

Include/internal/pycore_gc.h

@@ -118,7 +118,20 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
 /* Bit 1 is set when the object is in generation which is GCed currently. */
 #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2)
 
-#define _PyGC_PREV_SHIFT           2
+/* Use an epoch bit only where the existing header alignment permits it. */
+#if !defined(Py_GIL_DISABLED) && SIZEOF_VOID_P == 8
+#  ifdef __cplusplus
+#    define _PyGC_PREV_MASK_EPOCH \
+        ((uintptr_t)(alignof(PyGC_Head) >= 8 ? 4 : 0))
+#  else
+#    define _PyGC_PREV_MASK_EPOCH \
+        ((uintptr_t)(_Alignof(PyGC_Head) >= 8 ? 4 : 0))
+#  endif
+#else
+#  define _PyGC_PREV_MASK_EPOCH ((uintptr_t)0)
+#endif
+
+#define _PyGC_PREV_SHIFT           (2 + (_PyGC_PREV_MASK_EPOCH != 0))
 #define _PyGC_PREV_MASK            (((uintptr_t) -1) << _PyGC_PREV_SHIFT)
 
 /* set for debugging information */
@@ -151,7 +164,7 @@ static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) {
     gc->_gc_next = (uintptr_t)next;
 }
 
-// Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags.
+// Low bits of _gc_prev are used for _PyGC_PREV_MASK_* flags.
 static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) {
     uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK);
     return (PyGC_Head*)prev;
@@ -228,6 +241,10 @@ static inline void _PyObject_GC_TRACK(
     PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev);
     _PyGCHead_SET_NEXT(last, gc);
     _PyGCHead_SET_PREV(gc, last);
+    if (_PyGC_PREV_MASK_EPOCH) {
+        gc->_gc_prev = (gc->_gc_prev & ~_PyGC_PREV_MASK_EPOCH)
+            | gcstate->mark_epoch;
+    }
     _PyGCHead_SET_NEXT(gc, generation0);
     generation0->_gc_prev = (uintptr_t)gc;
     gcstate->heap_size++;

Include/internal/pycore_interp_structs.h

@@ -163,8 +163,8 @@ typedef struct {
     _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, uintptr_t) _gc_next;
 
     // Tagged pointer to previous object in the list.
-    // Lowest two bits are used for flags documented later.
-    // Those bits are made available by the struct's minimum alignment.
+    // Low bits are used for flags documented in pycore_gc.h.
+    // The available bits depend on the header's existing alignment.
     uintptr_t _gc_prev;
 } PyGC_Head;
 
@@ -244,6 +244,10 @@ struct _gc_runtime_state {
 #endif
     /* true if we are currently running the collector */
     int collecting;
+#ifndef Py_GIL_DISABLED
+    /* Epoch bit stamped on tracked objects; toggled by eligible full GCs. */
+    unsigned char mark_epoch;
+#endif
     // The frame that started the current collection. It might be NULL even when
     // collecting (if no Python frame is running):
     _PyInterpreterFrame *frame;

Include/object.h

@@ -112,7 +112,7 @@ whose size is determined when the object is allocated.
 
 /* PyObjects are given a minimum alignment so that the least significant bits
  * of an object pointer become available for other purposes.
- * This must be an integer literal with the value (1 << _PyGC_PREV_SHIFT), number of bytes.
+ * This must be an integer literal specifying the minimum number of bytes.
  */
 #define _PyObject_MIN_ALIGNMENT 4
 

InternalDocs/garbage_collector.md

@@ -661,10 +661,10 @@ of `PyGC_Head` discussed in the `Memory layout and object structure`_ section:
 - The `_gc_prev` field is normally used as the "previous" pointer to maintain the
   doubly linked list but its lowest two bits are used to keep the flags
   `PREV_MASK_COLLECTING` and `_PyGC_PREV_MASK_FINALIZED`. Between collections,
-  the only flag that can be present is `_PyGC_PREV_MASK_FINALIZED` that indicates
+  the `_PyGC_PREV_MASK_FINALIZED` flag indicates
   if an object has been already finalized. During collections `_gc_prev` is
   temporarily used for storing a copy of the reference count (`gc_ref`), in
-  addition to two flags, and the GC linked list becomes a singly linked list until
+  addition to these flags, and the GC linked list becomes a singly linked list until
   `_gc_prev` is restored.
 
 - The `_gc_next` field is used as the "next" pointer to maintain the doubly linked
@@ -676,6 +676,12 @@ of `PyGC_Head` discussed in the `Memory layout and object structure`_ section:
   currently in.  Instead, when that's needed, ad hoc tricks (like the
   `NEXT_MASK_UNREACHABLE` flag) are employed.
 
+On supported 64-bit GIL builds, `_gc_prev` also holds a mark-epoch bit.
+Eligible full collections first trace interpreter dictionaries and active frames,
+moving known-live objects to a separate list using the existing links. Only the
+remaining objects need trial reference counting. This shortcut is disabled for
+frozen heaps, secondary interpreters and after interpreter finalization.
+
 Optimization: delayed untracking containers
 ===========================================
 

Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-00-45-00.gh-issue-159137.gcRoots.rst

@@ -0,0 +1,2 @@
+Restore known-root marking in the GIL garbage collector to reduce trial
+reference counting during full collections.

Python/gc.c

@@ -22,6 +22,11 @@
 
 typedef struct _gc_runtime_state GCState;
 
+/* Root marking requires unique interpreter ownership. Keep this state outside
+ * _PyRuntime so secondary interpreters and runtime reinitialization disable it.
+ */
+static int gc_ownership_state;  // 0: never initialized, 1: unique, 2: unknown
+
 #ifdef Py_DEBUG
 #  define GC_DEBUG
 #endif
@@ -82,10 +87,10 @@ gc_set_refs(PyGC_Head *g, Py_ssize_t refs)
 }
 
 static inline void
-gc_reset_refs(PyGC_Head *g, Py_ssize_t refs)
+gc_reset_refs(PyGC_Head *g, Py_ssize_t refs, uintptr_t epoch)
 {
     g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED)
-        | PREV_MASK_COLLECTING
+        | epoch | PREV_MASK_COLLECTING
         | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);
 }
 
@@ -113,6 +118,10 @@ get_gc_state(void)
 void
 _PyGC_InitState(GCState *gcstate)
 {
+    int expected = 0;
+    if (!_Py_atomic_compare_exchange_int(&gc_ownership_state, &expected, 1)) {
+        _Py_atomic_store_int_relaxed(&gc_ownership_state, 2);
+    }
 #define INIT_HEAD(GEN) \
     do { \
         GEN.head._gc_next = (uintptr_t)&GEN.head; \
@@ -124,6 +133,7 @@ _PyGC_InitState(GCState *gcstate)
         INIT_HEAD(gcstate->generations[i]);
     };
     gcstate->generation0 = GEN_HEAD(gcstate, 0);
+    gcstate->mark_epoch = 0;
     INIT_HEAD(gcstate->permanent_generation);
 
 #undef INIT_HEAD
@@ -160,7 +170,7 @@ _gc_prev values
 
 Between collections, _gc_prev is used for doubly linked list.
 
-Lowest two bits of _gc_prev are used for flags.
+Low bits of _gc_prev are used for flags, including a mark epoch on supported ABIs.
 PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends
 or _PyObject_GC_UNTRACK() is called.
 
@@ -400,6 +410,7 @@ update_refs(PyGC_Head *containers)
     PyGC_Head *next;
     PyGC_Head *gc = GC_NEXT(containers);
     Py_ssize_t candidates = 0;
+    uintptr_t epoch = _PyGC_PREV_MASK_EPOCH ? get_gc_state()->mark_epoch : 0;
 
     while (gc != containers) {
         next = GC_NEXT(gc);
@@ -410,7 +421,7 @@ update_refs(PyGC_Head *containers)
             gc = next;
             continue;
         }
-        gc_reset_refs(gc, Py_REFCNT(op));
+        gc_reset_refs(gc, Py_REFCNT(op), epoch);
         /* Python's cyclic gc should never see an incoming refcount
          * of 0:  if something decref'ed to 0, it should have been
          * deallocated immediately at that time.
@@ -1227,6 +1238,98 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) {
     return candidates;
 }
 
+typedef struct {
+    PyGC_Head live;
+    uintptr_t epoch;
+    Py_ssize_t candidates;
+} GCMarkQueue;
+
+static int
+visit_add_to_container(PyObject *op, void *arg)
+{
+    OBJECT_STAT_INC(object_visits);
+    if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op) &&
+        _PyObject_GC_IS_TRACKED(op)) {
+        PyGC_Head *gc = AS_GC(op);
+        GCMarkQueue *queue = arg;
+        /* Recheck ownership after tp_is_gc, which can create an interpreter.
+         * A partial live set is safe: its outgoing references stay external.
+         */
+        if ((gc->_gc_prev & _PyGC_PREV_MASK_EPOCH) != queue->epoch &&
+            _Py_atomic_load_int_relaxed(&gc_ownership_state) == 1) {
+            gc->_gc_prev = (gc->_gc_prev & ~_PyGC_PREV_MASK_EPOCH)
+                | queue->epoch;
+            gc_list_move(gc, &queue->live);
+            queue->candidates++;
+        }
+    }
+    return 0;
+}
+
+/* Avoid extension callbacks while root enumeration holds HEAD_LOCK. */
+static int
+visit_known_root(PyObject *op, void *arg)
+{
+    if (op == NULL) {
+        return 0;
+    }
+    if (Py_TYPE(op)->tp_is_gc != NULL) {
+        OBJECT_STAT_INC(object_visits);
+        return 0;
+    }
+    return visit_add_to_container(op, arg);
+}
+
+static void
+mark_known_roots(GCMarkQueue *queue)
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    visit_known_root(interp->sysdict, queue);
+    visit_known_root(interp->builtins, queue);
+    visit_known_root(interp->dict, queue);
+    visit_known_root(interp->imports.modules, queue);
+    _Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) {
+        for (_PyInterpreterFrame *frame = tstate->current_frame;
+             frame != NULL; frame = frame->previous) {
+            if (frame->owner >= FRAME_OWNED_BY_INTERPRETER ||
+                frame->stackpointer == NULL || _PyFrame_IsIncomplete(frame)) {
+                continue;
+            }
+            (void)_PyFrame_Traverse(frame, visit_known_root, queue);
+        }
+    }
+    _Py_FOR_EACH_TSTATE_END(interp);
+    PyGC_Head *gc = GC_NEXT(&queue->live);
+    while (gc != &queue->live) {
+        PyObject *op = FROM_GC(gc);
+        (void)Py_TYPE(op)->tp_traverse(op, visit_add_to_container, queue);
+        /* Traversal can append objects, so read next afterward. */
+        gc = GC_NEXT(gc);
+    }
+}
+
+/* Toggle the epoch to distinguish unseen objects. GC_TRACK stamps new objects
+ * with the new epoch. The live list doubles as the pending work queue.
+ */
+static Py_ssize_t
+collect_from_known_roots(PyGC_Head *base, PyGC_Head *unreachable)
+{
+    GCState *gcstate = get_gc_state();
+    gcstate->mark_epoch ^= _PyGC_PREV_MASK_EPOCH;
+    GCMarkQueue queue = {.epoch = gcstate->mark_epoch};
+    gc_list_init(&queue.live);
+    mark_known_roots(&queue);
+
+    /* Known-live references stay external to the residual analysis.
+     * update_refs stamps the remaining candidates before callbacks can run.
+     */
+    Py_ssize_t candidates = deduce_unreachable(base, unreachable);
+    gc_list_merge(base, &queue.live);
+    gc_list_merge(&queue.live, base);
+    validate_list(base, collecting_clear_unreachable_clear);
+    return queue.candidates + candidates;
+}
+
 /* Handle objects that may have resurrected after a call to 'finalize_garbage', moving
    them to 'old_generation' and placing the rest on 'still_unreachable'.
 
@@ -1521,7 +1624,19 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
     }
     validate_list(old, collecting_clear_unreachable_clear);
 
-    stats.candidates = deduce_unreachable(young, &unreachable);
+    if (_PyGC_PREV_MASK_EPOCH && generation == NUM_GENERATIONS - 1 &&
+        _Py_IsMainInterpreter(tstate->interp) &&
+        gcstate->heap_size >= 4096 &&
+        gc_list_is_empty(&gcstate->permanent_generation.head) &&
+        _Py_atomic_load_int_relaxed(&gc_ownership_state) == 1) {
+        /* Without frozen or foreign objects, every tracked non-immortal
+         * object belongs to this full collection before update_refs.
+         */
+        stats.candidates = collect_from_known_roots(young, &unreachable);
+    }
+    else {
+        stats.candidates = deduce_unreachable(young, &unreachable);
+    }
 
     untrack_tuples(young);
     /* Move reachable objects to next generation. */
@@ -1901,6 +2016,7 @@ finalize_unlink_gc_head(PyGC_Head *gc) {
 void
 _PyGC_Fini(PyInterpreterState *interp)
 {
+    _Py_atomic_store_int_relaxed(&gc_ownership_state, 2);
     GCState *gcstate = &interp->gc;
     Py_CLEAR(gcstate->garbage);
     Py_CLEAR(gcstate->callbacks);

Tools/c-analyzer/cpython/ignored.tsv

@@ -22,6 +22,7 @@ Python/fileutils.c	set_inheritable	ioctl_works	-
 # (set lazily, atomically, *after* first init)
 Modules/posixmodule.c	os_dup2_impl	dup3_works_atomic	-
 Modules/posixmodule.c	os_pipe_impl	pipe2_works_atomic	-
+Python/gc.c	-	gc_ownership_state	Atomic process-lifetime ownership guard; never reset across interpreter or runtime finalization.
 
 ## guards around resource init
 Python/thread_pthread.h	PyThread__init_thread	lib_initialized	-