Skip to content

Commit 79c663f

Browse files
committed
CoW: Use exponential backoff when clearing dead references
1 parent 0021d24 commit 79c663f

File tree

2 files changed

+14
-5
lines changed

2 files changed

+14
-5
lines changed

doc/source/whatsnew/v2.1.2.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Fixed regressions
1717
- Fixed regression in :meth:`DataFrame.join` where result has missing values and dtype is arrow backed string (:issue:`55348`)
1818
- Fixed regression in :meth:`DataFrame.resample` which was extrapolating back to ``origin`` when ``origin`` was outside its bounds (:issue:`55064`)
1919
- Fixed regression in :meth:`DataFrame.sort_index` which was not sorting correctly when the index was a sliced :class:`MultiIndex` (:issue:`55379`)
20+
- Fixed performance regression in Copy-on-Write mechanism (:issue:`55256`, :issue:`55245`)
2021

2122
.. ---------------------------------------------------------------------------
2223
.. _whatsnew_212.bug_fixes:

pandas/_libs/internals.pyx

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -890,17 +890,25 @@ cdef class BlockValuesRefs:
890890
"""
891891
cdef:
892892
public list referenced_blocks
893+
public int clear_counter
893894

894895
def __cinit__(self, blk: Block | None = None) -> None:
895896
if blk is not None:
896897
self.referenced_blocks = [weakref.ref(blk)]
897898
else:
898899
self.referenced_blocks = []
900+
self.clear_counter = 500 # set reasonably high
899901

900-
def _clear_dead_references(self) -> None:
901-
self.referenced_blocks = [
902-
ref for ref in self.referenced_blocks if ref() is not None
903-
]
902+
def _clear_dead_references(self, force=False) -> None:
903+
if force or len(self.referenced_blocks) > self.clear_counter:
904+
self.referenced_blocks = [
905+
ref for ref in self.referenced_blocks if ref() is not None
906+
]
907+
nr_of_refs = len(self.referenced_blocks)
908+
if nr_of_refs < self.clear_counter // 2:
909+
self.clear_counter = self.clear_counter // 2
910+
elif nr_of_refs > self.clear_counter:
911+
self.clear_counter = self.clear_counter * 2
904912

905913
def add_reference(self, blk: Block) -> None:
906914
"""Adds a new reference to our reference collection.
@@ -934,6 +942,6 @@ cdef class BlockValuesRefs:
934942
-------
935943
bool
936944
"""
937-
self._clear_dead_references()
945+
self._clear_dead_references(force=True)
938946
# Checking for more references than block pointing to itself
939947
return len(self.referenced_blocks) > 1

0 commit comments

Comments
 (0)