add LimitedSemaphore (#13642)

* treat Semaphore._waiters as length zero when it is None

We access the `._waiters` private attribute of the Python asyncio
`Semaphore` class.  This was changed in Python 3.10.8 (and other versions)
to be initialized to `None` instead of an empty deque.  Our existing
unconditional length checks failed on the new `None` default.  This seems
to block syncing.

python/cpython#97020

https://github.com/python/cpython/compare/v3.10.7..v3.10.8#diff-0fee1befb15023abc0dad2623effa93a304946796929f6cb445d11a57821e737

Reported traceback:
```python-traceback
2022-10-12T20:03:59.367 full_node full_node_server : INFO Connected with full_node {'host': '65.34.144.6', 'port': 8444}
2022-10-12T20:03:59.370 full_node full_node_server : ERROR Exception: object of type 'NoneType' has no len(), {'host': '65.34.144.6', 'port': 8444}. Traceback (most recent call last):
File "/home/summa/chia-blockchain/chia/server/server.py", line 598, in wrapped_coroutine
result = await coroutine
File "/home/summa/chia-blockchain/chia/full_node/full_node_api.py", line 114, in new_peak
waiter_count = len(self.full_node.new_peak_sem._waiters)
TypeError: object of type 'NoneType' has no len()

2022-10-12T20:03:59.371 full_node full_node_server : ERROR Exception: object of type 'NoneType' has no len() <class 'TypeError'>, closing connection {'host': '65.34.144.6', 'port': 8444}. Traceback (most recent call last):
File "/home/summa/chia-blockchain/chia/server/server.py", line 608, in api_call
response: Optional[Message] = await asyncio.wait_for(wrapped_coroutine(), timeout=timeout)
File "/usr/lib/python3.10/asyncio/tasks.py", line 408, in wait_for
return await fut
File "/home/summa/chia-blockchain/chia/server/server.py", line 605, in wrapped_coroutine
raise e
File "/home/summa/chia-blockchain/chia/server/server.py", line 598, in wrapped_coroutine
result = await coroutine
File "/home/summa/chia-blockchain/chia/full_node/full_node_api.py", line 114, in new_peak
waiter_count = len(self.full_node.new_peak_sem._waiters)
TypeError: object of type 'NoneType' has no len()

2022-10-12T20:03:59.487 full_node full_node_server : INFO Connection closed: 65.34.144.6, node id: 506fe4c05ce6b72bb707471842e552307c7a547aa9ba981175db5c08fa3e47e6
```

* add LimitedSemaphore

Author: altendky

chia/full_node/full_node.py

@@ -72,6 +72,7 @@
 from chia.util.db_wrapper import DBWrapper2, manage_connection
 from chia.util.errors import ConsensusError, Err, ValidationError
 from chia.util.ints import uint8, uint32, uint64, uint128
+from chia.util.limited_semaphore import LimitedSemaphore
 from chia.util.path import path_from_root
 from chia.util.safe_cancel_task import cancel_task_safe
 from chia.util.profiler import profile_task
@@ -124,8 +125,8 @@ class FullNode:
     simulator_transaction_callback: Optional[Callable[[bytes32], Awaitable[None]]]
     _sync_task: Optional[asyncio.Task[None]]
     _transaction_queue: Optional[asyncio.PriorityQueue[Tuple[int, TransactionQueueEntry]]]
-    _compact_vdf_sem: Optional[asyncio.Semaphore]
-    _new_peak_sem: Optional[asyncio.Semaphore]
+    _compact_vdf_sem: Optional[LimitedSemaphore]
+    _new_peak_sem: Optional[LimitedSemaphore]
     _respond_transaction_semaphore: Optional[asyncio.Semaphore]
     _db_wrapper: Optional[DBWrapper2]
     _hint_store: Optional[HintStore]
@@ -266,12 +267,12 @@ def hint_store(self) -> HintStore:
         return self._hint_store
 
     @property
-    def new_peak_sem(self) -> asyncio.Semaphore:
+    def new_peak_sem(self) -> LimitedSemaphore:
         assert self._new_peak_sem is not None
         return self._new_peak_sem
 
     @property
-    def compact_vdf_sem(self) -> asyncio.Semaphore:
+    def compact_vdf_sem(self) -> LimitedSemaphore:
         assert self._compact_vdf_sem is not None
         return self._compact_vdf_sem
 
@@ -313,11 +314,11 @@ def _set_state_changed_callback(self, callback: Callable[..., Any]) -> None:
 
     async def _start(self) -> None:
         self._timelord_lock = asyncio.Lock()
-        self._compact_vdf_sem = asyncio.Semaphore(4)
+        self._compact_vdf_sem = LimitedSemaphore.create(active_limit=4, waiting_limit=20)
 
         # We don't want to run too many concurrent new_peak instances, because it would fetch the same block from
         # multiple peers and re-validate.
-        self._new_peak_sem = asyncio.Semaphore(2)
+        self._new_peak_sem = LimitedSemaphore.create(active_limit=2, waiting_limit=20)
 
         # These many respond_transaction tasks can be active at any point in time
         self._respond_transaction_semaphore = asyncio.Semaphore(200)

chia/full_node/full_node_api.py

@@ -55,6 +55,7 @@
 from chia.util.generator_tools import get_block_header, tx_removals_and_additions
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint32, uint64, uint128
+from chia.util.limited_semaphore import LimitedSemaphoreFullError
 from chia.util.merkle_set import MerkleSet
 from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions
 
@@ -120,17 +121,12 @@ async def new_peak(self, request: full_node_protocol.NewPeak, peer: ws.WSChiaCon
         """
         # this semaphore limits the number of tasks that can call new_peak() at
         # the same time, since it can be expensive
-        new_peak_sem = self.full_node.new_peak_sem
-        waiter_count = 0 if new_peak_sem._waiters is None else len(new_peak_sem._waiters)
-
-        if waiter_count > 0:
-            self.full_node.log.debug(f"new_peak Waiters: {waiter_count}")
-
-        if waiter_count > 20:
+        try:
+            async with self.full_node.new_peak_sem.acquire():
+                await self.full_node.new_peak(request, peer)
+        except LimitedSemaphoreFullError:
             return None
 
-        async with new_peak_sem:
-            await self.full_node.new_peak(request, peer)
         return None
 
     @api_request(peer_required=True)
@@ -1411,12 +1407,6 @@ async def new_compact_vdf(
         if self.full_node.sync_store.get_sync_mode():
             return None
 
-        compact_vdf_sem = self.full_node.compact_vdf_sem
-        waiter_count = 0 if compact_vdf_sem._waiters is None else len(compact_vdf_sem._waiters)
-        if waiter_count > 20:
-            self.log.debug(f"Ignoring NewCompactVDF: {request}, _waiters")
-            return None
-
         name = std_hash(request_bytes)
         if name in self.full_node.compact_vdf_requests:
             self.log.debug(f"Ignoring NewCompactVDF: {request}, already requested")
@@ -1425,11 +1415,16 @@ async def new_compact_vdf(
 
         # this semaphore will only allow a limited number of tasks call
         # new_compact_vdf() at a time, since it can be expensive
-        async with self.full_node.compact_vdf_sem:
-            try:
-                await self.full_node.new_compact_vdf(request, peer)
-            finally:
-                self.full_node.compact_vdf_requests.remove(name)
+        try:
+            async with self.full_node.compact_vdf_sem.acquire():
+                try:
+                    await self.full_node.new_compact_vdf(request, peer)
+                finally:
+                    self.full_node.compact_vdf_requests.remove(name)
+        except LimitedSemaphoreFullError:
+            self.log.debug(f"Ignoring NewCompactVDF: {request}, _waiters")
+            return None
+
         return None
 
     @api_request(peer_required=True, reply_types=[ProtocolMessageTypes.respond_compact_vdf])

chia/util/limited_semaphore.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from dataclasses import dataclass
+from typing import AsyncIterator
+
+from typing_extensions import final
+
+
+class LimitedSemaphoreFullError(Exception):
+    def __init__(self) -> None:
+        super().__init__("no waiting slot available")
+
+
+@final
+@dataclass
+class LimitedSemaphore:
+    _semaphore: asyncio.Semaphore
+    _available_count: int
+
+    @classmethod
+    def create(cls, active_limit: int, waiting_limit: int) -> LimitedSemaphore:
+        return cls(
+            _semaphore=asyncio.Semaphore(active_limit),
+            _available_count=active_limit + waiting_limit,
+        )
+
+    @contextlib.asynccontextmanager
+    async def acquire(self) -> AsyncIterator[int]:
+        if self._available_count < 1:
+            raise LimitedSemaphoreFullError()
+
+        self._available_count -= 1
+        try:
+            async with self._semaphore:
+                yield self._available_count
+        finally:
+            self._available_count += 1

tests/util/test_limited_semaphore.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Optional
+
+import pytest
+
+from chia.util.limited_semaphore import LimitedSemaphore, LimitedSemaphoreFullError
+
+
+@pytest.mark.asyncio
+async def test_stuff() -> None:
+    active_limit = 2
+    waiting_limit = 4
+    total_limit = active_limit + waiting_limit
+    beyond_limit = 3
+    semaphore = LimitedSemaphore.create(active_limit=active_limit, waiting_limit=waiting_limit)
+    finish_event = asyncio.Event()
+
+    async def acquire(entered_event: Optional[asyncio.Event] = None) -> None:
+        async with semaphore.acquire():
+            assert entered_event is not None
+            entered_event.set()
+            await finish_event.wait()
+
+    entered_events = [asyncio.Event() for _ in range(active_limit)]
+    waiting_events = [asyncio.Event() for _ in range(waiting_limit)]
+    failed_events = [asyncio.Event() for _ in range(beyond_limit)]
+
+    entered_tasks = [asyncio.create_task(acquire(entered_event=event)) for event in entered_events]
+    waiting_tasks = [asyncio.create_task(acquire(entered_event=event)) for event in waiting_events]
+
+    await asyncio.gather(*(event.wait() for event in entered_events))
+    assert all(event.is_set() for event in entered_events)
+    assert all(not event.is_set() for event in waiting_events)
+
+    assert semaphore._available_count == 0
+
+    failure_tasks = [asyncio.create_task(acquire()) for _ in range(beyond_limit)]
+
+    failure_results = await asyncio.gather(*failure_tasks, return_exceptions=True)
+    assert [str(error) for error in failure_results] == [str(LimitedSemaphoreFullError())] * beyond_limit
+    assert all(not event.is_set() for event in failed_events)
+
+    assert semaphore._available_count == 0
+
+    finish_event.set()
+    success_results = await asyncio.gather(*entered_tasks, *waiting_tasks)
+    assert all(event.is_set() for event in waiting_events)
+    assert success_results == [None] * total_limit
+
+    assert semaphore._available_count == total_limit