Skip to content

Commit 4720808

Browse files
authored
Merge pull request #1060 from bluetech/pre-typing-fixes
Pre-typing fixes/improvements
2 parents 3e92604 + cdff86a commit 4720808

File tree

12 files changed

+70
-84
lines changed

12 files changed

+70
-84
lines changed

.pre-commit-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ repos:
3131
additional_dependencies:
3232
- pytest>=7.0.0
3333
- execnet>=2.1.0
34-
- py>=1.10.0
34+
- types-psutil

pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ select = [
8585
"W", # pycodestyle
8686
"T10", # flake8-debugger
8787
"PIE", # flake8-pie
88+
"FA", # flake8-future-annotations
8889
"PGH", # pygrep-hooks
8990
"PLE", # pylint error
9091
"PLW", # pylint warning
@@ -135,6 +136,7 @@ lines-after-imports = 2
135136

136137
[tool.mypy]
137138
mypy_path = ["src"]
139+
files = ["src", "testing"]
138140
# TODO: Enable this & fix errors.
139141
# check_untyped_defs = true
140142
disallow_any_generics = true

src/xdist/dsession.py

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -317,13 +317,6 @@ def worker_collectreport(self, node, rep):
317317
assert not rep.passed
318318
self._failed_worker_collectreport(node, rep)
319319

320-
def worker_warning_captured(self, warning_message, when, item):
321-
"""Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
322-
# This hook as been removed in pytest 7.1, and we can remove support once we only
323-
# support pytest >=7.1.
324-
kwargs = dict(warning_message=warning_message, when=when, item=item)
325-
self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
326-
327320
def worker_warning_recorded(self, warning_message, when, nodeid, location):
328321
"""Emitted when a node calls the pytest_warning_recorded hook."""
329322
kwargs = dict(
@@ -374,10 +367,9 @@ def triggershutdown(self):
374367
def handle_crashitem(self, nodeid, worker):
375368
# XXX get more reporting info by recording pytest_runtest_logstart?
376369
# XXX count no of failures and retry N times
377-
runner = self.config.pluginmanager.getplugin("runner")
378370
fspath = nodeid.split("::")[0]
379371
msg = f"worker {worker.gateway.id!r} crashed while running {nodeid!r}"
380-
rep = runner.TestReport(
372+
rep = pytest.TestReport(
381373
nodeid, (fspath, None, fspath), (), "failed", msg, "???"
382374
)
383375
rep.node = worker

src/xdist/looponfail.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@
77
the controlling process which should best never happen.
88
"""
99

10+
from __future__ import annotations
11+
1012
import os
1113
from pathlib import Path
1214
import sys
1315
import time
14-
from typing import Dict
1516
from typing import Sequence
1617

1718
from _pytest._io import TerminalWriter
@@ -45,7 +46,7 @@ def pytest_cmdline_main(config):
4546
return 2 # looponfail only can get stop with ctrl-C anyway
4647

4748

48-
def looponfail_main(config: "pytest.Config") -> None:
49+
def looponfail_main(config: pytest.Config) -> None:
4950
remotecontrol = RemoteControl(config)
5051
config_roots = config.getini("looponfailroots")
5152
if not config_roots:
@@ -79,9 +80,7 @@ def trace(self, *args):
7980
def initgateway(self):
8081
return execnet.makegateway("popen")
8182

82-
def setup(self, out=None):
83-
if out is None:
84-
out = TerminalWriter()
83+
def setup(self):
8584
if hasattr(self, "gateway"):
8685
raise ValueError("already have gateway %r" % self.gateway)
8786
self.trace("setting up worker session")
@@ -93,6 +92,8 @@ def setup(self, out=None):
9392
)
9493
remote_outchannel = channel.receive()
9594

95+
out = TerminalWriter()
96+
9697
def write(s):
9798
out._file.write(s)
9899
out._file.flush()
@@ -238,7 +239,7 @@ def main(self):
238239
class StatRecorder:
239240
def __init__(self, rootdirlist: Sequence[Path]) -> None:
240241
self.rootdirlist = rootdirlist
241-
self.statcache: Dict[Path, os.stat_result] = {}
242+
self.statcache: dict[Path, os.stat_result] = {}
242243
self.check() # snapshot state
243244

244245
def fil(self, p: Path) -> bool:
@@ -256,7 +257,7 @@ def waitonchange(self, checkinterval=1.0):
256257

257258
def check(self, removepycfiles: bool = True) -> bool:
258259
changed = False
259-
newstat: Dict[Path, os.stat_result] = {}
260+
newstat: dict[Path, os.stat_result] = {}
260261
for rootdir in self.rootdirlist:
261262
for path in visit_path(rootdir, filter=self.fil, recurse=self.rec):
262263
oldstat = self.statcache.pop(path, None)

src/xdist/remote.py

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"""
88

99
import contextlib
10+
import enum
1011
import os
1112
import sys
1213
import time
@@ -57,10 +58,12 @@ def worker_title(title):
5758
pass
5859

5960

60-
class WorkerInteractor:
61-
SHUTDOWN_MARK = object()
62-
QUEUE_REPLACED_MARK = object()
61+
class Marker(enum.Enum):
62+
SHUTDOWN = 0
63+
QUEUE_REPLACED = 1
64+
6365

66+
class WorkerInteractor:
6467
def __init__(self, config, channel):
6568
self.config = config
6669
self.workerid = config.workerinput.get("workerid", "?")
@@ -79,7 +82,7 @@ def _get_next_item_index(self):
7982
is replaced concurrently in another thread.
8083
"""
8184
result = self.torun.get()
82-
while result is self.QUEUE_REPLACED_MARK:
85+
while result is Marker.QUEUE_REPLACED:
8386
result = self.torun.get()
8487
return result
8588

@@ -114,8 +117,8 @@ def pytest_collection(self, session):
114117
self.sendevent("collectionstart")
115118

116119
def handle_command(self, command):
117-
if command is self.SHUTDOWN_MARK:
118-
self.torun.put(self.SHUTDOWN_MARK)
120+
if command is Marker.SHUTDOWN:
121+
self.torun.put(Marker.SHUTDOWN)
119122
return
120123

121124
name, kwargs = command
@@ -128,7 +131,7 @@ def handle_command(self, command):
128131
for i in range(len(self.session.items)):
129132
self.torun.put(i)
130133
elif name == "shutdown":
131-
self.torun.put(self.SHUTDOWN_MARK)
134+
self.torun.put(Marker.SHUTDOWN)
132135
elif name == "steal":
133136
self.steal(kwargs["indices"])
134137

@@ -149,14 +152,14 @@ def old_queue_get_nowait_noraise():
149152
self.torun.put(i)
150153

151154
self.sendevent("unscheduled", indices=stolen)
152-
old_queue.put(self.QUEUE_REPLACED_MARK)
155+
old_queue.put(Marker.QUEUE_REPLACED)
153156

154157
@pytest.hookimpl
155158
def pytest_runtestloop(self, session):
156159
self.log("entering main loop")
157-
self.channel.setcallback(self.handle_command, endmarker=self.SHUTDOWN_MARK)
160+
self.channel.setcallback(self.handle_command, endmarker=Marker.SHUTDOWN)
158161
self.nextitem_index = self._get_next_item_index()
159-
while self.nextitem_index is not self.SHUTDOWN_MARK:
162+
while self.nextitem_index is not Marker.SHUTDOWN:
160163
self.run_one_test()
161164
if session.shouldfail or session.shouldstop:
162165
break
@@ -168,16 +171,16 @@ def run_one_test(self):
168171

169172
items = self.session.items
170173
item = items[self.item_index]
171-
if self.nextitem_index is self.SHUTDOWN_MARK:
174+
if self.nextitem_index is Marker.SHUTDOWN:
172175
nextitem = None
173176
else:
174177
nextitem = items[self.nextitem_index]
175178

176179
worker_title("[pytest-xdist running] %s" % item.nodeid)
177180

178-
start = time.time()
181+
start = time.perf_counter()
179182
self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
180-
duration = time.time() - start
183+
duration = time.perf_counter() - start
181184

182185
worker_title("[pytest-xdist idle]")
183186

src/xdist/scheduler/each.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,7 @@ def mark_test_complete(self, node, item_index, duration=0):
101101
self.node2pending[node].remove(item_index)
102102

103103
def mark_test_pending(self, item):
104-
self.pending.insert(
105-
0,
106-
self.collection.index(item),
107-
)
108-
for node in self.node2pending:
109-
self.check_schedule(node)
104+
raise NotImplementedError()
110105

111106
def remove_node(self, node):
112107
# KeyError if we didn't get an add_node() yet

src/xdist/scheduler/loadscope.py

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,8 @@ def __init__(self, config, log=None):
9090
self.collection = None
9191

9292
self.workqueue = OrderedDict()
93-
self.assigned_work = OrderedDict()
94-
self.registered_collections = OrderedDict()
93+
self.assigned_work = {}
94+
self.registered_collections = {}
9595

9696
if log is None:
9797
self.log = Producer("loadscopesched")
@@ -156,7 +156,7 @@ def add_node(self, node):
156156
bootstraps a new node.
157157
"""
158158
assert node not in self.assigned_work
159-
self.assigned_work[node] = OrderedDict()
159+
self.assigned_work[node] = {}
160160

161161
def remove_node(self, node):
162162
"""Remove a node from the scheduler.
@@ -252,7 +252,7 @@ def _assign_work_unit(self, node):
252252
scope, work_unit = self.workqueue.popitem(last=False)
253253

254254
# Keep track of the assigned work
255-
assigned_to_node = self.assigned_work.setdefault(node, default=OrderedDict())
255+
assigned_to_node = self.assigned_work.setdefault(node, {})
256256
assigned_to_node[scope] = work_unit
257257

258258
# Ask the node to execute the workload
@@ -349,10 +349,10 @@ def schedule(self):
349349
return
350350

351351
# Determine chunks of work (scopes)
352-
unsorted_workqueue = OrderedDict()
352+
unsorted_workqueue = {}
353353
for nodeid in self.collection:
354354
scope = self._split_scope(nodeid)
355-
work_unit = unsorted_workqueue.setdefault(scope, default=OrderedDict())
355+
work_unit = unsorted_workqueue.setdefault(scope, {})
356356
work_unit[nodeid] = False
357357

358358
# Insert tests scopes into work queue ordered by number of tests.
@@ -368,7 +368,7 @@ def schedule(self):
368368
self.log(f"Shutting down {extra_nodes} nodes")
369369

370370
for _ in range(extra_nodes):
371-
unused_node, assigned = self.assigned_work.popitem(last=True)
371+
unused_node, assigned = self.assigned_work.popitem()
372372

373373
self.log(f"Shutting down unused node {unused_node}")
374374
unused_node.shutdown()
@@ -407,9 +407,6 @@ def _check_nodes_have_same_collection(self):
407407
same_collection = False
408408
self.log(msg)
409409

410-
if self.config is None:
411-
continue
412-
413410
rep = pytest.CollectReport(
414411
nodeid=node.gateway.id,
415412
outcome="failed",

src/xdist/workermanage.py

Lines changed: 14 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
1+
from __future__ import annotations
2+
3+
import enum
14
import fnmatch
25
import os
36
from pathlib import Path
47
import re
58
import sys
69
from typing import Any
7-
from typing import List
8-
from typing import Optional
910
from typing import Sequence
10-
from typing import Set
11-
from typing import Tuple
1211
from typing import Union
1312
import uuid
1413

@@ -60,7 +59,7 @@ def __init__(self, config, specs=None, defaultchdir="pyexecnetcache") -> None:
6059
self.specs.append(spec)
6160
self.roots = self._getrsyncdirs()
6261
self.rsyncoptions = self._getrsyncoptions()
63-
self._rsynced_specs: Set[Tuple[Any, Any]] = set()
62+
self._rsynced_specs: set[tuple[Any, Any]] = set()
6463

6564
def rsync_roots(self, gateway):
6665
"""Rsync the set of roots to the node's gateway cwd."""
@@ -89,7 +88,7 @@ def teardown_nodes(self):
8988
def _getxspecs(self):
9089
return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
9190

92-
def _getrsyncdirs(self) -> List[Path]:
91+
def _getrsyncdirs(self) -> list[Path]:
9392
for spec in self.specs:
9493
if not spec.popen or spec.chdir:
9594
break
@@ -174,7 +173,7 @@ def __init__(
174173
self,
175174
sourcedir: PathLike,
176175
*,
177-
ignores: Optional[Sequence[PathLike]] = None,
176+
ignores: Sequence[PathLike] | None = None,
178177
verbose: bool = True,
179178
) -> None:
180179
if ignores is None:
@@ -201,7 +200,7 @@ def _report_send_file(self, gateway, modified_rel_path):
201200
print(f"{gateway.spec}:{remotepath} <= {path}")
202201

203202

204-
def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]:
203+
def make_reltoroot(roots: Sequence[Path], args: list[str]) -> list[str]:
205204
# XXX introduce/use public API for splitting pytest args
206205
splitcode = "::"
207206
result = []
@@ -216,7 +215,7 @@ def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]:
216215
result.append(arg)
217216
continue
218217
for root in roots:
219-
x: Optional[Path]
218+
x: Path | None
220219
try:
221220
x = fspath.relative_to(root)
222221
except ValueError:
@@ -230,9 +229,11 @@ def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]:
230229
return result
231230

232231

233-
class WorkerController:
234-
ENDMARK = -1
232+
class Marker(enum.Enum):
233+
END = -1
234+
235235

236+
class WorkerController:
236237
class RemoteHook:
237238
@pytest.hookimpl(trylast=True)
238239
def pytest_xdist_getremotemodule(self):
@@ -283,7 +284,7 @@ def setup(self):
283284
self.channel.send((self.workerinput, args, option_dict, change_sys_path))
284285

285286
if self.putevent:
286-
self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
287+
self.channel.setcallback(self.process_from_remote, endmarker=Marker.END)
287288

288289
def ensure_teardown(self):
289290
if hasattr(self, "channel"):
@@ -331,7 +332,7 @@ def process_from_remote(self, eventcall):
331332
avoid raising exceptions or doing heavy work.
332333
"""
333334
try:
334-
if eventcall == self.ENDMARK:
335+
if eventcall is Marker.END:
335336
err = self.channel._getremoteerror()
336337
if not self._down:
337338
if not err or isinstance(err, EOFError):
@@ -374,16 +375,6 @@ def process_from_remote(self, eventcall):
374375
nodeid=kwargs["nodeid"],
375376
fslocation=kwargs["nodeid"],
376377
)
377-
elif eventname == "warning_captured":
378-
warning_message = unserialize_warning_message(
379-
kwargs["warning_message_data"]
380-
)
381-
self.notify_inproc(
382-
eventname,
383-
warning_message=warning_message,
384-
when=kwargs["when"],
385-
item=kwargs["item"],
386-
)
387378
elif eventname == "warning_recorded":
388379
warning_message = unserialize_warning_message(
389380
kwargs["warning_message_data"]

0 commit comments

Comments
 (0)