Skip to content

Commit 6b5a279

Browse files
authored
bpo-32410: Implement loop.sock_sendfile() (#4976)
1 parent c495e79 commit 6b5a279

File tree

8 files changed

+609
-0
lines changed

8 files changed

+609
-0
lines changed

Doc/library/asyncio-eventloop.rst

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,36 @@ Low-level socket operations
701701

702702
:meth:`AbstractEventLoop.create_server` and :func:`start_server`.
703703

704+
.. coroutinemethod:: AbstractEventLoop.sock_sendfile(sock, file, \
705+
offset=0, count=None, \
706+
*, fallback=True)
707+
708+
Send a file using high-performance :mod:`os.sendfile` if possible
709+
and return the total number of bytes which were sent.
710+
711+
Asynchronous version of :meth:`socket.socket.sendfile`.
712+
713+
*sock* must be non-blocking :class:`~socket.socket` of
714+
:const:`socket.SOCK_STREAM` type.
715+
716+
*file* must be a regular file object opened in binary mode.
717+
718+
*offset* tells from where to start reading the file. If specified,
719+
*count* is the total number of bytes to transmit as opposed to
720+
sending the file until EOF is reached. File position is updated on
721+
return or also in case of error in which case :meth:`file.tell()
722+
<io.IOBase.tell>` can be used to figure out the number of bytes
723+
which were sent.
724+
725+
*fallback* set to ``True`` makes asyncio to manually read and send
726+
the file when the platform does not support the sendfile syscall
727+
(e.g. Windows or SSL socket on Unix).
728+
729+
Raise :exc:`RuntimeError` if the system does not support
730+
*sendfile* syscall and *fallback* is ``False``.
731+
732+
.. versionadded:: 3.7
733+
704734

705735
Resolve host name
706736
-----------------

Lib/asyncio/base_events.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,10 @@ def _run_until_complete_cb(fut):
154154
futures._get_loop(fut).stop()
155155

156156

157+
class _SendfileNotAvailable(RuntimeError):
158+
pass
159+
160+
157161
class Server(events.AbstractServer):
158162

159163
def __init__(self, loop, sockets):
@@ -647,6 +651,72 @@ async def getnameinfo(self, sockaddr, flags=0):
647651
return await self.run_in_executor(
648652
None, socket.getnameinfo, sockaddr, flags)
649653

654+
async def sock_sendfile(self, sock, file, offset=0, count=None,
655+
*, fallback=True):
656+
if self._debug and sock.gettimeout() != 0:
657+
raise ValueError("the socket must be non-blocking")
658+
self._check_sendfile_params(sock, file, offset, count)
659+
try:
660+
return await self._sock_sendfile_native(sock, file,
661+
offset, count)
662+
except _SendfileNotAvailable as exc:
663+
if fallback:
664+
return await self._sock_sendfile_fallback(sock, file,
665+
offset, count)
666+
else:
667+
raise RuntimeError(exc.args[0]) from None
668+
669+
async def _sock_sendfile_native(self, sock, file, offset, count):
670+
# NB: sendfile syscall is not supported for SSL sockets and
671+
# non-mmap files even if sendfile is supported by OS
672+
raise _SendfileNotAvailable(
673+
f"syscall sendfile is not available for socket {sock!r} "
674+
"and file {file!r} combination")
675+
676+
async def _sock_sendfile_fallback(self, sock, file, offset, count):
677+
if offset:
678+
file.seek(offset)
679+
blocksize = min(count, 16384) if count else 16384
680+
buf = bytearray(blocksize)
681+
total_sent = 0
682+
try:
683+
while True:
684+
if count:
685+
blocksize = min(count - total_sent, blocksize)
686+
if blocksize <= 0:
687+
break
688+
view = memoryview(buf)[:blocksize]
689+
read = file.readinto(view)
690+
if not read:
691+
break # EOF
692+
await self.sock_sendall(sock, view)
693+
total_sent += read
694+
return total_sent
695+
finally:
696+
if total_sent > 0 and hasattr(file, 'seek'):
697+
file.seek(offset + total_sent)
698+
699+
def _check_sendfile_params(self, sock, file, offset, count):
700+
if 'b' not in getattr(file, 'mode', 'b'):
701+
raise ValueError("file should be opened in binary mode")
702+
if not sock.type == socket.SOCK_STREAM:
703+
raise ValueError("only SOCK_STREAM type sockets are supported")
704+
if count is not None:
705+
if not isinstance(count, int):
706+
raise TypeError(
707+
"count must be a positive integer (got {!r})".format(count))
708+
if count <= 0:
709+
raise ValueError(
710+
"count must be a positive integer (got {!r})".format(count))
711+
if not isinstance(offset, int):
712+
raise TypeError(
713+
"offset must be a non-negative integer (got {!r})".format(
714+
offset))
715+
if offset < 0:
716+
raise ValueError(
717+
"offset must be a non-negative integer (got {!r})".format(
718+
offset))
719+
650720
async def create_connection(
651721
self, protocol_factory, host=None, port=None,
652722
*, ssl=None, family=0,

Lib/asyncio/events.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,10 @@ async def sock_connect(self, sock, address):
464464
async def sock_accept(self, sock):
465465
raise NotImplementedError
466466

467+
async def sock_sendfile(self, sock, file, offset=0, count=None,
468+
*, fallback=None):
469+
raise NotImplementedError
470+
467471
# Signal handling.
468472

469473
def add_signal_handler(self, sig, callback, *args):

Lib/asyncio/unix_events.py

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Selector event loop for Unix with signal handling."""
22

33
import errno
4+
import io
45
import os
56
import selectors
67
import signal
@@ -308,6 +309,98 @@ async def create_unix_server(
308309
ssl_handshake_timeout=ssl_handshake_timeout)
309310
return server
310311

312+
async def _sock_sendfile_native(self, sock, file, offset, count):
313+
try:
314+
os.sendfile
315+
except AttributeError as exc:
316+
raise base_events._SendfileNotAvailable(
317+
"os.sendfile() is not available")
318+
try:
319+
fileno = file.fileno()
320+
except (AttributeError, io.UnsupportedOperation) as err:
321+
raise base_events._SendfileNotAvailable("not a regular file")
322+
try:
323+
fsize = os.fstat(fileno).st_size
324+
except OSError as err:
325+
raise base_events._SendfileNotAvailable("not a regular file")
326+
blocksize = count if count else fsize
327+
if not blocksize:
328+
return 0 # empty file
329+
330+
fut = self.create_future()
331+
self._sock_sendfile_native_impl(fut, None, sock, fileno,
332+
offset, count, blocksize, 0)
333+
return await fut
334+
335+
def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno,
336+
offset, count, blocksize, total_sent):
337+
fd = sock.fileno()
338+
if registered_fd is not None:
339+
# Remove the callback early. It should be rare that the
340+
# selector says the fd is ready but the call still returns
341+
# EAGAIN, and I am willing to take a hit in that case in
342+
# order to simplify the common case.
343+
self.remove_writer(registered_fd)
344+
if fut.cancelled():
345+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
346+
return
347+
if count:
348+
blocksize = count - total_sent
349+
if blocksize <= 0:
350+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
351+
fut.set_result(total_sent)
352+
return
353+
354+
try:
355+
sent = os.sendfile(fd, fileno, offset, blocksize)
356+
except (BlockingIOError, InterruptedError):
357+
if registered_fd is None:
358+
self._sock_add_cancellation_callback(fut, sock)
359+
self.add_writer(fd, self._sock_sendfile_native_impl, fut,
360+
fd, sock, fileno,
361+
offset, count, blocksize, total_sent)
362+
except OSError as exc:
363+
if total_sent == 0:
364+
# We can get here for different reasons, the main
365+
# one being 'file' is not a regular mmap(2)-like
366+
# file, in which case we'll fall back on using
367+
# plain send().
368+
err = base_events._SendfileNotAvailable(
369+
"os.sendfile call failed")
370+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
371+
fut.set_exception(err)
372+
else:
373+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
374+
fut.set_exception(exc)
375+
except Exception as exc:
376+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
377+
fut.set_exception(exc)
378+
else:
379+
if sent == 0:
380+
# EOF
381+
self._sock_sendfile_update_filepos(fileno, offset, total_sent)
382+
fut.set_result(total_sent)
383+
else:
384+
offset += sent
385+
total_sent += sent
386+
if registered_fd is None:
387+
self._sock_add_cancellation_callback(fut, sock)
388+
self.add_writer(fd, self._sock_sendfile_native_impl, fut,
389+
fd, sock, fileno,
390+
offset, count, blocksize, total_sent)
391+
392+
def _sock_sendfile_update_filepos(self, fileno, offset, total_sent):
393+
if total_sent > 0:
394+
os.lseek(fileno, offset, os.SEEK_SET)
395+
396+
def _sock_add_cancellation_callback(self, fut, sock):
397+
def cb(fut):
398+
if fut.cancelled():
399+
fd = sock.fileno()
400+
if fd != -1:
401+
self.remove_writer(fd)
402+
fut.add_done_callback(cb)
403+
311404

312405
class _UnixReadPipeTransport(transports.ReadTransport):
313406

0 commit comments

Comments
 (0)