[2.7] bpo-24658: Fix read/write greater than 2 GiB on macOS (pythonGH-1705)

On macOS, fix reading from and writing into a file with a size larger
than 2 GiB.

Author: matrixise

Lib/test/test_largefile.py

@@ -8,7 +8,7 @@
 import sys
 import unittest
 from test.test_support import run_unittest, TESTFN, verbose, requires, \
-                              unlink
+                              unlink, bigmemtest
 import io  # C implementation of io
 import _pyio as pyio # Python implementation of io
 
@@ -48,6 +48,15 @@ def test_seek(self):
                 print('check file size with os.fstat')
             self.assertEqual(os.fstat(f.fileno())[stat.ST_SIZE], size+1)
 
+    # _pyio.FileIO.readall() uses a temporary bytearry then casted to bytes,
+    # so memuse=2 is needed
+    @bigmemtest(minsize=size, memuse=2)
+    def test_large_read(self, _size):
+        # bpo-24658: Test that a read greater than 2GB does not fail.
+        with self.open(TESTFN, "rb") as f:
+            self.assertEqual(len(f.read()), size + 1)
+            self.assertEqual(f.tell(), size + 1)
+
     def test_osstat(self):
         if verbose:
             print('check file size with os.stat')

Misc/NEWS.d/next/macOS/2018-10-18-02-20-38.bpo-24658.c7F8Pu.rst

@@ -0,0 +1,2 @@
+On macOS, fix reading from and writing into a file with a size larger than 2
+GiB.

Modules/_io/fileio.c

@@ -37,6 +37,19 @@
 #include <windows.h>
 #endif
 
+#if defined (MS_WINDOWS) || defined(__APPLE__)
+    /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611).
+       On macOS 10.13, read() and write() with more than INT_MAX bytes
+       fail with EINVAL (bpo-24658). */
+#   define _PY_READ_MAX  INT_MAX
+#   define _PY_WRITE_MAX INT_MAX
+#else
+    /* write() should truncate the input to PY_SSIZE_T_MAX bytes,
+       but it's safer to do it ourself to have a portable behaviour */
+#   define _PY_READ_MAX  PY_SSIZE_T_MAX
+#   define _PY_WRITE_MAX PY_SSIZE_T_MAX
+#endif
+
 #if BUFSIZ < (8*1024)
 #define SMALLCHUNK (8*1024)
 #elif (BUFSIZ >= (2 << 25))
@@ -604,9 +617,9 @@ fileio_readall(fileio *self)
         Py_BEGIN_ALLOW_THREADS
         errno = 0;
         n = newsize - total;
+        if (n > _PY_READ_MAX)
+            n = _PY_READ_MAX;
 #if defined(MS_WIN64) || defined(MS_WINDOWS)
-        if (n > INT_MAX)
-            n = INT_MAX;
         n = read(self->fd,
                  PyBytes_AS_STRING(result) + total,
                  (int)n);
@@ -668,10 +681,10 @@ fileio_read(fileio *self, PyObject *args)
         return fileio_readall(self);
     }
 
-#if defined(MS_WIN64) || defined(MS_WINDOWS)
-    if (size > INT_MAX)
-        size = INT_MAX;
-#endif
+    if (size > _PY_READ_MAX) {
+        size = _PY_READ_MAX;
+    }
+
     bytes = PyBytes_FromStringAndSize(NULL, size);
     if (bytes == NULL)
         return NULL;
@@ -723,9 +736,9 @@ fileio_write(fileio *self, PyObject *args)
         Py_BEGIN_ALLOW_THREADS
         errno = 0;
         len = pbuf.len;
+        if (len > _PY_WRITE_MAX)
+            len = _PY_WRITE_MAX;
 #if defined(MS_WIN64) || defined(MS_WINDOWS)
-        if (len > INT_MAX)
-            len = INT_MAX;
         n = write(self->fd, pbuf.buf, (int)len);
 #else
         n = write(self->fd, pbuf.buf, len);