Skip to content

Commit 26e0e87

Browse files
committed
bpo-24658: Fix read/write on file with a size greater than 2GB on OSX
bpo-24658: Fix read/write on file with a size greater than 2GB on OSX
1 parent 8175547 commit 26e0e87

File tree

4 files changed

+33
-25
lines changed

4 files changed

+33
-25
lines changed

Include/fileutils.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@ PyAPI_FUNC(char*) Py_EncodeLocale(
1919

2020
PyAPI_FUNC(PyObject *) _Py_device_encoding(int);
2121

22+
#if defined(MS_WINDOWS) || defined(__APPLE__)
23+
/* On Windows, the count parameter of read() is an int
24+
See issue #24658
25+
*/
26+
#define _PY_READ_MAX INT_MAX
27+
#define _PY_WRITE_MAX INT_MAX
28+
#else
29+
#define _PY_READ_MAX PY_SSIZE_T_MAX
30+
#define _PY_WRITE_MAX PY_SSIZE_T_MAX
31+
#endif
32+
2233
#ifdef MS_WINDOWS
2334
struct _Py_stat_struct {
2435
unsigned long st_dev;

Lib/test/test_largefile.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import stat
66
import sys
77
import unittest
8-
from test.support import TESTFN, requires, unlink
8+
from test.support import TESTFN, requires, unlink, bigmemtest, _2G
99
import io # C implementation of io
1010
import _pyio as pyio # Python implementation of io
1111

@@ -45,6 +45,19 @@ def tearDownClass(cls):
4545
raise cls.failureException('File was not truncated by opening '
4646
'with mode "wb"')
4747

48+
def test_large_reads_writes(self):
49+
# see issue #24658
50+
requires('largefile',
51+
'test requires %s bytes and a long time to run' % size)
52+
with self.open(TESTFN, "wb") as f:
53+
b = b'x' * size
54+
self.assertEqual(f.write(b), size)
55+
self.assertEqual(f.tell(), size)
56+
57+
with self.open(TESTFN, "rb") as f:
58+
f.read()
59+
self.assertEqual(f.tell(), size)
60+
4861
def test_osstat(self):
4962
self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1)
5063

Modules/_io/fileio.c

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -788,11 +788,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size)
788788
if (size < 0)
789789
return _io_FileIO_readall_impl(self);
790790

791-
#ifdef MS_WINDOWS
792-
/* On Windows, the count parameter of read() is an int */
793-
if (size > INT_MAX)
794-
size = INT_MAX;
795-
#endif
791+
if (size > _PY_READ_MAX) {
792+
size = _PY_READ_MAX;
793+
}
796794

797795
bytes = PyBytes_FromStringAndSize(NULL, size);
798796
if (bytes == NULL)

Python/fileutils.c

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1175,18 +1175,9 @@ _Py_read(int fd, void *buf, size_t count)
11751175
* handler raised an exception. */
11761176
assert(!PyErr_Occurred());
11771177

1178-
#ifdef MS_WINDOWS
1179-
if (count > INT_MAX) {
1180-
/* On Windows, the count parameter of read() is an int */
1181-
count = INT_MAX;
1182-
}
1183-
#else
1184-
if (count > PY_SSIZE_T_MAX) {
1185-
/* if count is greater than PY_SSIZE_T_MAX,
1186-
* read() result is undefined */
1187-
count = PY_SSIZE_T_MAX;
1178+
if (count > _PY_READ_MAX) {
1179+
count = _PY_READ_MAX;
11881180
}
1189-
#endif
11901181

11911182
_Py_BEGIN_SUPPRESS_IPH
11921183
do {
@@ -1237,15 +1228,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held)
12371228
depending on heap usage). */
12381229
count = 32767;
12391230
}
1240-
else if (count > INT_MAX)
1241-
count = INT_MAX;
1242-
#else
1243-
if (count > PY_SSIZE_T_MAX) {
1244-
/* write() should truncate count to PY_SSIZE_T_MAX, but it's safer
1245-
* to do it ourself to have a portable behaviour. */
1246-
count = PY_SSIZE_T_MAX;
1247-
}
12481231
#endif
1232+
if (count > _PY_WRITE_MAX) {
1233+
count = _PY_WRITE_MAX;
1234+
}
12491235

12501236
if (gil_held) {
12511237
do {

0 commit comments

Comments
 (0)