Skip to content

Commit 6401e56

Browse files
[2.7] bpo-31530: Stop crashes when iterating over a file on multiple threads. (#3672)
1 parent 1bce4ef commit 6401e56

File tree

3 files changed

+52
-3
lines changed

3 files changed

+52
-3
lines changed

Lib/test/test_file2k.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,6 +652,38 @@ def io_func():
652652
self.f.writelines('')
653653
self._test_close_open_io(io_func)
654654

655+
def test_iteration_torture(self):
656+
# bpo-31530: Crash when concurrently iterate over a file.
657+
with open(self.filename, "wb") as fp:
658+
for i in xrange(2**20):
659+
fp.write(b"0"*50 + b"\n")
660+
with open(self.filename, "rb") as f:
661+
def iterate():
662+
try:
663+
for l in f:
664+
pass
665+
except IOError:
666+
pass
667+
self._run_workers(iterate, 10)
668+
669+
def test_iteration_seek(self):
670+
# bpo-31530: Crash when concurrently seek and iterate over a file.
671+
with open(self.filename, "wb") as fp:
672+
for i in xrange(10000):
673+
fp.write(b"0"*50 + b"\n")
674+
with open(self.filename, "rb") as f:
675+
it = iter([1] + [0]*10) # one thread reads, others seek
676+
def iterate():
677+
try:
678+
if next(it):
679+
for l in f:
680+
pass
681+
else:
682+
for i in range(100):
683+
f.seek(i*100, 0)
684+
except IOError:
685+
pass
686+
self._run_workers(iterate, 10)
655687

656688
@unittest.skipUnless(os.name == 'posix', 'test requires a posix system.')
657689
class TestFileSignalEINTR(unittest.TestCase):
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fixed crashes when iterating over a file on multiple threads.
2+
seek() and next() methods of file objects now raise an exception during
3+
concurrent operation on the same file object.
4+
A lock can be used to prevent the error.

Objects/fileobject.c

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -430,15 +430,15 @@ close_the_file(PyFileObject *f)
430430
if (Py_REFCNT(f) > 0) {
431431
PyErr_SetString(PyExc_IOError,
432432
"close() called during concurrent "
433-
"operation on the same file object.");
433+
"operation on the same file object");
434434
} else {
435435
/* This should not happen unless someone is
436436
* carelessly playing with the PyFileObject
437437
* struct fields and/or its associated FILE
438438
* pointer. */
439439
PyErr_SetString(PyExc_SystemError,
440440
"PyFileObject locking error in "
441-
"destructor (refcnt <= 0 at close).");
441+
"destructor (refcnt <= 0 at close)");
442442
}
443443
return NULL;
444444
}
@@ -762,6 +762,12 @@ file_seek(PyFileObject *f, PyObject *args)
762762

763763
if (f->f_fp == NULL)
764764
return err_closed();
765+
if (f->unlocked_count > 0) {
766+
PyErr_SetString(PyExc_IOError,
767+
"seek() called during concurrent "
768+
"operation on the same file object");
769+
return NULL;
770+
}
765771
drop_readahead(f);
766772
whence = 0;
767773
if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence))
@@ -2238,6 +2244,7 @@ readahead(PyFileObject *f, Py_ssize_t bufsize)
22382244
{
22392245
Py_ssize_t chunksize;
22402246

2247+
assert(f->unlocked_count == 0);
22412248
if (f->f_buf != NULL) {
22422249
if( (f->f_bufend - f->f_bufptr) >= 1)
22432250
return 0;
@@ -2279,6 +2286,12 @@ readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize)
22792286
char *buf;
22802287
Py_ssize_t len;
22812288

2289+
if (f->unlocked_count > 0) {
2290+
PyErr_SetString(PyExc_IOError,
2291+
"next() called during concurrent "
2292+
"operation on the same file object");
2293+
return NULL;
2294+
}
22822295
if (f->f_buf == NULL)
22832296
if (readahead(f, bufsize) < 0)
22842297
return NULL;
@@ -2692,7 +2705,7 @@ int PyObject_AsFileDescriptor(PyObject *o)
26922705
}
26932706
else {
26942707
PyErr_SetString(PyExc_TypeError,
2695-
"argument must be an int, or have a fileno() method.");
2708+
"argument must be an int, or have a fileno() method");
26962709
return -1;
26972710
}
26982711

0 commit comments

Comments
 (0)