Skip to content

Commit a2c5f53

Browse files
jmikolajmikola
committed
PHPLIB-247: Use strings instead of memory stream for GridFS download buffering
1 parent 3de2734 commit a2c5f53

File tree

4 files changed

+139
-122
lines changed

4 files changed

+139
-122
lines changed

‎src/GridFS/CollectionWrapper.php

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
use MongoDB\Driver\Cursor;
88
use MongoDB\Driver\Manager;
99
use MongoDB\Driver\ReadPreference;
10-
use IteratorIterator;
1110
use stdClass;
1211

1312
/**
@@ -72,6 +71,27 @@ public function dropCollections()
7271
$this->chunksCollection->drop(['typeMap' => []]);
7372
}
7473

74+
/**
75+
* Finds GridFS chunk documents for a given file ID and optional offset.
76+
*
77+
* @param mixed $id File ID
78+
* @param integer $fromChunk Starting chunk (inclusive)
79+
* @return Cursor
80+
*/
81+
public function findChunksByFileId($id, $fromChunk = 0)
82+
{
83+
return $this->chunksCollection->find(
84+
[
85+
'files_id' => $id,
86+
'n' => ['$gte' => $fromChunk],
87+
],
88+
[
89+
'sort' => ['n' => 1],
90+
'typeMap' => ['root' => 'stdClass'],
91+
]
92+
);
93+
}
94+
7595
/**
7696
* Finds a GridFS file document for a given filename and revision.
7797
*
@@ -162,25 +182,6 @@ public function getBucketName()
162182
return $this->bucketName;
163183
}
164184

165-
/**
166-
* Returns a chunks iterator for a given file ID.
167-
*
168-
* @param mixed $id
169-
* @return IteratorIterator
170-
*/
171-
public function getChunksIteratorByFilesId($id)
172-
{
173-
$cursor = $this->chunksCollection->find(
174-
['files_id' => $id],
175-
[
176-
'sort' => ['n' => 1],
177-
'typeMap' => ['root' => 'stdClass'],
178-
]
179-
);
180-
181-
return new IteratorIterator($cursor);
182-
}
183-
184185
/**
185186
* Return the database name.
186187
*

‎src/GridFS/ReadableStream.php

Lines changed: 102 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
use MongoDB\Exception\InvalidArgumentException;
66
use MongoDB\GridFS\Exception\CorruptFileException;
7+
use IteratorIterator;
78
use stdClass;
89

910
/**
@@ -14,18 +15,15 @@
1415
class ReadableStream
1516
{
1617
private $buffer;
17-
private $bufferEmpty;
18-
private $bufferFresh;
19-
private $bytesSeen = 0;
18+
private $bufferOffset = 0;
2019
private $chunkSize;
2120
private $chunkOffset = 0;
2221
private $chunksIterator;
2322
private $collectionWrapper;
23+
private $expectedLastChunkSize = 0;
2424
private $file;
25-
private $firstCheck = true;
26-
private $iteratorEmpty = false;
2725
private $length;
28-
private $numChunks;
26+
private $numChunks = 0;
2927

3028
/**
3129
* Constructs a readable GridFS stream.
@@ -49,13 +47,15 @@ public function __construct(CollectionWrapper $collectionWrapper, stdClass $file
4947
}
5048

5149
$this->file = $file;
52-
$this->chunkSize = $file->chunkSize;
53-
$this->length = $file->length;
50+
$this->chunkSize = (integer) $file->chunkSize;
51+
$this->length = (integer) $file->length;
5452

55-
$this->chunksIterator = $collectionWrapper->getChunksIteratorByFilesId($file->_id);
5653
$this->collectionWrapper = $collectionWrapper;
57-
$this->numChunks = ceil($this->length / $this->chunkSize);
58-
$this->initEmptyBuffer();
54+
55+
if ($this->length > 0) {
56+
$this->numChunks = (integer) ceil($this->length / $this->chunkSize);
57+
$this->expectedLastChunkSize = ($this->length - (($this->numChunks - 1) * $this->chunkSize));
58+
}
5959
}
6060

6161
/**
@@ -75,56 +75,7 @@ public function __debugInfo()
7575

7676
public function close()
7777
{
78-
fclose($this->buffer);
79-
}
80-
81-
/**
82-
* Read bytes from the stream.
83-
*
84-
* Note: this method may return a string smaller than the requested length
85-
* if data is not available to be read.
86-
*
87-
* @param integer $numBytes Number of bytes to read
88-
* @return string
89-
* @throws InvalidArgumentException if $numBytes is negative
90-
*/
91-
public function downloadNumBytes($numBytes)
92-
{
93-
if ($numBytes < 0) {
94-
throw new InvalidArgumentException(sprintf('$numBytes must be >= zero; given: %d', $numBytes));
95-
}
96-
97-
if ($numBytes == 0) {
98-
return '';
99-
}
100-
101-
if ($this->bufferFresh) {
102-
rewind($this->buffer);
103-
$this->bufferFresh = false;
104-
}
105-
106-
// TODO: Should we be checking for fread errors here?
107-
$output = fread($this->buffer, $numBytes);
108-
109-
if (strlen($output) == $numBytes) {
110-
return $output;
111-
}
112-
113-
$this->initEmptyBuffer();
114-
115-
$bytesLeft = $numBytes - strlen($output);
116-
117-
while (strlen($output) < $numBytes && $this->advanceChunks()) {
118-
$bytesLeft = $numBytes - strlen($output);
119-
$output .= substr($this->chunksIterator->current()->data->getData(), 0, $bytesLeft);
120-
}
121-
122-
if ( ! $this->iteratorEmpty && $this->length > 0 && $bytesLeft < strlen($this->chunksIterator->current()->data->getData())) {
123-
fwrite($this->buffer, substr($this->chunksIterator->current()->data->getData(), $bytesLeft));
124-
$this->bufferEmpty = false;
125-
}
126-
127-
return $output;
78+
// Nothing to do
12879
}
12980

13081
/**
@@ -147,58 +98,123 @@ public function getSize()
14798
return $this->length;
14899
}
149100

101+
/**
102+
* Return whether the current read position is at the end of the stream.
103+
*
104+
* @return boolean
105+
*/
150106
public function isEOF()
151107
{
152-
return ($this->iteratorEmpty && $this->bufferEmpty);
108+
if ($this->chunkOffset === $this->numChunks - 1) {
109+
return $this->bufferOffset >= $this->expectedLastChunkSize;
110+
}
111+
112+
return $this->chunkOffset >= $this->numChunks;
153113
}
154114

155-
private function advanceChunks()
115+
/**
116+
* Read bytes from the stream.
117+
*
118+
* Note: this method may return a string smaller than the requested length
119+
* if data is not available to be read.
120+
*
121+
* @param integer $length Number of bytes to read
122+
* @return string
123+
* @throws InvalidArgumentException if $length is negative
124+
*/
125+
public function readBytes($length)
156126
{
157-
if ($this->chunkOffset >= $this->numChunks) {
158-
$this->iteratorEmpty = true;
127+
if ($length < 0) {
128+
throw new InvalidArgumentException(sprintf('$length must be >= 0; given: %d', $length));
129+
}
159130

160-
return false;
131+
if ($this->chunksIterator === null) {
132+
$this->initChunksIterator();
133+
}
134+
135+
if ($this->buffer === null && ! $this->initBufferFromCurrentChunk()) {
136+
return '';
161137
}
162138

163-
if ($this->firstCheck) {
164-
$this->chunksIterator->rewind();
165-
$this->firstCheck = false;
166-
} else {
167-
$this->chunksIterator->next();
139+
$data = '';
140+
141+
while (strlen($data) < $length) {
142+
if ($this->bufferOffset >= strlen($this->buffer) && ! $this->initBufferFromNextChunk()) {
143+
break;
144+
}
145+
146+
$initialDataLength = strlen($data);
147+
$data .= substr($this->buffer, $this->bufferOffset, $length - $initialDataLength);
148+
$this->bufferOffset += strlen($data) - $initialDataLength;
149+
}
150+
151+
return $data;
152+
}
153+
154+
/**
155+
* Initialize the buffer to the current chunk's data.
156+
*
157+
* @return boolean Whether there was a current chunk to read
158+
* @throws CorruptFileException if an expected chunk could not be read successfully
159+
*/
160+
private function initBufferFromCurrentChunk()
161+
{
162+
if ($this->chunkOffset === 0 && $this->numChunks === 0) {
163+
return false;
168164
}
169165

170166
if ( ! $this->chunksIterator->valid()) {
171167
throw CorruptFileException::missingChunk($this->chunkOffset);
172168
}
173169

174-
if ($this->chunksIterator->current()->n != $this->chunkOffset) {
175-
throw CorruptFileException::unexpectedIndex($this->chunksIterator->current()->n, $this->chunkOffset);
170+
$currentChunk = $this->chunksIterator->current();
171+
172+
if ($currentChunk->n !== $this->chunkOffset) {
173+
throw CorruptFileException::unexpectedIndex($currentChunk->n, $this->chunkOffset);
176174
}
177175

178-
$actualChunkSize = strlen($this->chunksIterator->current()->data->getData());
176+
$this->buffer = $currentChunk->data->getData();
179177

180-
$expectedChunkSize = ($this->chunkOffset == $this->numChunks - 1)
181-
? ($this->length - $this->bytesSeen)
178+
$actualChunkSize = strlen($this->buffer);
179+
180+
$expectedChunkSize = ($this->chunkOffset === $this->numChunks - 1)
181+
? $this->expectedLastChunkSize
182182
: $this->chunkSize;
183183

184-
if ($actualChunkSize != $expectedChunkSize) {
184+
if ($actualChunkSize !== $expectedChunkSize) {
185185
throw CorruptFileException::unexpectedSize($actualChunkSize, $expectedChunkSize);
186186
}
187187

188-
$this->bytesSeen += $actualChunkSize;
189-
$this->chunkOffset++;
190-
191188
return true;
192189
}
193190

194-
private function initEmptyBuffer()
191+
/**
192+
* Advance to the next chunk and initialize the buffer to its data.
193+
*
194+
* @return boolean Whether there was a next chunk to read
195+
* @throws CorruptFileException if an expected chunk could not be read successfully
196+
*/
197+
private function initBufferFromNextChunk()
195198
{
196-
if (isset($this->buffer)) {
197-
fclose($this->buffer);
199+
if ($this->chunkOffset === $this->numChunks - 1) {
200+
return false;
198201
}
199202

200-
$this->buffer = fopen("php://memory", "w+b");
201-
$this->bufferEmpty = true;
202-
$this->bufferFresh = true;
203+
$this->bufferOffset = 0;
204+
$this->chunkOffset++;
205+
$this->chunksIterator->next();
206+
207+
return $this->initBufferFromCurrentChunk();
208+
}
209+
210+
/**
211+
* Initializes the chunk iterator starting from the current offset.
212+
*/
213+
private function initChunksIterator()
214+
{
215+
$cursor = $this->collectionWrapper->findChunksByFileId($this->file->_id, $this->chunkOffset);
216+
217+
$this->chunksIterator = new IteratorIterator($cursor);
218+
$this->chunksIterator->rewind();
203219
}
204220
}

‎src/GridFS/StreamWrapper.php

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,17 +103,17 @@ public function stream_open($path, $mode, $options, &$openedPath)
103103
* if data is not available to be read.
104104
*
105105
* @see http://php.net/manual/en/streamwrapper.stream-read.php
106-
* @param integer $count Number of bytes to read
106+
* @param integer $length Number of bytes to read
107107
* @return string
108108
*/
109-
public function stream_read($count)
109+
public function stream_read($length)
110110
{
111111
if ( ! $this->stream instanceof ReadableStream) {
112112
return '';
113113
}
114114

115115
try {
116-
return $this->stream->downloadNumBytes($count);
116+
return $this->stream->readBytes($length);
117117
} catch (Exception $e) {
118118
trigger_error(sprintf('%s: %s', get_class($e), $e->getMessage()), \E_USER_WARNING);
119119
return false;

0 commit comments

Comments
 (0)