Merge "Maintenance: Add an easy way to access Config instances"
[lhc/web/wiklou.git] / includes / utils / ZipDirectoryReader.php
1 <?php
2 /**
3 * ZIP file directories reader, for the purposes of upload verification.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 */
22
23 /**
24 * A class for reading ZIP file directories, for the purposes of upload
25 * verification.
26 *
27 * Only a functional interface is provided: ZipFileReader::read(). No access is
28 * given to object instances.
29 *
30 */
31 class ZipDirectoryReader {
32 /**
33 * Read a ZIP file and call a function for each file discovered in it.
34 *
35 * Because this class is aimed at verification, an error is raised on
36 * suspicious or ambiguous input, instead of emulating some standard
37 * behavior.
38 *
39 * @param string $fileName The archive file name
40 * @param array $callback The callback function. It will be called for each file
41 * with a single associative array each time, with members:
42 *
43 * - name: The file name. Directories conventionally have a trailing
44 * slash.
45 *
46 * - mtime: The file modification time, in MediaWiki 14-char format
47 *
48 * - size: The uncompressed file size
49 *
50 * @param array $options An associative array of read options, with the option
51 * name in the key. This may currently contain:
52 *
53 * - zip64: If this is set to true, then we will emulate a
54 * library with ZIP64 support, like OpenJDK 7. If it is set to
55 * false, then we will emulate a library with no knowledge of
56 * ZIP64.
57 *
58 * NOTE: The ZIP64 code is untested and probably doesn't work. It
59 * turned out to be easier to just reject ZIP64 archive uploads,
60 * since they are likely to be very rare. Confirming safety of a
61 * ZIP64 file is fairly complex. What do you do with a file that is
62 * ambiguous and broken when read with a non-ZIP64 reader, but valid
63 * when read with a ZIP64 reader? This situation is normal for a
64 * valid ZIP64 file, and working out what non-ZIP64 readers will make
65 * of such a file is not trivial.
66 *
67 * @return Status A Status object. The following fatal errors are defined:
68 *
69 * - zip-file-open-error: The file could not be opened.
70 *
71 * - zip-wrong-format: The file does not appear to be a ZIP file.
72 *
73 * - zip-bad: There was something wrong or ambiguous about the file
74 * data.
75 *
76 * - zip-unsupported: The ZIP file uses features which
77 * ZipDirectoryReader does not support.
78 *
79 * The default messages for those fatal errors are written in a way that
80 * makes sense for upload verification.
81 *
82 * If a fatal error is returned, more information about the error will be
83 * available in the debug log.
84 *
85 * Note that the callback function may be called any number of times before
86 * a fatal error is returned. If this occurs, the data sent to the callback
87 * function should be discarded.
88 */
89 public static function read( $fileName, $callback, $options = array() ) {
90 $zdr = new self( $fileName, $callback, $options );
91
92 return $zdr->execute();
93 }
94
95 /** The file name */
96 protected $fileName;
97
98 /** The opened file resource */
99 protected $file;
100
101 /** The cached length of the file, or null if it has not been loaded yet. */
102 protected $fileLength;
103
104 /** A segmented cache of the file contents */
105 protected $buffer;
106
107 /** The file data callback */
108 protected $callback;
109
110 /** The ZIP64 mode */
111 protected $zip64 = false;
112
113 /** Stored headers */
114 protected $eocdr, $eocdr64, $eocdr64Locator;
115
116 protected $data;
117
118 /** The "extra field" ID for ZIP64 central directory entries */
119 const ZIP64_EXTRA_HEADER = 0x0001;
120
121 /** The segment size for the file contents cache */
122 const SEGSIZE = 16384;
123
124 /** The index of the "general field" bit for UTF-8 file names */
125 const GENERAL_UTF8 = 11;
126
127 /** The index of the "general field" bit for central directory encryption */
128 const GENERAL_CD_ENCRYPTED = 13;
129
130 /**
131 * Private constructor
132 * @param string $fileName
133 * @param callable $callback
134 * @param array $options
135 */
136 protected function __construct( $fileName, $callback, $options ) {
137 $this->fileName = $fileName;
138 $this->callback = $callback;
139
140 if ( isset( $options['zip64'] ) ) {
141 $this->zip64 = $options['zip64'];
142 }
143 }
144
145 /**
146 * Read the directory according to settings in $this.
147 *
148 * @return Status
149 */
150 function execute() {
151 $this->file = fopen( $this->fileName, 'r' );
152 $this->data = array();
153 if ( !$this->file ) {
154 return Status::newFatal( 'zip-file-open-error' );
155 }
156
157 $status = Status::newGood();
158 try {
159 $this->readEndOfCentralDirectoryRecord();
160 if ( $this->zip64 ) {
161 list( $offset, $size ) = $this->findZip64CentralDirectory();
162 $this->readCentralDirectory( $offset, $size );
163 } else {
164 if ( $this->eocdr['CD size'] == 0xffffffff
165 || $this->eocdr['CD offset'] == 0xffffffff
166 || $this->eocdr['CD entries total'] == 0xffff
167 ) {
168 $this->error( 'zip-unsupported', 'Central directory header indicates ZIP64, ' .
169 'but we are in legacy mode. Rejecting this upload is necessary to avoid ' .
170 'opening vulnerabilities on clients using OpenJDK 7 or later.' );
171 }
172
173 list( $offset, $size ) = $this->findOldCentralDirectory();
174 $this->readCentralDirectory( $offset, $size );
175 }
176 } catch ( ZipDirectoryReaderError $e ) {
177 $status->fatal( $e->getErrorCode() );
178 }
179
180 fclose( $this->file );
181
182 return $status;
183 }
184
185 /**
186 * Throw an error, and log a debug message
187 * @param mixed $code
188 * @param string $debugMessage
189 */
190 function error( $code, $debugMessage ) {
191 wfDebug( __CLASS__ . ": Fatal error: $debugMessage\n" );
192 throw new ZipDirectoryReaderError( $code );
193 }
194
195 /**
196 * Read the header which is at the end of the central directory,
197 * unimaginatively called the "end of central directory record" by the ZIP
198 * spec.
199 */
200 function readEndOfCentralDirectoryRecord() {
201 $info = array(
202 'signature' => 4,
203 'disk' => 2,
204 'CD start disk' => 2,
205 'CD entries this disk' => 2,
206 'CD entries total' => 2,
207 'CD size' => 4,
208 'CD offset' => 4,
209 'file comment length' => 2,
210 );
211 $structSize = $this->getStructSize( $info );
212 $startPos = $this->getFileLength() - 65536 - $structSize;
213 if ( $startPos < 0 ) {
214 $startPos = 0;
215 }
216
217 $block = $this->getBlock( $startPos );
218 $sigPos = strrpos( $block, "PK\x05\x06" );
219 if ( $sigPos === false ) {
220 $this->error( 'zip-wrong-format',
221 "zip file lacks EOCDR signature. It probably isn't a zip file." );
222 }
223
224 $this->eocdr = $this->unpack( substr( $block, $sigPos ), $info );
225 $this->eocdr['EOCDR size'] = $structSize + $this->eocdr['file comment length'];
226
227 if ( $structSize + $this->eocdr['file comment length'] != strlen( $block ) - $sigPos ) {
228 $this->error( 'zip-bad', 'trailing bytes after the end of the file comment' );
229 }
230 if ( $this->eocdr['disk'] !== 0
231 || $this->eocdr['CD start disk'] !== 0
232 ) {
233 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR)' );
234 }
235 $this->eocdr += $this->unpack(
236 $block,
237 array( 'file comment' => array( 'string', $this->eocdr['file comment length'] ) ),
238 $sigPos + $structSize );
239 $this->eocdr['position'] = $startPos + $sigPos;
240 }
241
242 /**
243 * Read the header called the "ZIP64 end of central directory locator". An
244 * error will be raised if it does not exist.
245 */
246 function readZip64EndOfCentralDirectoryLocator() {
247 $info = array(
248 'signature' => array( 'string', 4 ),
249 'eocdr64 start disk' => 4,
250 'eocdr64 offset' => 8,
251 'number of disks' => 4,
252 );
253 $structSize = $this->getStructSize( $info );
254
255 $start = $this->getFileLength() - $this->eocdr['EOCDR size'] - $structSize;
256 $block = $this->getBlock( $start, $structSize );
257 $this->eocdr64Locator = $data = $this->unpack( $block, $info );
258
259 if ( $data['signature'] !== "PK\x06\x07" ) {
260 // Note: Java will allow this and continue to read the
261 // EOCDR64, so we have to reject the upload, we can't
262 // just use the EOCDR header instead.
263 $this->error( 'zip-bad', 'wrong signature on Zip64 end of central directory locator' );
264 }
265 }
266
267 /**
268 * Read the header called the "ZIP64 end of central directory record". It
269 * may replace the regular "end of central directory record" in ZIP64 files.
270 */
271 function readZip64EndOfCentralDirectoryRecord() {
272 if ( $this->eocdr64Locator['eocdr64 start disk'] != 0
273 || $this->eocdr64Locator['number of disks'] != 0
274 ) {
275 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR64 locator)' );
276 }
277
278 $info = array(
279 'signature' => array( 'string', 4 ),
280 'EOCDR64 size' => 8,
281 'version made by' => 2,
282 'version needed' => 2,
283 'disk' => 4,
284 'CD start disk' => 4,
285 'CD entries this disk' => 8,
286 'CD entries total' => 8,
287 'CD size' => 8,
288 'CD offset' => 8
289 );
290 $structSize = $this->getStructSize( $info );
291 $block = $this->getBlock( $this->eocdr64Locator['eocdr64 offset'], $structSize );
292 $this->eocdr64 = $data = $this->unpack( $block, $info );
293 if ( $data['signature'] !== "PK\x06\x06" ) {
294 $this->error( 'zip-bad', 'wrong signature on Zip64 end of central directory record' );
295 }
296 if ( $data['disk'] !== 0
297 || $data['CD start disk'] !== 0
298 ) {
299 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR64)' );
300 }
301 }
302
303 /**
304 * Find the location of the central directory, as would be seen by a
305 * non-ZIP64 reader.
306 *
307 * @return array List containing offset, size and end position.
308 */
309 function findOldCentralDirectory() {
310 $size = $this->eocdr['CD size'];
311 $offset = $this->eocdr['CD offset'];
312 $endPos = $this->eocdr['position'];
313
314 // Some readers use the EOCDR position instead of the offset field
315 // to find the directory, so to be safe, we check if they both agree.
316 if ( $offset + $size != $endPos ) {
317 $this->error( 'zip-bad', 'the central directory does not immediately precede the end ' .
318 'of central directory record' );
319 }
320
321 return array( $offset, $size );
322 }
323
324 /**
325 * Find the location of the central directory, as would be seen by a
326 * ZIP64-compliant reader.
327 *
328 * @return array List containing offset, size and end position.
329 */
330 function findZip64CentralDirectory() {
331 // The spec is ambiguous about the exact rules of precedence between the
332 // ZIP64 headers and the original headers. Here we follow zip_util.c
333 // from OpenJDK 7.
334 $size = $this->eocdr['CD size'];
335 $offset = $this->eocdr['CD offset'];
336 $numEntries = $this->eocdr['CD entries total'];
337 $endPos = $this->eocdr['position'];
338 if ( $size == 0xffffffff
339 || $offset == 0xffffffff
340 || $numEntries == 0xffff
341 ) {
342 $this->readZip64EndOfCentralDirectoryLocator();
343
344 if ( isset( $this->eocdr64Locator['eocdr64 offset'] ) ) {
345 $this->readZip64EndOfCentralDirectoryRecord();
346 if ( isset( $this->eocdr64['CD offset'] ) ) {
347 $size = $this->eocdr64['CD size'];
348 $offset = $this->eocdr64['CD offset'];
349 $endPos = $this->eocdr64Locator['eocdr64 offset'];
350 }
351 }
352 }
353 // Some readers use the EOCDR position instead of the offset field
354 // to find the directory, so to be safe, we check if they both agree.
355 if ( $offset + $size != $endPos ) {
356 $this->error( 'zip-bad', 'the central directory does not immediately precede the end ' .
357 'of central directory record' );
358 }
359
360 return array( $offset, $size );
361 }
362
363 /**
364 * Read the central directory at the given location
365 * @param int $offset
366 * @param int $size
367 */
368 function readCentralDirectory( $offset, $size ) {
369 $block = $this->getBlock( $offset, $size );
370
371 $fixedInfo = array(
372 'signature' => array( 'string', 4 ),
373 'version made by' => 2,
374 'version needed' => 2,
375 'general bits' => 2,
376 'compression method' => 2,
377 'mod time' => 2,
378 'mod date' => 2,
379 'crc-32' => 4,
380 'compressed size' => 4,
381 'uncompressed size' => 4,
382 'name length' => 2,
383 'extra field length' => 2,
384 'comment length' => 2,
385 'disk number start' => 2,
386 'internal attrs' => 2,
387 'external attrs' => 4,
388 'local header offset' => 4,
389 );
390 $fixedSize = $this->getStructSize( $fixedInfo );
391
392 $pos = 0;
393 while ( $pos < $size ) {
394 $data = $this->unpack( $block, $fixedInfo, $pos );
395 $pos += $fixedSize;
396
397 if ( $data['signature'] !== "PK\x01\x02" ) {
398 $this->error( 'zip-bad', 'Invalid signature found in directory entry' );
399 }
400
401 $variableInfo = array(
402 'name' => array( 'string', $data['name length'] ),
403 'extra field' => array( 'string', $data['extra field length'] ),
404 'comment' => array( 'string', $data['comment length'] ),
405 );
406 $data += $this->unpack( $block, $variableInfo, $pos );
407 $pos += $this->getStructSize( $variableInfo );
408
409 if ( $this->zip64 && (
410 $data['compressed size'] == 0xffffffff
411 || $data['uncompressed size'] == 0xffffffff
412 || $data['local header offset'] == 0xffffffff )
413 ) {
414 $zip64Data = $this->unpackZip64Extra( $data['extra field'] );
415 if ( $zip64Data ) {
416 $data = $zip64Data + $data;
417 }
418 }
419
420 if ( $this->testBit( $data['general bits'], self::GENERAL_CD_ENCRYPTED ) ) {
421 $this->error( 'zip-unsupported', 'central directory encryption is not supported' );
422 }
423
424 // Convert the timestamp into MediaWiki format
425 // For the format, please see the MS-DOS 2.0 Programmer's Reference,
426 // pages 3-5 and 3-6.
427 $time = $data['mod time'];
428 $date = $data['mod date'];
429
430 $year = 1980 + ( $date >> 9 );
431 $month = ( $date >> 5 ) & 15;
432 $day = $date & 31;
433 $hour = ( $time >> 11 ) & 31;
434 $minute = ( $time >> 5 ) & 63;
435 $second = ( $time & 31 ) * 2;
436 $timestamp = sprintf( "%04d%02d%02d%02d%02d%02d",
437 $year, $month, $day, $hour, $minute, $second );
438
439 // Convert the character set in the file name
440 if ( $this->testBit( $data['general bits'], self::GENERAL_UTF8 ) ) {
441 $name = $data['name'];
442 } else {
443 $name = iconv( 'CP437', 'UTF-8', $data['name'] );
444 }
445
446 // Compile a data array for the user, with a sensible format
447 $userData = array(
448 'name' => $name,
449 'mtime' => $timestamp,
450 'size' => $data['uncompressed size'],
451 );
452 call_user_func( $this->callback, $userData );
453 }
454 }
455
456 /**
457 * Interpret ZIP64 "extra field" data and return an associative array.
458 * @param string $extraField
459 * @return array|bool
460 */
461 function unpackZip64Extra( $extraField ) {
462 $extraHeaderInfo = array(
463 'id' => 2,
464 'size' => 2,
465 );
466 $extraHeaderSize = $this->getStructSize( $extraHeaderInfo );
467
468 $zip64ExtraInfo = array(
469 'uncompressed size' => 8,
470 'compressed size' => 8,
471 'local header offset' => 8,
472 'disk number start' => 4,
473 );
474
475 $extraPos = 0;
476 while ( $extraPos < strlen( $extraField ) ) {
477 $extra = $this->unpack( $extraField, $extraHeaderInfo, $extraPos );
478 $extraPos += $extraHeaderSize;
479 $extra += $this->unpack( $extraField,
480 array( 'data' => array( 'string', $extra['size'] ) ),
481 $extraPos );
482 $extraPos += $extra['size'];
483
484 if ( $extra['id'] == self::ZIP64_EXTRA_HEADER ) {
485 return $this->unpack( $extra['data'], $zip64ExtraInfo );
486 }
487 }
488
489 return false;
490 }
491
492 /**
493 * Get the length of the file.
494 * @return int
495 */
496 function getFileLength() {
497 if ( $this->fileLength === null ) {
498 $stat = fstat( $this->file );
499 $this->fileLength = $stat['size'];
500 }
501
502 return $this->fileLength;
503 }
504
505 /**
506 * Get the file contents from a given offset. If there are not enough bytes
507 * in the file to satisfy the request, an exception will be thrown.
508 *
509 * @param int $start The byte offset of the start of the block.
510 * @param int $length The number of bytes to return. If omitted, the remainder
511 * of the file will be returned.
512 *
513 * @return string
514 */
515 function getBlock( $start, $length = null ) {
516 $fileLength = $this->getFileLength();
517 if ( $start >= $fileLength ) {
518 $this->error( 'zip-bad', "getBlock() requested position $start, " .
519 "file length is $fileLength" );
520 }
521 if ( $length === null ) {
522 $length = $fileLength - $start;
523 }
524 $end = $start + $length;
525 if ( $end > $fileLength ) {
526 $this->error( 'zip-bad', "getBlock() requested end position $end, " .
527 "file length is $fileLength" );
528 }
529 $startSeg = floor( $start / self::SEGSIZE );
530 $endSeg = ceil( $end / self::SEGSIZE );
531
532 $block = '';
533 for ( $segIndex = $startSeg; $segIndex <= $endSeg; $segIndex++ ) {
534 $block .= $this->getSegment( $segIndex );
535 }
536
537 $block = substr( $block,
538 $start - $startSeg * self::SEGSIZE,
539 $length );
540
541 if ( strlen( $block ) < $length ) {
542 $this->error( 'zip-bad', 'getBlock() returned an unexpectedly small amount of data' );
543 }
544
545 return $block;
546 }
547
548 /**
549 * Get a section of the file starting at position $segIndex * self::SEGSIZE,
550 * of length self::SEGSIZE. The result is cached. This is a helper function
551 * for getBlock().
552 *
553 * If there are not enough bytes in the file to satisfy the request, the
554 * return value will be truncated. If a request is made for a segment beyond
555 * the end of the file, an empty string will be returned.
556 *
557 * @param int $segIndex
558 *
559 * @return string
560 */
561 function getSegment( $segIndex ) {
562 if ( !isset( $this->buffer[$segIndex] ) ) {
563 $bytePos = $segIndex * self::SEGSIZE;
564 if ( $bytePos >= $this->getFileLength() ) {
565 $this->buffer[$segIndex] = '';
566
567 return '';
568 }
569 if ( fseek( $this->file, $bytePos ) ) {
570 $this->error( 'zip-bad', "seek to $bytePos failed" );
571 }
572 $seg = fread( $this->file, self::SEGSIZE );
573 if ( $seg === false ) {
574 $this->error( 'zip-bad', "read from $bytePos failed" );
575 }
576 $this->buffer[$segIndex] = $seg;
577 }
578
579 return $this->buffer[$segIndex];
580 }
581
582 /**
583 * Get the size of a structure in bytes. See unpack() for the format of $struct.
584 * @param array $struct
585 * @return int
586 */
587 function getStructSize( $struct ) {
588 $size = 0;
589 foreach ( $struct as $type ) {
590 if ( is_array( $type ) ) {
591 list( , $fieldSize ) = $type;
592 $size += $fieldSize;
593 } else {
594 $size += $type;
595 }
596 }
597
598 return $size;
599 }
600
601 /**
602 * Unpack a binary structure. This is like the built-in unpack() function
603 * except nicer.
604 *
605 * @param string $string The binary data input
606 *
607 * @param array $struct An associative array giving structure members and their
608 * types. In the key is the field name. The value may be either an
609 * integer, in which case the field is a little-endian unsigned integer
610 * encoded in the given number of bytes, or an array, in which case the
611 * first element of the array is the type name, and the subsequent
612 * elements are type-dependent parameters. Only one such type is defined:
613 * - "string": The second array element gives the length of string.
614 * Not null terminated.
615 *
616 * @param int $offset The offset into the string at which to start unpacking.
617 *
618 * @throws MWException
619 * @return array Unpacked associative array. Note that large integers in the input
620 * may be represented as floating point numbers in the return value, so
621 * the use of weak comparison is advised.
622 */
623 function unpack( $string, $struct, $offset = 0 ) {
624 $size = $this->getStructSize( $struct );
625 if ( $offset + $size > strlen( $string ) ) {
626 $this->error( 'zip-bad', 'unpack() would run past the end of the supplied string' );
627 }
628
629 $data = array();
630 $pos = $offset;
631 foreach ( $struct as $key => $type ) {
632 if ( is_array( $type ) ) {
633 list( $typeName, $fieldSize ) = $type;
634 switch ( $typeName ) {
635 case 'string':
636 $data[$key] = substr( $string, $pos, $fieldSize );
637 $pos += $fieldSize;
638 break;
639 default:
640 throw new MWException( __METHOD__ . ": invalid type \"$typeName\"" );
641 }
642 } else {
643 // Unsigned little-endian integer
644 $length = intval( $type );
645
646 // Calculate the value. Use an algorithm which automatically
647 // upgrades the value to floating point if necessary.
648 $value = 0;
649 for ( $i = $length - 1; $i >= 0; $i-- ) {
650 $value *= 256;
651 $value += ord( $string[$pos + $i] );
652 }
653
654 // Throw an exception if there was loss of precision
655 if ( $value > pow( 2, 52 ) ) {
656 $this->error( 'zip-unsupported', 'number too large to be stored in a double. ' .
657 'This could happen if we tried to unpack a 64-bit structure ' .
658 'at an invalid location.' );
659 }
660 $data[$key] = $value;
661 $pos += $length;
662 }
663 }
664
665 return $data;
666 }
667
668 /**
669 * Returns a bit from a given position in an integer value, converted to
670 * boolean.
671 *
672 * @param int $value
673 * @param int $bitIndex The index of the bit, where 0 is the LSB.
674 * @return bool
675 */
676 function testBit( $value, $bitIndex ) {
677 return (bool)( ( $value >> $bitIndex ) & 1 );
678 }
679
680 /**
681 * Debugging helper function which dumps a string in hexdump -C format.
682 * @param string $s
683 */
684 function hexDump( $s ) {
685 $n = strlen( $s );
686 for ( $i = 0; $i < $n; $i += 16 ) {
687 printf( "%08X ", $i );
688 for ( $j = 0; $j < 16; $j++ ) {
689 print " ";
690 if ( $j == 8 ) {
691 print " ";
692 }
693 if ( $i + $j >= $n ) {
694 print " ";
695 } else {
696 printf( "%02X", ord( $s[$i + $j] ) );
697 }
698 }
699
700 print " |";
701 for ( $j = 0; $j < 16; $j++ ) {
702 if ( $i + $j >= $n ) {
703 print " ";
704 } elseif ( ctype_print( $s[$i + $j] ) ) {
705 print $s[$i + $j];
706 } else {
707 print '.';
708 }
709 }
710 print "|\n";
711 }
712 }
713 }
714
715 /**
716 * Internal exception class. Will be caught by private code.
717 */
718 class ZipDirectoryReaderError extends Exception {
719 protected $errorCode;
720
721 function __construct( $code ) {
722 $this->errorCode = $code;
723 parent::__construct( "ZipDirectoryReader error: $code" );
724 }
725
726 /**
727 * @return mixed
728 */
729 function getErrorCode() {
730 return $this->errorCode;
731 }
732 }