dépôts / lhc / web / wiklou.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Merge "Add 'contentmodel' to resultproperties in prop=revisions"
[lhc/web/wiklou.git] / includes / filebackend / SwiftFileBackend.php
1 <?php
2 /**
3 * OpenStack Swift based file backend.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 * @author Russ Nelson
23 * @author Aaron Schulz
24 */
25
26 /**
27 * @brief Class for an OpenStack Swift based file backend.
28 *
29 * This requires the SwiftCloudFiles MediaWiki extension, which includes
30 * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
31 * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
32 *
33 * Status messages should avoid mentioning the Swift account name.
34 * Likewise, error suppression should be used to avoid path disclosure.
35 *
36 * @ingroup FileBackend
37 * @since 1.19
38 */
39 class SwiftFileBackend extends FileBackendStore {
40 /** @var CF_Authentication */
41 protected $auth; // Swift authentication handler
42 protected $authTTL; // integer seconds
43 protected $swiftTempUrlKey; // string; shared secret value for making temp urls
44 protected $swiftAnonUser; // string; username to handle unauthenticated requests
45 protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
46 protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
47 protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
48
49 /** @var CF_Connection */
50 protected $conn; // Swift connection handle
51 protected $sessionStarted = 0; // integer UNIX timestamp
52
53 /** @var CloudFilesException */
54 protected $connException;
55 protected $connErrorTime = 0; // UNIX timestamp
56
57 /** @var BagOStuff */
58 protected $srvCache;
59
60 /** @var ProcessCacheLRU */
61 protected $connContainerCache; // container object cache
62
63 /**
64 * @see FileBackendStore::__construct()
65 * Additional $config params include:
66 * - swiftAuthUrl : Swift authentication server URL
67 * - swiftUser : Swift user used by MediaWiki (account:username)
68 * - swiftKey : Swift authentication key for the above user
69 * - swiftAuthTTL : Swift authentication TTL (seconds)
70 * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
71 * Do not set this until it has been set in the backend.
72 * - swiftAnonUser : Swift user used for end-user requests (account:username).
73 * If set, then views of public containers are assumed to go
74 * through this user. If not set, then public containers are
75 * accessible to unauthenticated requests via ".r:*" in the ACL.
76 * - swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up
77 * - swiftCDNExpiry : How long (in seconds) to store content in the CDN.
78 * If files may likely change, this should probably not exceed
79 * a few days. For example, deletions may take this long to apply.
80 * If object purging is enabled, however, this is not an issue.
81 * - swiftCDNPurgable : Whether object purge requests are allowed by the CDN.
82 * - shardViaHashLevels : Map of container names to sharding config with:
83 * - base : base of hash characters, 16 or 36
84 * - levels : the number of hash levels (and digits)
85 * - repeat : hash subdirectories are prefixed with all the
86 * parent hash directory names (e.g. "a/ab/abc")
87 * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
88 * If those are not available, then the main cache will be used.
89 * This is probably insecure in shared hosting environments.
90 */
91 public function __construct( array $config ) {
92 parent::__construct( $config );
93 if ( !MWInit::classExists( 'CF_Constants' ) ) {
94 throw new MWException( 'SwiftCloudFiles extension not installed.' );
95 }
96 // Required settings
97 $this->auth = new CF_Authentication(
98 $config['swiftUser'],
99 $config['swiftKey'],
100 null, // account; unused
101 $config['swiftAuthUrl']
102 );
103 // Optional settings
104 $this->authTTL = isset( $config['swiftAuthTTL'] )
105 ? $config['swiftAuthTTL']
106 : 5 * 60; // some sane number
107 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
108 ? $config['swiftAnonUser']
109 : '';
110 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
111 ? $config['swiftTempUrlKey']
112 : '';
113 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
114 ? $config['shardViaHashLevels']
115 : '';
116 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
117 ? $config['swiftUseCDN']
118 : false;
119 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
120 ? $config['swiftCDNExpiry']
121 : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
122 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
123 ? $config['swiftCDNPurgable']
124 : true;
125 // Cache container information to mask latency
126 $this->memCache = wfGetMainCache();
127 // Process cache for container info
128 $this->connContainerCache = new ProcessCacheLRU( 300 );
129 // Cache auth token information to avoid RTTs
130 if ( !empty( $config['cacheAuthInfo'] ) ) {
131 if ( php_sapi_name() === 'cli' ) {
132 $this->srvCache = wfGetMainCache(); // preferrably memcached
133 } else {
134 try { // look for APC, XCache, WinCache, ect...
135 $this->srvCache = ObjectCache::newAccelerator( array() );
136 } catch ( Exception $e ) {}
137 }
138 }
139 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
140 }
141
142 /**
143 * @see FileBackendStore::resolveContainerPath()
144 * @return null
145 */
146 protected function resolveContainerPath( $container, $relStoragePath ) {
147 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
148 return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
149 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
150 return null; // too long for Swift
151 }
152 return $relStoragePath;
153 }
154
155 /**
156 * @see FileBackendStore::isPathUsableInternal()
157 * @return bool
158 */
159 public function isPathUsableInternal( $storagePath ) {
160 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
161 if ( $rel === null ) {
162 return false; // invalid
163 }
164
165 try {
166 $this->getContainer( $container );
167 return true; // container exists
168 } catch ( NoSuchContainerException $e ) {
169 } catch ( CloudFilesException $e ) { // some other exception?
170 $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
171 }
172
173 return false;
174 }
175
176 /**
177 * @param $disposition string Content-Disposition header value
178 * @return string Truncated Content-Disposition header value to meet Swift limits
179 */
180 protected function truncDisp( $disposition ) {
181 $res = '';
182 foreach ( explode( ';', $disposition ) as $part ) {
183 $part = trim( $part );
184 $new = ( $res === '' ) ? $part : "{$res};{$part}";
185 if ( strlen( $new ) <= 255 ) {
186 $res = $new;
187 } else {
188 break; // too long; sigh
189 }
190 }
191 return $res;
192 }
193
194 /**
195 * @see FileBackendStore::doCreateInternal()
196 * @return Status
197 */
198 protected function doCreateInternal( array $params ) {
199 $status = Status::newGood();
200
201 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
202 if ( $dstRel === null ) {
203 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
204 return $status;
205 }
206
207 // (a) Check the destination container and object
208 try {
209 $dContObj = $this->getContainer( $dstCont );
210 } catch ( NoSuchContainerException $e ) {
211 $status->fatal( 'backend-fail-create', $params['dst'] );
212 return $status;
213 } catch ( CloudFilesException $e ) { // some other exception?
214 $this->handleException( $e, $status, __METHOD__, $params );
215 return $status;
216 }
217
218 // (b) Get a SHA-1 hash of the object
219 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
220
221 // (c) Actually create the object
222 try {
223 // Create a fresh CF_Object with no fields preloaded.
224 // We don't want to preserve headers, metadata, and such.
225 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
226 $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
227 // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
228 // The MD5 here will be checked within Swift against its own MD5.
229 $obj->set_etag( md5( $params['content'] ) );
230 // Use the same content type as StreamFile for security
231 $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
232 if ( !strlen( $obj->content_type ) ) { // special case
233 $obj->content_type = 'unknown/unknown';
234 }
235 // Set the Content-Disposition header if requested
236 if ( isset( $params['disposition'] ) ) {
237 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
238 }
239 if ( !empty( $params['async'] ) ) { // deferred
240 $op = $obj->write_async( $params['content'] );
241 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
242 $status->value->affectedObjects[] = $obj;
243 } else { // actually write the object in Swift
244 $obj->write( $params['content'] );
245 $this->purgeCDNCache( array( $obj ) );
246 }
247 } catch ( CDNNotEnabledException $e ) {
248 // CDN not enabled; nothing to see here
249 } catch ( BadContentTypeException $e ) {
250 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
251 } catch ( CloudFilesException $e ) { // some other exception?
252 $this->handleException( $e, $status, __METHOD__, $params );
253 }
254
255 return $status;
256 }
257
258 /**
259 * @see SwiftFileBackend::doExecuteOpHandlesInternal()
260 */
261 protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
262 try {
263 $cfOp->getLastResponse();
264 } catch ( BadContentTypeException $e ) {
265 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
266 }
267 }
268
269 /**
270 * @see FileBackendStore::doStoreInternal()
271 * @return Status
272 */
273 protected function doStoreInternal( array $params ) {
274 $status = Status::newGood();
275
276 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
277 if ( $dstRel === null ) {
278 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
279 return $status;
280 }
281
282 // (a) Check the destination container and object
283 try {
284 $dContObj = $this->getContainer( $dstCont );
285 } catch ( NoSuchContainerException $e ) {
286 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
287 return $status;
288 } catch ( CloudFilesException $e ) { // some other exception?
289 $this->handleException( $e, $status, __METHOD__, $params );
290 return $status;
291 }
292
293 // (b) Get a SHA-1 hash of the object
294 $sha1Hash = sha1_file( $params['src'] );
295 if ( $sha1Hash === false ) { // source doesn't exist?
296 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
297 return $status;
298 }
299 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
300
301 // (c) Actually store the object
302 try {
303 // Create a fresh CF_Object with no fields preloaded.
304 // We don't want to preserve headers, metadata, and such.
305 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
306 $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
307 // The MD5 here will be checked within Swift against its own MD5.
308 $obj->set_etag( md5_file( $params['src'] ) );
309 // Use the same content type as StreamFile for security
310 $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
311 if ( !strlen( $obj->content_type ) ) { // special case
312 $obj->content_type = 'unknown/unknown';
313 }
314 // Set the Content-Disposition header if requested
315 if ( isset( $params['disposition'] ) ) {
316 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
317 }
318 if ( !empty( $params['async'] ) ) { // deferred
319 wfSuppressWarnings();
320 $fp = fopen( $params['src'], 'rb' );
321 wfRestoreWarnings();
322 if ( !$fp ) {
323 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
324 } else {
325 $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
326 $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
327 $status->value->resourcesToClose[] = $fp;
328 $status->value->affectedObjects[] = $obj;
329 }
330 } else { // actually write the object in Swift
331 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
332 $this->purgeCDNCache( array( $obj ) );
333 }
334 } catch ( CDNNotEnabledException $e ) {
335 // CDN not enabled; nothing to see here
336 } catch ( BadContentTypeException $e ) {
337 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
338 } catch ( IOException $e ) {
339 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
340 } catch ( CloudFilesException $e ) { // some other exception?
341 $this->handleException( $e, $status, __METHOD__, $params );
342 }
343
344 return $status;
345 }
346
347 /**
348 * @see SwiftFileBackend::doExecuteOpHandlesInternal()
349 */
350 protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
351 try {
352 $cfOp->getLastResponse();
353 } catch ( BadContentTypeException $e ) {
354 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
355 } catch ( IOException $e ) {
356 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
357 }
358 }
359
360 /**
361 * @see FileBackendStore::doCopyInternal()
362 * @return Status
363 */
364 protected function doCopyInternal( array $params ) {
365 $status = Status::newGood();
366
367 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
368 if ( $srcRel === null ) {
369 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
370 return $status;
371 }
372
373 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
374 if ( $dstRel === null ) {
375 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
376 return $status;
377 }
378
379 // (a) Check the source/destination containers and destination object
380 try {
381 $sContObj = $this->getContainer( $srcCont );
382 $dContObj = $this->getContainer( $dstCont );
383 } catch ( NoSuchContainerException $e ) {
384 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
385 return $status;
386 } catch ( CloudFilesException $e ) { // some other exception?
387 $this->handleException( $e, $status, __METHOD__, $params );
388 return $status;
389 }
390
391 // (b) Actually copy the file to the destination
392 try {
393 $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
394 $hdrs = array(); // source file headers to override with new values
395 if ( isset( $params['disposition'] ) ) {
396 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
397 }
398 if ( !empty( $params['async'] ) ) { // deferred
399 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
400 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
401 $status->value->affectedObjects[] = $dstObj;
402 } else { // actually write the object in Swift
403 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
404 $this->purgeCDNCache( array( $dstObj ) );
405 }
406 } catch ( CDNNotEnabledException $e ) {
407 // CDN not enabled; nothing to see here
408 } catch ( NoSuchObjectException $e ) { // source object does not exist
409 if ( empty( $params['ignoreMissingSource'] ) ) {
410 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
411 }
412 } catch ( CloudFilesException $e ) { // some other exception?
413 $this->handleException( $e, $status, __METHOD__, $params );
414 }
415
416 return $status;
417 }
418
419 /**
420 * @see SwiftFileBackend::doExecuteOpHandlesInternal()
421 */
422 protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
423 try {
424 $cfOp->getLastResponse();
425 } catch ( NoSuchObjectException $e ) { // source object does not exist
426 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
427 }
428 }
429
430 /**
431 * @see FileBackendStore::doMoveInternal()
432 * @return Status
433 */
434 protected function doMoveInternal( array $params ) {
435 $status = Status::newGood();
436
437 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
438 if ( $srcRel === null ) {
439 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
440 return $status;
441 }
442
443 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
444 if ( $dstRel === null ) {
445 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
446 return $status;
447 }
448
449 // (a) Check the source/destination containers and destination object
450 try {
451 $sContObj = $this->getContainer( $srcCont );
452 $dContObj = $this->getContainer( $dstCont );
453 } catch ( NoSuchContainerException $e ) {
454 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
455 return $status;
456 } catch ( CloudFilesException $e ) { // some other exception?
457 $this->handleException( $e, $status, __METHOD__, $params );
458 return $status;
459 }
460
461 // (b) Actually move the file to the destination
462 try {
463 $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
464 $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
465 $hdrs = array(); // source file headers to override with new values
466 if ( isset( $params['disposition'] ) ) {
467 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
468 }
469 if ( !empty( $params['async'] ) ) { // deferred
470 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
471 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
472 $status->value->affectedObjects[] = $srcObj;
473 $status->value->affectedObjects[] = $dstObj;
474 } else { // actually write the object in Swift
475 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
476 $this->purgeCDNCache( array( $srcObj ) );
477 $this->purgeCDNCache( array( $dstObj ) );
478 }
479 } catch ( CDNNotEnabledException $e ) {
480 // CDN not enabled; nothing to see here
481 } catch ( NoSuchObjectException $e ) { // source object does not exist
482 if ( empty( $params['ignoreMissingSource'] ) ) {
483 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
484 }
485 } catch ( CloudFilesException $e ) { // some other exception?
486 $this->handleException( $e, $status, __METHOD__, $params );
487 }
488
489 return $status;
490 }
491
492 /**
493 * @see SwiftFileBackend::doExecuteOpHandlesInternal()
494 */
495 protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
496 try {
497 $cfOp->getLastResponse();
498 } catch ( NoSuchObjectException $e ) { // source object does not exist
499 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
500 }
501 }
502
503 /**
504 * @see FileBackendStore::doDeleteInternal()
505 * @return Status
506 */
507 protected function doDeleteInternal( array $params ) {
508 $status = Status::newGood();
509
510 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
511 if ( $srcRel === null ) {
512 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
513 return $status;
514 }
515
516 try {
517 $sContObj = $this->getContainer( $srcCont );
518 $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
519 if ( !empty( $params['async'] ) ) { // deferred
520 $op = $sContObj->delete_object_async( $srcRel );
521 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
522 $status->value->affectedObjects[] = $srcObj;
523 } else { // actually write the object in Swift
524 $sContObj->delete_object( $srcRel );
525 $this->purgeCDNCache( array( $srcObj ) );
526 }
527 } catch ( CDNNotEnabledException $e ) {
528 // CDN not enabled; nothing to see here
529 } catch ( NoSuchContainerException $e ) {
530 $status->fatal( 'backend-fail-delete', $params['src'] );
531 } catch ( NoSuchObjectException $e ) {
532 if ( empty( $params['ignoreMissingSource'] ) ) {
533 $status->fatal( 'backend-fail-delete', $params['src'] );
534 }
535 } catch ( CloudFilesException $e ) { // some other exception?
536 $this->handleException( $e, $status, __METHOD__, $params );
537 }
538
539 return $status;
540 }
541
542 /**
543 * @see SwiftFileBackend::doExecuteOpHandlesInternal()
544 */
545 protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
546 try {
547 $cfOp->getLastResponse();
548 } catch ( NoSuchContainerException $e ) {
549 $status->fatal( 'backend-fail-delete', $params['src'] );
550 } catch ( NoSuchObjectException $e ) {
551 if ( empty( $params['ignoreMissingSource'] ) ) {
552 $status->fatal( 'backend-fail-delete', $params['src'] );
553 }
554 }
555 }
556
557 /**
558 * @see FileBackendStore::doPrepareInternal()
559 * @return Status
560 */
561 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
562 $status = Status::newGood();
563
564 // (a) Check if container already exists
565 try {
566 $contObj = $this->getContainer( $fullCont );
567 // NoSuchContainerException not thrown: container must exist
568 return $status; // already exists
569 } catch ( NoSuchContainerException $e ) {
570 // NoSuchContainerException thrown: container does not exist
571 } catch ( CloudFilesException $e ) { // some other exception?
572 $this->handleException( $e, $status, __METHOD__, $params );
573 return $status;
574 }
575
576 // (b) Create container as needed
577 try {
578 $contObj = $this->createContainer( $fullCont );
579 if ( !empty( $params['noAccess'] ) ) {
580 // Make container private to end-users...
581 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
582 } else {
583 // Make container public to end-users...
584 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
585 }
586 if ( $this->swiftUseCDN ) { // Rackspace style CDN
587 $contObj->make_public( $this->swiftCDNExpiry );
588 }
589 } catch ( CDNNotEnabledException $e ) {
590 // CDN not enabled; nothing to see here
591 } catch ( CloudFilesException $e ) { // some other exception?
592 $this->handleException( $e, $status, __METHOD__, $params );
593 return $status;
594 }
595
596 return $status;
597 }
598
599 /**
600 * @see FileBackendStore::doSecureInternal()
601 * @return Status
602 */
603 protected function doSecureInternal( $fullCont, $dir, array $params ) {
604 $status = Status::newGood();
605 if ( empty( $params['noAccess'] ) ) {
606 return $status; // nothing to do
607 }
608
609 // Restrict container from end-users...
610 try {
611 // doPrepareInternal() should have been called,
612 // so the Swift container should already exist...
613 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
614 // NoSuchContainerException not thrown: container must exist
615
616 // Make container private to end-users...
617 $status->merge( $this->setContainerAccess(
618 $contObj,
619 array( $this->auth->username ), // read
620 array( $this->auth->username ) // write
621 ) );
622 if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
623 $contObj->make_private();
624 }
625 } catch ( CDNNotEnabledException $e ) {
626 // CDN not enabled; nothing to see here
627 } catch ( CloudFilesException $e ) { // some other exception?
628 $this->handleException( $e, $status, __METHOD__, $params );
629 }
630
631 return $status;
632 }
633
634 /**
635 * @see FileBackendStore::doPublishInternal()
636 * @return Status
637 */
638 protected function doPublishInternal( $fullCont, $dir, array $params ) {
639 $status = Status::newGood();
640
641 // Unrestrict container from end-users...
642 try {
643 // doPrepareInternal() should have been called,
644 // so the Swift container should already exist...
645 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
646 // NoSuchContainerException not thrown: container must exist
647
648 // Make container public to end-users...
649 if ( $this->swiftAnonUser != '' ) {
650 $status->merge( $this->setContainerAccess(
651 $contObj,
652 array( $this->auth->username, $this->swiftAnonUser ), // read
653 array( $this->auth->username, $this->swiftAnonUser ) // write
654 ) );
655 } else {
656 $status->merge( $this->setContainerAccess(
657 $contObj,
658 array( $this->auth->username, '.r:*' ), // read
659 array( $this->auth->username ) // write
660 ) );
661 }
662 if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
663 $contObj->make_public();
664 }
665 } catch ( CDNNotEnabledException $e ) {
666 // CDN not enabled; nothing to see here
667 } catch ( CloudFilesException $e ) { // some other exception?
668 $this->handleException( $e, $status, __METHOD__, $params );
669 }
670
671 return $status;
672 }
673
674 /**
675 * @see FileBackendStore::doCleanInternal()
676 * @return Status
677 */
678 protected function doCleanInternal( $fullCont, $dir, array $params ) {
679 $status = Status::newGood();
680
681 // Only containers themselves can be removed, all else is virtual
682 if ( $dir != '' ) {
683 return $status; // nothing to do
684 }
685
686 // (a) Check the container
687 try {
688 $contObj = $this->getContainer( $fullCont, true );
689 } catch ( NoSuchContainerException $e ) {
690 return $status; // ok, nothing to do
691 } catch ( CloudFilesException $e ) { // some other exception?
692 $this->handleException( $e, $status, __METHOD__, $params );
693 return $status;
694 }
695
696 // (b) Delete the container if empty
697 if ( $contObj->object_count == 0 ) {
698 try {
699 $this->deleteContainer( $fullCont );
700 } catch ( NoSuchContainerException $e ) {
701 return $status; // race?
702 } catch ( NonEmptyContainerException $e ) {
703 return $status; // race? consistency delay?
704 } catch ( CloudFilesException $e ) { // some other exception?
705 $this->handleException( $e, $status, __METHOD__, $params );
706 return $status;
707 }
708 }
709
710 return $status;
711 }
712
713 /**
714 * @see FileBackendStore::doFileExists()
715 * @return array|bool|null
716 */
717 protected function doGetFileStat( array $params ) {
718 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
719 if ( $srcRel === null ) {
720 return false; // invalid storage path
721 }
722
723 $stat = false;
724 try {
725 $contObj = $this->getContainer( $srcCont );
726 $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
727 $this->addMissingMetadata( $srcObj, $params['src'] );
728 $stat = array(
729 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
730 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
731 'size' => (int)$srcObj->content_length,
732 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
733 );
734 } catch ( NoSuchContainerException $e ) {
735 } catch ( NoSuchObjectException $e ) {
736 } catch ( CloudFilesException $e ) { // some other exception?
737 $stat = null;
738 $this->handleException( $e, null, __METHOD__, $params );
739 }
740
741 return $stat;
742 }
743
744 /**
745 * Fill in any missing object metadata and save it to Swift
746 *
747 * @param $obj CF_Object
748 * @param $path string Storage path to object
749 * @return bool Success
750 * @throws Exception cloudfiles exceptions
751 */
752 protected function addMissingMetadata( CF_Object $obj, $path ) {
753 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
754 return true; // nothing to do
755 }
756 wfProfileIn( __METHOD__ );
757 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
758 $status = Status::newGood();
759 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
760 if ( $status->isOK() ) {
761 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
762 if ( $tmpFile ) {
763 $hash = $tmpFile->getSha1Base36();
764 if ( $hash !== false ) {
765 $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
766 $obj->sync_metadata(); // save to Swift
767 wfProfileOut( __METHOD__ );
768 return true; // success
769 }
770 }
771 }
772 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
773 wfProfileOut( __METHOD__ );
774 return false; // failed
775 }
776
777 /**
778 * @see FileBackendStore::doGetFileContentsMulti()
779 * @return Array
780 */
781 protected function doGetFileContentsMulti( array $params ) {
782 $contents = array();
783
784 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
785 // Blindly create tmp files and stream to them, catching any exception if the file does
786 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
787 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
788 $cfOps = array(); // (path => CF_Async_Op)
789
790 foreach ( $pathBatch as $path ) { // each path in this concurrent batch
791 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
792 if ( $srcRel === null ) {
793 $contents[$path] = false;
794 continue;
795 }
796 $data = false;
797 try {
798 $sContObj = $this->getContainer( $srcCont );
799 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
800 // Get source file extension
801 $ext = FileBackend::extensionFromPath( $path );
802 // Create a new temporary memory file...
803 $handle = fopen( 'php://temp', 'wb' );
804 if ( $handle ) {
805 $headers = $this->headersFromParams( $params );
806 if ( count( $pathBatch ) > 1 ) {
807 $cfOps[$path] = $obj->stream_async( $handle, $headers );
808 $cfOps[$path]->_file_handle = $handle; // close this later
809 } else {
810 $obj->stream( $handle, $headers );
811 rewind( $handle ); // start from the beginning
812 $data = stream_get_contents( $handle );
813 fclose( $handle );
814 }
815 } else {
816 $data = false;
817 }
818 } catch ( NoSuchContainerException $e ) {
819 $data = false;
820 } catch ( NoSuchObjectException $e ) {
821 $data = false;
822 } catch ( CloudFilesException $e ) { // some other exception?
823 $data = false;
824 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
825 }
826 $contents[$path] = $data;
827 }
828
829 $batch = new CF_Async_Op_Batch( $cfOps );
830 $cfOps = $batch->execute();
831 foreach ( $cfOps as $path => $cfOp ) {
832 try {
833 $cfOp->getLastResponse();
834 rewind( $cfOp->_file_handle ); // start from the beginning
835 $contents[$path] = stream_get_contents( $cfOp->_file_handle );
836 } catch ( NoSuchContainerException $e ) {
837 $contents[$path] = false;
838 } catch ( NoSuchObjectException $e ) {
839 $contents[$path] = false;
840 } catch ( CloudFilesException $e ) { // some other exception?
841 $contents[$path] = false;
842 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
843 }
844 fclose( $cfOp->_file_handle ); // close open handle
845 }
846 }
847
848 return $contents;
849 }
850
851 /**
852 * @see FileBackendStore::doDirectoryExists()
853 * @return bool|null
854 */
855 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
856 try {
857 $container = $this->getContainer( $fullCont );
858 $prefix = ( $dir == '' ) ? null : "{$dir}/";
859 return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
860 } catch ( NoSuchContainerException $e ) {
861 return false;
862 } catch ( CloudFilesException $e ) { // some other exception?
863 $this->handleException( $e, null, __METHOD__,
864 array( 'cont' => $fullCont, 'dir' => $dir ) );
865 }
866
867 return null; // error
868 }
869
870 /**
871 * @see FileBackendStore::getDirectoryListInternal()
872 * @return SwiftFileBackendDirList
873 */
874 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
875 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
876 }
877
878 /**
879 * @see FileBackendStore::getFileListInternal()
880 * @return SwiftFileBackendFileList
881 */
882 public function getFileListInternal( $fullCont, $dir, array $params ) {
883 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
884 }
885
886 /**
887 * Do not call this function outside of SwiftFileBackendFileList
888 *
889 * @param $fullCont string Resolved container name
890 * @param $dir string Resolved storage directory with no trailing slash
891 * @param $after string|null Storage path of file to list items after
892 * @param $limit integer Max number of items to list
893 * @param $params Array Includes flag for 'topOnly'
894 * @return Array List of relative paths of dirs directly under $dir
895 */
896 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
897 $dirs = array();
898 if ( $after === INF ) {
899 return $dirs; // nothing more
900 }
901 wfProfileIn( __METHOD__ . '-' . $this->name );
902
903 try {
904 $container = $this->getContainer( $fullCont );
905 $prefix = ( $dir == '' ) ? null : "{$dir}/";
906 // Non-recursive: only list dirs right under $dir
907 if ( !empty( $params['topOnly'] ) ) {
908 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
909 foreach ( $objects as $object ) { // files and dirs
910 if ( substr( $object, -1 ) === '/' ) {
911 $dirs[] = $object; // directories end in '/'
912 }
913 }
914 // Recursive: list all dirs under $dir and its subdirs
915 } else {
916 // Get directory from last item of prior page
917 $lastDir = $this->getParentDir( $after ); // must be first page
918 $objects = $container->list_objects( $limit, $after, $prefix );
919 foreach ( $objects as $object ) { // files
920 $objectDir = $this->getParentDir( $object ); // directory of object
921 if ( $objectDir !== false ) { // file has a parent dir
922 // Swift stores paths in UTF-8, using binary sorting.
923 // See function "create_container_table" in common/db.py.
924 // If a directory is not "greater" than the last one,
925 // then it was already listed by the calling iterator.
926 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
927 $pDir = $objectDir;
928 do { // add dir and all its parent dirs
929 $dirs[] = "{$pDir}/";
930 $pDir = $this->getParentDir( $pDir );
931 } while ( $pDir !== false // sanity
932 && strcmp( $pDir, $lastDir ) > 0 // not done already
933 && strlen( $pDir ) > strlen( $dir ) // within $dir
934 );
935 }
936 $lastDir = $objectDir;
937 }
938 }
939 }
940 if ( count( $objects ) < $limit ) {
941 $after = INF; // avoid a second RTT
942 } else {
943 $after = end( $objects ); // update last item
944 }
945 } catch ( NoSuchContainerException $e ) {
946 } catch ( CloudFilesException $e ) { // some other exception?
947 $this->handleException( $e, null, __METHOD__,
948 array( 'cont' => $fullCont, 'dir' => $dir ) );
949 }
950
951 wfProfileOut( __METHOD__ . '-' . $this->name );
952 return $dirs;
953 }
954
955 protected function getParentDir( $path ) {
956 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
957 }
958
959 /**
960 * Do not call this function outside of SwiftFileBackendFileList
961 *
962 * @param $fullCont string Resolved container name
963 * @param $dir string Resolved storage directory with no trailing slash
964 * @param $after string|null Storage path of file to list items after
965 * @param $limit integer Max number of items to list
966 * @param $params Array Includes flag for 'topOnly'
967 * @return Array List of relative paths of files under $dir
968 */
969 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
970 $files = array();
971 if ( $after === INF ) {
972 return $files; // nothing more
973 }
974 wfProfileIn( __METHOD__ . '-' . $this->name );
975
976 try {
977 $container = $this->getContainer( $fullCont );
978 $prefix = ( $dir == '' ) ? null : "{$dir}/";
979 // Non-recursive: only list files right under $dir
980 if ( !empty( $params['topOnly'] ) ) { // files and dirs
981 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
982 foreach ( $objects as $object ) {
983 if ( substr( $object, -1 ) !== '/' ) {
984 $files[] = $object; // directories end in '/'
985 }
986 }
987 // Recursive: list all files under $dir and its subdirs
988 } else { // files
989 $objects = $container->list_objects( $limit, $after, $prefix );
990 $files = $objects;
991 }
992 if ( count( $objects ) < $limit ) {
993 $after = INF; // avoid a second RTT
994 } else {
995 $after = end( $objects ); // update last item
996 }
997 } catch ( NoSuchContainerException $e ) {
998 } catch ( CloudFilesException $e ) { // some other exception?
999 $this->handleException( $e, null, __METHOD__,
1000 array( 'cont' => $fullCont, 'dir' => $dir ) );
1001 }
1002
1003 wfProfileOut( __METHOD__ . '-' . $this->name );
1004 return $files;
1005 }
1006
1007 /**
1008 * @see FileBackendStore::doGetFileSha1base36()
1009 * @return bool
1010 */
1011 protected function doGetFileSha1base36( array $params ) {
1012 $stat = $this->getFileStat( $params );
1013 if ( $stat ) {
1014 return $stat['sha1'];
1015 } else {
1016 return false;
1017 }
1018 }
1019
1020 /**
1021 * @see FileBackendStore::doStreamFile()
1022 * @return Status
1023 */
1024 protected function doStreamFile( array $params ) {
1025 $status = Status::newGood();
1026
1027 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1028 if ( $srcRel === null ) {
1029 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1030 }
1031
1032 try {
1033 $cont = $this->getContainer( $srcCont );
1034 } catch ( NoSuchContainerException $e ) {
1035 $status->fatal( 'backend-fail-stream', $params['src'] );
1036 return $status;
1037 } catch ( CloudFilesException $e ) { // some other exception?
1038 $this->handleException( $e, $status, __METHOD__, $params );
1039 return $status;
1040 }
1041
1042 try {
1043 $output = fopen( 'php://output', 'wb' );
1044 $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1045 $obj->stream( $output, $this->headersFromParams( $params ) );
1046 } catch ( NoSuchObjectException $e ) {
1047 $status->fatal( 'backend-fail-stream', $params['src'] );
1048 } catch ( CloudFilesException $e ) { // some other exception?
1049 $this->handleException( $e, $status, __METHOD__, $params );
1050 }
1051
1052 return $status;
1053 }
1054
1055 /**
1056 * @see FileBackendStore::doGetLocalCopyMulti()
1057 * @return null|TempFSFile
1058 */
1059 protected function doGetLocalCopyMulti( array $params ) {
1060 $tmpFiles = array();
1061
1062 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1063 // Blindly create tmp files and stream to them, catching any exception if the file does
1064 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1065 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1066 $cfOps = array(); // (path => CF_Async_Op)
1067
1068 foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1069 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1070 if ( $srcRel === null ) {
1071 $tmpFiles[$path] = null;
1072 continue;
1073 }
1074 $tmpFile = null;
1075 try {
1076 $sContObj = $this->getContainer( $srcCont );
1077 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1078 // Get source file extension
1079 $ext = FileBackend::extensionFromPath( $path );
1080 // Create a new temporary file...
1081 $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1082 if ( $tmpFile ) {
1083 $handle = fopen( $tmpFile->getPath(), 'wb' );
1084 if ( $handle ) {
1085 $headers = $this->headersFromParams( $params );
1086 if ( count( $pathBatch ) > 1 ) {
1087 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1088 $cfOps[$path]->_file_handle = $handle; // close this later
1089 } else {
1090 $obj->stream( $handle, $headers );
1091 fclose( $handle );
1092 }
1093 } else {
1094 $tmpFile = null;
1095 }
1096 }
1097 } catch ( NoSuchContainerException $e ) {
1098 $tmpFile = null;
1099 } catch ( NoSuchObjectException $e ) {
1100 $tmpFile = null;
1101 } catch ( CloudFilesException $e ) { // some other exception?
1102 $tmpFile = null;
1103 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1104 }
1105 $tmpFiles[$path] = $tmpFile;
1106 }
1107
1108 $batch = new CF_Async_Op_Batch( $cfOps );
1109 $cfOps = $batch->execute();
1110 foreach ( $cfOps as $path => $cfOp ) {
1111 try {
1112 $cfOp->getLastResponse();
1113 } catch ( NoSuchContainerException $e ) {
1114 $tmpFiles[$path] = null;
1115 } catch ( NoSuchObjectException $e ) {
1116 $tmpFiles[$path] = null;
1117 } catch ( CloudFilesException $e ) { // some other exception?
1118 $tmpFiles[$path] = null;
1119 $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1120 }
1121 fclose( $cfOp->_file_handle ); // close open handle
1122 }
1123 }
1124
1125 return $tmpFiles;
1126 }
1127
1128 /**
1129 * @see FileBackendStore::getFileHttpUrl()
1130 * @return string|null
1131 */
1132 public function getFileHttpUrl( array $params ) {
1133 if ( $this->swiftTempUrlKey != '' ) { // temp urls enabled
1134 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1135 if ( $srcRel === null ) {
1136 return null; // invalid path
1137 }
1138 try {
1139 $sContObj = $this->getContainer( $srcCont );
1140 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1141 return $obj->get_temp_url( $this->swiftTempUrlKey, 86400, "GET" );
1142 } catch ( NoSuchContainerException $e ) {
1143 } catch ( CloudFilesException $e ) { // some other exception?
1144 $this->handleException( $e, null, __METHOD__, $params );
1145 }
1146 }
1147 return null;
1148 }
1149
1150 /**
1151 * @see FileBackendStore::directoriesAreVirtual()
1152 * @return bool
1153 */
1154 protected function directoriesAreVirtual() {
1155 return true;
1156 }
1157
1158 /**
1159 * Get headers to send to Swift when reading a file based
1160 * on a FileBackend params array, e.g. that of getLocalCopy().
1161 * $params is currently only checked for a 'latest' flag.
1162 *
1163 * @param $params Array
1164 * @return Array
1165 */
1166 protected function headersFromParams( array $params ) {
1167 $hdrs = array();
1168 if ( !empty( $params['latest'] ) ) {
1169 $hdrs[] = 'X-Newest: true';
1170 }
1171 return $hdrs;
1172 }
1173
1174 /**
1175 * @see FileBackendStore::doExecuteOpHandlesInternal()
1176 * @return Array List of corresponding Status objects
1177 */
1178 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1179 $statuses = array();
1180
1181 $cfOps = array(); // list of CF_Async_Op objects
1182 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1183 $cfOps[$index] = $fileOpHandle->cfOp;
1184 }
1185 $batch = new CF_Async_Op_Batch( $cfOps );
1186
1187 $cfOps = $batch->execute();
1188 foreach ( $cfOps as $index => $cfOp ) {
1189 $status = Status::newGood();
1190 try { // catch exceptions; update status
1191 $function = '_getResponse' . $fileOpHandles[$index]->call;
1192 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1193 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1194 } catch ( CloudFilesException $e ) { // some other exception?
1195 $this->handleException( $e, $status,
1196 __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1197 }
1198 $statuses[$index] = $status;
1199 }
1200
1201 return $statuses;
1202 }
1203
1204 /**
1205 * Set read/write permissions for a Swift container.
1206 *
1207 * $readGrps is a list of the possible criteria for a request to have
1208 * access to read a container. Each item is one of the following formats:
1209 * - account:user : Grants access if the request is by the given user
1210 * - ".r:<regex>" : Grants access if the request is from a referrer host that
1211 * matches the expression and the request is not for a listing.
1212 * Setting this to '*' effectively makes a container public.
1213 * -".rlistings:<regex>": Grants access if the request is from a referrer host that
1214 * matches the expression and the request for a listing.
1215 *
1216 * $writeGrps is a list of the possible criteria for a request to have
1217 * access to write to a container. Each item is of the following format:
1218 * - account:user : Grants access if the request is by the given user
1219 *
1220 * @see http://swift.openstack.org/misc.html#acls
1221 *
1222 * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1223 * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1224 *
1225 * @param $contObj CF_Container Swift container
1226 * @param $readGrps Array List of read access routes
1227 * @param $writeGrps Array List of write access routes
1228 * @return Status
1229 */
1230 protected function setContainerAccess(
1231 CF_Container $contObj, array $readGrps, array $writeGrps
1232 ) {
1233 $creds = $contObj->cfs_auth->export_credentials();
1234
1235 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1236
1237 // Note: 10 second timeout consistent with php-cloudfiles
1238 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1239 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1240 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1241 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1242
1243 return $req->execute(); // should return 204
1244 }
1245
1246 /**
1247 * Purge the CDN cache of affected objects if CDN caching is enabled.
1248 * This is for Rackspace/Akamai CDNs.
1249 *
1250 * @param $objects Array List of CF_Object items
1251 * @return void
1252 */
1253 public function purgeCDNCache( array $objects ) {
1254 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1255 foreach ( $objects as $object ) {
1256 try {
1257 $object->purge_from_cdn();
1258 } catch ( CDNNotEnabledException $e ) {
1259 // CDN not enabled; nothing to see here
1260 } catch ( CloudFilesException $e ) {
1261 $this->handleException( $e, null, __METHOD__,
1262 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1263 }
1264 }
1265 }
1266 }
1267
1268 /**
1269 * Get an authenticated connection handle to the Swift proxy
1270 *
1271 * @return CF_Connection|bool False on failure
1272 * @throws CloudFilesException
1273 */
1274 protected function getConnection() {
1275 if ( $this->connException instanceof CloudFilesException ) {
1276 if ( ( time() - $this->connErrorTime ) < 60 ) {
1277 throw $this->connException; // failed last attempt; don't bother
1278 } else { // actually retry this time
1279 $this->connException = null;
1280 $this->connErrorTime = 0;
1281 }
1282 }
1283 // Session keys expire after a while, so we renew them periodically
1284 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1285 // Authenticate with proxy and get a session key...
1286 if ( !$this->conn || $reAuth ) {
1287 $this->sessionStarted = 0;
1288 $this->connContainerCache->clear();
1289 $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1290 $creds = $this->srvCache->get( $cacheKey ); // credentials
1291 if ( is_array( $creds ) ) { // cache hit
1292 $this->auth->load_cached_credentials(
1293 $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1294 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1295 } else { // cache miss
1296 try {
1297 $this->auth->authenticate();
1298 $creds = $this->auth->export_credentials();
1299 $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1300 $this->sessionStarted = time();
1301 } catch ( CloudFilesException $e ) {
1302 $this->connException = $e; // don't keep re-trying
1303 $this->connErrorTime = time();
1304 throw $e; // throw it back
1305 }
1306 }
1307 if ( $this->conn ) { // re-authorizing?
1308 $this->conn->close(); // close active cURL handles in CF_Http object
1309 }
1310 $this->conn = new CF_Connection( $this->auth );
1311 }
1312 return $this->conn;
1313 }
1314
1315 /**
1316 * Close the connection to the Swift proxy
1317 *
1318 * @return void
1319 */
1320 protected function closeConnection() {
1321 if ( $this->conn ) {
1322 $this->conn->close(); // close active cURL handles in CF_Http object
1323 $this->sessionStarted = 0;
1324 $this->connContainerCache->clear();
1325 }
1326 }
1327
1328 /**
1329 * Get the cache key for a container
1330 *
1331 * @param $username string
1332 * @return string
1333 */
1334 private function getCredsCacheKey( $username ) {
1335 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1336 }
1337
1338 /**
1339 * Get a Swift container object, possibly from process cache.
1340 * Use $reCache if the file count or byte count is needed.
1341 *
1342 * @param $container string Container name
1343 * @param $bypassCache bool Bypass all caches and load from Swift
1344 * @return CF_Container
1345 * @throws CloudFilesException
1346 */
1347 protected function getContainer( $container, $bypassCache = false ) {
1348 $conn = $this->getConnection(); // Swift proxy connection
1349 if ( $bypassCache ) { // purge cache
1350 $this->connContainerCache->clear( $container );
1351 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1352 $this->primeContainerCache( array( $container ) ); // check persistent cache
1353 }
1354 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1355 $contObj = $conn->get_container( $container );
1356 // NoSuchContainerException not thrown: container must exist
1357 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1358 if ( !$bypassCache ) {
1359 $this->setContainerCache( $container, // update persistent cache
1360 array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1361 );
1362 }
1363 }
1364 return $this->connContainerCache->get( $container, 'obj' );
1365 }
1366
1367 /**
1368 * Create a Swift container
1369 *
1370 * @param $container string Container name
1371 * @return CF_Container
1372 * @throws CloudFilesException
1373 */
1374 protected function createContainer( $container ) {
1375 $conn = $this->getConnection(); // Swift proxy connection
1376 $contObj = $conn->create_container( $container );
1377 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1378 return $contObj;
1379 }
1380
1381 /**
1382 * Delete a Swift container
1383 *
1384 * @param $container string Container name
1385 * @return void
1386 * @throws CloudFilesException
1387 */
1388 protected function deleteContainer( $container ) {
1389 $conn = $this->getConnection(); // Swift proxy connection
1390 $this->connContainerCache->clear( $container ); // purge
1391 $conn->delete_container( $container );
1392 }
1393
1394 /**
1395 * @see FileBackendStore::doPrimeContainerCache()
1396 * @return void
1397 */
1398 protected function doPrimeContainerCache( array $containerInfo ) {
1399 try {
1400 $conn = $this->getConnection(); // Swift proxy connection
1401 foreach ( $containerInfo as $container => $info ) {
1402 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1403 $container, $info['count'], $info['bytes'] );
1404 $this->connContainerCache->set( $container, 'obj', $contObj );
1405 }
1406 } catch ( CloudFilesException $e ) { // some other exception?
1407 $this->handleException( $e, null, __METHOD__, array() );
1408 }
1409 }
1410
1411 /**
1412 * Log an unexpected exception for this backend.
1413 * This also sets the Status object to have a fatal error.
1414 *
1415 * @param $e Exception
1416 * @param $status Status|null
1417 * @param $func string
1418 * @param $params Array
1419 * @return void
1420 */
1421 protected function handleException( Exception $e, $status, $func, array $params ) {
1422 if ( $status instanceof Status ) {
1423 if ( $e instanceof AuthenticationException ) {
1424 $status->fatal( 'backend-fail-connect', $this->name );
1425 } else {
1426 $status->fatal( 'backend-fail-internal', $this->name );
1427 }
1428 }
1429 if ( $e->getMessage() ) {
1430 trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1431 }
1432 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1433 $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1434 $this->closeConnection(); // force a re-connect and re-auth next time
1435 }
1436 wfDebugLog( 'SwiftBackend',
1437 get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1438 ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1439 );
1440 }
1441 }
1442
1443 /**
1444 * @see FileBackendStoreOpHandle
1445 */
1446 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1447 /** @var CF_Async_Op */
1448 public $cfOp;
1449 /** @var Array */
1450 public $affectedObjects = array();
1451
1452 public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1453 $this->backend = $backend;
1454 $this->params = $params;
1455 $this->call = $call;
1456 $this->cfOp = $cfOp;
1457 }
1458 }
1459
1460 /**
1461 * SwiftFileBackend helper class to page through listings.
1462 * Swift also has a listing limit of 10,000 objects for sanity.
1463 * Do not use this class from places outside SwiftFileBackend.
1464 *
1465 * @ingroup FileBackend
1466 */
1467 abstract class SwiftFileBackendList implements Iterator {
1468 /** @var Array */
1469 protected $bufferIter = array();
1470 protected $bufferAfter = null; // string; list items *after* this path
1471 protected $pos = 0; // integer
1472 /** @var Array */
1473 protected $params = array();
1474
1475 /** @var SwiftFileBackend */
1476 protected $backend;
1477 protected $container; // string; container name
1478 protected $dir; // string; storage directory
1479 protected $suffixStart; // integer
1480
1481 const PAGE_SIZE = 9000; // file listing buffer size
1482
1483 /**
1484 * @param $backend SwiftFileBackend
1485 * @param $fullCont string Resolved container name
1486 * @param $dir string Resolved directory relative to container
1487 * @param $params Array
1488 */
1489 public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1490 $this->backend = $backend;
1491 $this->container = $fullCont;
1492 $this->dir = $dir;
1493 if ( substr( $this->dir, -1 ) === '/' ) {
1494 $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1495 }
1496 if ( $this->dir == '' ) { // whole container
1497 $this->suffixStart = 0;
1498 } else { // dir within container
1499 $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1500 }
1501 $this->params = $params;
1502 }
1503
1504 /**
1505 * @see Iterator::key()
1506 * @return integer
1507 */
1508 public function key() {
1509 return $this->pos;
1510 }
1511
1512 /**
1513 * @see Iterator::next()
1514 * @return void
1515 */
1516 public function next() {
1517 // Advance to the next file in the page
1518 next( $this->bufferIter );
1519 ++$this->pos;
1520 // Check if there are no files left in this page and
1521 // advance to the next page if this page was not empty.
1522 if ( !$this->valid() && count( $this->bufferIter ) ) {
1523 $this->bufferIter = $this->pageFromList(
1524 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1525 ); // updates $this->bufferAfter
1526 }
1527 }
1528
1529 /**
1530 * @see Iterator::rewind()
1531 * @return void
1532 */
1533 public function rewind() {
1534 $this->pos = 0;
1535 $this->bufferAfter = null;
1536 $this->bufferIter = $this->pageFromList(
1537 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1538 ); // updates $this->bufferAfter
1539 }
1540
1541 /**
1542 * @see Iterator::valid()
1543 * @return bool
1544 */
1545 public function valid() {
1546 if ( $this->bufferIter === null ) {
1547 return false; // some failure?
1548 } else {
1549 return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1550 }
1551 }
1552
1553 /**
1554 * Get the given list portion (page)
1555 *
1556 * @param $container string Resolved container name
1557 * @param $dir string Resolved path relative to container
1558 * @param $after string|null
1559 * @param $limit integer
1560 * @param $params Array
1561 * @return Traversable|Array|null Returns null on failure
1562 */
1563 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1564 }
1565
1566 /**
1567 * Iterator for listing directories
1568 */
1569 class SwiftFileBackendDirList extends SwiftFileBackendList {
1570 /**
1571 * @see Iterator::current()
1572 * @return string|bool String (relative path) or false
1573 */
1574 public function current() {
1575 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1576 }
1577
1578 /**
1579 * @see SwiftFileBackendList::pageFromList()
1580 * @return Array|null
1581 */
1582 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1583 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1584 }
1585 }
1586
1587 /**
1588 * Iterator for listing regular files
1589 */
1590 class SwiftFileBackendFileList extends SwiftFileBackendList {
1591 /**
1592 * @see Iterator::current()
1593 * @return string|bool String (relative path) or false
1594 */
1595 public function current() {
1596 return substr( current( $this->bufferIter ), $this->suffixStart );
1597 }
1598
1599 /**
1600 * @see SwiftFileBackendList::pageFromList()
1601 * @return Array|null
1602 */
1603 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1604 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1605 }
1606 }
Wiklou, le Wiki du Wiklou