From: jenkins-bot Date: Tue, 29 Nov 2016 08:00:43 +0000 (+0000) Subject: Merge "Don't fallback from uk to ru" X-Git-Tag: 1.31.0-rc.0~4746 X-Git-Url: https://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=commitdiff_plain;h=9964ca1a390c446397dcd466916ffed356cdc3c9;hp=98585221d685e7cfcb5a3bb0d766c6cd431f2b05 Merge "Don't fallback from uk to ru" --- diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000000..044dd7202d --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": "wikimedia", + "env": { + "browser": true, + "jquery": true, + "qunit": true + }, + "globals": { + "require": false, + "module": false, + "mediaWiki": false, + "mwPerformance": false, + "OO": false + }, + "rules": { + "dot-notation": 0 + } +} diff --git a/.gitreview b/.gitreview index 0ec44b8359..148be026f5 100644 --- a/.gitreview +++ b/.gitreview @@ -2,5 +2,5 @@ host=gerrit.wikimedia.org port=29418 project=mediawiki/core.git -defaultbranch=master +track=1 defaultrebase=0 diff --git a/.jscsrc b/.jscsrc deleted file mode 100644 index f3db218447..0000000000 --- a/.jscsrc +++ /dev/null @@ -1,37 +0,0 @@ -{ - "preset": "wikimedia", - "es3": true, - - "requireVarDeclFirst": null, - - "requireDotNotation": { "allExcept": [ "keywords" ] }, - "jsDoc": { - "checkAnnotations": { - "preset": "jsduck5", - "extra": { - "context": "some", - "source": "some", - "see": "some" - } - }, - "checkParamNames": true, - "checkRedundantAccess": true, - "checkRedundantReturns": true, - "checkTypes": "strictNativeCase", - "requireNewlineAfterDescription": true, - "requireParamTypes": true, - "requireReturnTypes": true - }, - - "excludeFiles": [ - "docs/**", - "extensions/**", - "node_modules/**", - "resources/lib/**", - "resources/src/jquery.tipsy/**", - "resources/src/jquery/jquery.farbtastic.js", - "resources/src/mediawiki.libs/**", - "skins/**", - "vendor/**" - ] -} diff --git a/.jshintignore b/.jshintignore deleted file mode 100644 index fdde7d054d..0000000000 --- a/.jshintignore +++ /dev/null @@ -1,12 +0,0 @@ -# Generated documentation -docs/** - -# third-party libs -extensions/** -node_modules/** -resources/lib/** -resources/src/jquery.tipsy/** -resources/src/jquery/jquery.farbtastic.js -resources/src/mediawiki.libs/** -skins/** -vendor/** diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 441c4e310b..0000000000 --- a/.jshintrc +++ /dev/null @@ -1,33 +0,0 @@ -{ - // Enforcing - "bitwise": true, - "eqeqeq": true, - "esversion": 3, - "freeze": true, - "futurehostile": true, - "latedef": "nofunc", - "noarg": true, - "nonew": true, - "strict": false, - "undef": true, - "unused": true, - - // Relaxing - "laxbreak": true, - "multistr": true, - - // Environment - "browser": true, - - "globals": { - "require": false, - "module": false, - "mediaWiki": true, - "JSON": true, - "OO": true, - "mwPerformance": true, - "jQuery": false, - "QUnit": false, - "sinon": false - } -} diff --git a/.mailmap b/.mailmap index 5c82af8118..dd968e8d08 100644 --- a/.mailmap +++ b/.mailmap @@ -1,30 +1,62 @@ +# Map author and committer names and email addresses to canonical real names +# and email addresses. +# +# To update the CREDITS file, run maintenance/updateCredits.php +# +# Two types of entries are useful here. The first sets a canonical author +# name for a given email address: +# +# Canonical Author Name +# +# The second allows collecting alternate email addresses into a single +# canonical author name and email address: +# +# Canonical Author Name +# +# Mappings are only needed for authors who have used multiple author names +# and/or author emails for revisions over time. Author names beginning with +# "[BOT]" will be omitted from the CREDITS file. +# +# See also: https://git-scm.com/docs/git-shortlog#_mapping_authors +# +[BOT] Gerrit Code Review [BOT] Gerrit Patch Uploader +[BOT] jenkins-bot +[BOT] jenkins-bot [BOT] Translation updater bot Aaron Schulz Aaron Schulz Adam Roses Wight +Adam Roses Wight addshore +Aditya Sastry Adrian Heine -Alex Monk -Alex Monk -Alex Z -Alexander Emsenhuber -Alexander Emsenhuber -Alexander Emsenhuber +Alex Z. +Alexandre Emsenhuber +Alexandre Emsenhuber +Alexandre Emsenhuber +Alexander Monk +Alexander Monk +Alexander Monk Alexia E. Smith Amir E. Aharoni Amir E. Aharoni +Amir Sarabadani Anders Wegge Jakobsen Andre Engels +Andrew Garrett Andrew Garrett Angela Beesley Starling Antoine Musso Antoine Musso Aran Dunkley Ariel Glenn +Ariel Glenn Arlo Breault +Arthur Richards Arthur Richards Aryeh Gregor +Asher Feldman Asher Feldman aude Audrey Tang @@ -32,25 +64,30 @@ Audrey Tang ayush_garg Bahodir Mansurov Bartosz Dziewoński -Bartosz Dziewoński Bartosz Dziewoński +Bartosz Dziewoński Ben Hartshorne Bene -Benjamin Lees +Bene +Benny Situ Benny Situ Bertrand Grondin Brad Jorsch +Brad Jorsch Brandon Harris -Brian Wolff Brian Wolff +Brian Wolff +Brian Wolff Brion Vibber Brion Vibber Brion Vibber Bryan Davis +Bryan Davis +Bryan Tong Minh Bryan Tong Minh C. Scott Ananian C. Scott Ananian -cacycle@gerrit.wikimedia.org +Cacycle cenarium Chad Horohoe Chad Horohoe @@ -58,44 +95,64 @@ Charles Melbye Chiefwei Chris McMahon Chris Steipp -Christian Aistleitner Christian Aistleitner +Christian Aistleitner Christian Williams Christian Williams Christian Williams +Christopher Johnson +church of emacs +Cindy Cicalese ckoerner Conrad Irwin Dan Duvall dan-nl Daniel A. R. Werner Daniel Cannon +Daniel Friesen +Daniel Friesen Daniel Friesen +Daniel Friesen Daniel Kinzler Daniel Kinzler -Danny B +Danny B. +Danny B. +Danny B. +Danny B. +Darian Anthony Patrick +Darkdragon09 David Chan +Dereckson +Derk-Jan Hartman +Derk-Jan Hartman Derk-Jan Hartman -Derk-Jan Hartman Diederik van Liere Domas Mituzas Douglas Gardner DPStokesNZ Ebrahim Byagowi Ed Sanders -Elliott Eggleston +Elliott Eggleston +Elliott Eggleston Emmanuel Engelhart -eranroz +Emufarmers +Emufarmers +Entlinkt +Eranroz Erik Bernhardson Erik Moeller Erik Moeller Erwin Dokter Evan McIntire +Evan Prodromou Federico Leva Fenzik Joseph -Florianschmidtwelzow -Florianschmidtwelzow Florian -Fomafix +Florian Schmidt +Florian Schmidt +fomafix +Fran Rogers Fran Rogers +freakolowsky FunPika Gabriel Wicke Gabriel Wicke @@ -110,31 +167,38 @@ glaisher Greg Sabino Mullane Greg Sabino Mullane Greg Sabino Mullane +Grunny Guy Van den Broeck Happy-melon Helder Helder Hoo man +Huji Huji Ian Baker Ilmari Karonen Inez Korczyński Inez Korczyński isarra +isarra Ivan Lanin -Jack Phoenix Jack Phoenix +Jack Phoenix Jackmcbarn -Jackmcbarn +Jackmcbarn jagori -James D. Forrester +James Forrester Jan Gerber +Jan Luca Naumann Jan Luca Naumann Jan Paul Posma Jan Zerebecki +Jared Flores Jaroslav Škarvada jarrettmunton +Jason Richey Jason Richey +Jason Richey Jeff Hall Jeff Hall Jeff Janes @@ -151,40 +215,58 @@ Jon Robson Juliusz Gonera Juliusz Gonera JuneHyeon Bae +Jure Kajzer Jure Kajzer +Karun Dambiec +Katie Filbert Katie Filbert Kevin Israel -Kunal Mehta -Kunal Mehta +Kunal Grover +Kunal Mehta +Kunal Mehta +Kunal Mehta Kwan Ting Chan lekshmi Leo Koppelkamm +Leon Liesener Leon Weber Leonardo Gregianin Leons Petrazickis -Liangent +liangent Lisa Ridley Ljudusika Luis Felipe Schenone +Lupo m4tx +Madman Magnus Manske Manuel Schneider <80686@users.mediawiki.org> +Marc-André Pelletier +Marcin Cieślak Marcin Cieślak +Marco Falke +MarcoAurelio Marielle Volz Marius Hoch -Mark A. Hershberger -Mark A. Hershberger -Mark A. Hershberger Mark Clements +Mark Hershberger +Mark Hershberger +Mark Hershberger +Mark Hershberger Mark Holmquist +Mark Holmquist Marko Obrovac +Markus Glaser +Markus Glaser Matt Johnston Matthew Britton Matthew Flaschen Matthias Mullie +Matthias Mullie Matěj Grabovský Max Semenik Max Semenik +Max Semenik mgooley Michael Dale mjbmr @@ -192,23 +274,30 @@ Mohamed Magdy Moriel Schottlender Moriel Schottlender Mormegil +MrBlueSky +MrBlueSky Mukunda Modell +Mwalker MZMcBride nadeesha Namit Nathaniel Herman Neil Kandalgaonkar Nemo bis -Nephele +nephele Nick Jenkins Nik Everett Niklas Laxström Niklas Laxström Nimish Gautam Nuria Ruiz -Ori.livneh +Ori Livneh +Ori Livneh OverlordQ +Owen Davis +Owen Davis paladox +Patrick Reilly Patrick Reilly Patrick Westerhoff Paul Copperman @@ -223,9 +312,9 @@ PranavK Prateek Saxena Prateek Saxena Priyanka Dhanda -Purodha B Blissenbach -Purodha B Blissenbach -Purodha B Blissenbach +Purodha Blissenbach +Purodha Blissenbach +Purodha Blissenbach Raimond Spekking Raimond Spekking Remember the dot @@ -233,13 +322,15 @@ Reza Ricordisamoa rillke rillke -River Tarnell River Tarnell +River Tarnell Roan Kattouw Roan Kattouw Roan Kattouw Rob Church +Rob Lanphier Rob Lanphier +Rob Lanphier Rob Moen Rob Moen Rob Moen @@ -247,24 +338,30 @@ Robert Hoenig Robert Leverington Robert Rohde Robert Stojnić +Robin Pepermans Robin Pepermans robinhood701 Rohan Rotem Liss Rummana Yasmeen Russ Nelson -Ryan Kaldari Ryan Kaldari Ryan Kaldari +Ryan Kaldari +Ryan Lane Ryan Lane +Ryan Lane +Ryan Schmidt +Ryan Schmidt Ryan Schmidt S Page Sam Reed +Sam Reed +Sam Reed Sam Reed Sam Smith -Santhosh Thottingal Santhosh Thottingal -saper +Santhosh Thottingal Schnark Scimonster Sean Colombo @@ -272,11 +369,14 @@ Sean Pringle Seb35 Sergio Santoro Shahyar +Shinjiman Shinjiman Siebrand Mazeland Siebrand Mazeland Siebrand Mazeland Siebrand Mazeland +Smriti Singh +Sorawee Porncharoenwase Southparkfan SQL Stanislav Malyshev @@ -288,9 +388,10 @@ Steven Roddis Subramanya Sastry Sucheta Ghoshal Sumit Asthana +Swalling Thalia Chan -TheDJ Thiemo Mättig (WMDE) +Thiemo Mättig (WMDE) This, that and the other tholam Thomas Bleher @@ -305,29 +406,45 @@ Timo Tijhof Timo Tijhof Timo Tijhof Tina Johnson +Tisane +Tjones Tom Maaswinkel Tomasz Finc +Tomasz W. Kozlowski +Tomasz W. Kozlowski +Tomasz W. Kozlowski Tony Thomas <01tonythomas@gmail.com> +Tpt Trevor Parscal Trevor Parscal Trevor Parscal Tyler Cipriani Tyler Romeo -umherirrender +Umherirrender +Victor Vasiliev Victor Vasiliev +Victor Vasiliev Vikas S Yaligar Vivek Ghaisas wctaiwan withoutaname X! +Yaron Koren +Yaron Koren Yaroslav Melnychuk +Yongmin Hong +Yongmin Hong +Yongmin Hong Yuri Astrakhan +Yuri Astrakhan Yuri Astrakhan Yusuke Matsubara -YuviPanda +Yuvi Panda Zak Greant +Zhengzhu Feng +Zhengzhu Feng +Zppix Ævar Arnfjörð Bjarmason +Étienne Beaulé Željko Filipin Željko Filipin -Zhengzhu Feng -Zhengzhu Feng diff --git a/.travis.yml b/.travis.yml index 9062194628..973860569d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,6 @@ notifications: email: false irc: channels: - - "chat.freenode.net#mediawiki-core" - "chat.freenode.net#mediawiki-feed" on_success: change on_failure: change diff --git a/CREDITS b/CREDITS index dca597ef0d..d9ff9709f1 100644 --- a/CREDITS +++ b/CREDITS @@ -6,251 +6,656 @@ following names for their contribution to the product. --> -== Developers == -* Aaron Schulz -* Alex Z. -* Alexander Monk -* Alexandre Emsenhuber -* Andrew Garrett -* Antoine Musso -* Arthur Richards -* Aryeh Gregor -* Bartosz Dziewoński -* Bertrand Grondin -* Brad Jorsch -* Brian Wolff -* Brion Vibber -* Bryan Davis -* Bryan Tong Minh -* Chad Horohoe -* Charles Melbye -* Chris Steipp -* church of emacs -* Daniel Friesen -* Daniel Kinzler -* Daniel Renfro -* Danny B. -* David McCabe -* Derk-Jan Hartman -* Domas Mituzas -* Emufarmers -* Fran Rogers -* Greg Sabino Mullane -* Guy Van den Broeck -* Happy-melon -* Hojjat -* Ian Baker -* Ilmari Karonen -* Jack D. Pond -* Jack Phoenix -* Jackmcbarn -* James Forrester -* Jan Paul Posma -* Jason Richey -* Jeroen De Dauw -* John Du Hart -* Jon Harald Søby -* Juliano F. Ravasi -* Leo Koppelkamm -* Leon Weber -* Leslie Hoare -* Marco Schuster -* Marius Hoch -* Matěj Grabovský -* Matt Johnston -* Matthew Flaschen -* Max Semenik -* Meno25 -* MinuteElectron -* Mohamed Magdy -* Nathaniel Herman -* Neil Kandalgaonkar -* Nicolas Dumazet -* Niklas Laxström -* Ori Livneh -* Patrick Reilly -* Philip Tzou -* Platonides -* Purodha Blissenbach -* Raimond Spekking -* Remember the dot -* Roan Kattouw -* Robert Stojnić -* Robin Pepermans -* Rotem Liss -* Ryan Kaldari -* Ryan Lane -* Ryan Schmidt -* Sam Reed -* Shinjiman -* Siebrand Mazeland -* Soxred93 -* SQL -* Szymon Świerkosz -* This, that and the other -* Thomas Bleher -* Thomas Gries -* Tim Starling -* Timo Tijhof -* Trevor Parscal -* Tyler Anthony Romeo -* Victor Vasiliev -* Yesid Carrillo -* Yuri Astrakhan - -== Patch Contributors == +== Contributors == + + +* aalekhN * Aaron Ball * Aaron Pramana +* Aaron Schulz +* Aarti Dwivedi +* Aashaka Shah +* abhinand +* Abhishek Das +* Adam Miller +* Adam Roses Wight +* addshore +* Aditya Sastry +* Adrian Heine +* Adrian Lang +* Ævar Arnfjörð Bjarmason * Agbad * Ahmad Sherif +* Ajayrahul P +* Alangi Derick +* Albert221 * Alejandro Mery +* AlephNull +* Alex Ivanov +* Alex Shih-Han Lin +* Alex Z. +* Alexander I. Mashin +* Alexander Lehmann +* Alexander Monk +* Alexander Sigachov +* Alexandre Emsenhuber +* Alexia E. Smith * Amalthea * Amir E. Aharoni +* Amir Sarabadani +* ananay +* Anders Wegge Jakobsen +* Andre Engels +* Andrew Bogott * Andrew Dunbar +* Andrew Garrett +* Andrew Green +* Andrew H +* Andrew Harris +* Andrew Otto +* Andrius R +* andymw +* Angela Beesley Starling +* ankur +* Antoine Musso * Antonio Ospite +* apexkid +* April King +* Aran Dunkley +* Arash Boostani +* Arcane21 +* Ariel Glenn +* Arlo Breault +* Arne Heizmann +* Arthur Richards +* Aryeh Gregor +* Asher Feldman * Asier Lostalé +* ayush_garg * Azliq7 * Bagariavivek +* Bahodir Mansurov +* balloonguy +* Bartosz Dziewoński * Beau +* Ben Davis +* Ben Hartshorne +* Bene * Benny Situ * Bergi +* Bertrand Grondin +* Bill Traynor +* Billinghurst +* billm +* blotmandroid +* Bogdan Stancescu +* Boris Nagaev * Borislav Manolov +* Brad Jorsch +* Brandon Black +* Brandon Harris * Brent G +* Brent Garber +* Brian Wolff * Brianna Laugher +* Brion Vibber +* Bryan Davis +* Bryan Tong Minh +* burthsceh +* C. Scott Ananian +* Cacycle +* Calak +* Camille Constans +* Carl Fürstenberg * Carlin * Carsten Nielsen +* Cblair91 +* cenarium +* Chad Horohoe +* Charles Melbye +* Chiefwei +* Chris McMahon +* Chris Seaton +* Chris Steipp * Christian Aistleitner +* Christian List * Christian Neubauer +* Christopher Johnson +* church of emacs +* Cindy Cicalese +* ckoerner * Conrad Irwin * cryptocoryne * Dan Barrett * Dan Collins +* Dan Duvall * Dan Nessett +* Dan Poltawski +* dan-nl +* Daniel A. R. Werner * Daniel Arnold +* Daniel Cannon +* Daniel De Marco +* Daniel Evans +* Daniel Friesen +* Daniel Kinzler +* Daniel Renfro * Daniel Werner +* DanielRenfro +* Danny B. +* Darian Anthony Patrick +* Darkdragon09 +* DaSch * David Baumgarten +* David Chan +* David E. Narváez +* David Lynch +* David McCabe +* David Mudrák +* dcausse +* dennisroczek * Denny Vrandecic +* Dereckson +* Derk-Jan Hartman +* Derric Atzrott +* Derrick Coetzee * Dévai Tamás +* Devi Krishnan +* Diederik van Liere +* Domas Mituzas +* Douglas Gardner +* DPStokesNZ +* dr0ptp4kt * Ebrahim Byagowi +* Ed Sanders +* Edward Chernenko * Edward Z. Yang +* Elisabeth Bauer +* Elliott Eggleston * Elvis Stansvik +* Emil Podlaszewski +* Emmanuel Engelhart +* Emmanuel Gil Peyrot +* Emmet Hikory +* Emufarmers +* enigmaeth +* Entlinkt * Eranroz +* Eric Evans +* Eric Schneider +* Erich Lerch +* Erick Guan +* Erik Bernhardson +* Erik Moeller * Erwin Dokter * Étienne Beaulé +* Evan McIntire +* Evan Prodromou +* ExplosiveHippo +* Faidon Liambotis * Federico Leva +* Fenzik Joseph +* firebus * Florian Schmidt * fomafix +* Fran Rogers +* Fred Emmott * FunPika * Gabriel Wicke +* Gary Guo +* gbt248 * Geoffrey Mon +* georggi +* Gergő Tisza * Gero Scholz +* gicode +* Giftpflanze +* Gilles Dubuc * Gilles van den Hoven +* Giuseppe Lavagetto +* gladoscc +* glaisher +* Greg Maxwell +* Greg Sabino Mullane +* Gregory Szorc * Grunny +* Guillaume Blanchard +* Guy Van den Broeck +* Happy-melon +* haritha28 * Harry Burt +* Hazard-SJ +* Hector A Escobedo +* Helder +* Henning Snater +* Hojjat +* Huji +* Hydriz +* Ian Baker +* Ilmari Karonen +* Inez Korczyński +* IoannisKydonis * Ireas +* isarra +* Ivan Lanin +* Jack D. Pond +* Jack Phoenix +* Jackmcbarn * Jacob Block +* Jacob Clark +* jagori +* Jakub Vrana +* James Earl Douglas +* James Forrester +* Jan Berkel +* Jan Drewniak * Jan Gerber * Jan Luca Naumann +* Jan Paul Posma +* Jan Zerebecki +* Jared Flores +* Jaroslav Škarvada +* jarrettmunton +* jarry1250 * Jaska Zedlik +* Jason Richey +* jeblad +* Jeff Janes +* jeff303 +* Jens Frank +* Jens Ohlig +* Jérémie Roquet * Jeremy Baron +* Jeremy Postlethwaite +* jeremyb +* Jeroen De Dauw +* Jerome Jamnicky +* Jesús Martínez Novo +* jhobs +* Jiabao * Jidanni +* Jimmy Collins * Jimmy Xu +* joakin +* Joan Creus +* Joel Natividad +* Joerg +* Johan Dahlin +* John Du Hart * John N +* Jon Harald Søby +* Jon Robson * Jonathan Wiltshire +* Jools Wills +* jsahleen +* Julian Ostrow +* Juliano F. Ravasi +* Juliusz Gonera * JuneHyeon Bae * Jure Kajzer +* Justin Du +* Kai_WMDE +* kaligula +* Kartik Mistry * Karun Dambiec * Katie Filbert * Kevin Israel +* Kghbln +* Kim Eik * Kim Hyun-Joon +* kipod +* kishanio +* konarak +* krishna keshav +* Krzysztof Krzyzaniak +* Krzysztof Zbudniewek +* Kunal Grover +* Kunal Mehta +* Kwan Ting Chan +* Laurence Parry +* Lee Bousfield +* Lee Daniel Crocker * Lee Worden * Lejonel +* lekshmi +* Leo Koppelkamm * Leon Liesener +* Leon Weber +* Leonardo Gregianin +* Leons Petrazickis +* Leslie Hoare +* Leszek Manicki +* lethosor +* Lewis Cawte +* Liam Edwards-Playne * liangent +* Lisa Ridley +* Ljudusika +* Lojjik Braughler * Louperivois +* Ltrlg +* Luc Van Oostenryck * Lucas Garczewski * Luigi Corsaro +* Luis Felipe Schenone * Luke Faraone +* Lupin * Lupo +* lwelling +* m4tx * Madman +* madurangasiriwardena +* Magnus Manske * Manuel Menal +* Manuel Schneider +* Marc Ordinas i Llopis * Marc-André Pelletier * Marcin Cieślak +* Marco Falke +* Marco Schuster +* MarcoAurelio * Marcus Buck +* Marius Hoch +* Mark Bergsma +* Mark Clements * Mark Hershberger * Mark Holmquist +* Marko Obrovac +* Markus Glaser +* Markus Krötzsch * Marooned +* Martin Urbanec +* Massaf +* Matěj Grabovský +* matejsuchanek * Mathias Ertl * mati +* Matt Fitzpatrick +* Matt Johnston +* Matt Russell +* Matthew Bowker * Matthew Britton +* Matthew Flaschen +* Matthias Jordan * Matthias Mullie +* MatthiasDD * Max +* Max Semenik * Max Sikström +* mayankmadan +* Meno25 * merl +* Merlijn S. van Deen +* MGChecker +* mgooley +* mhutti1 * Michael Dale * Michael De La Rue +* Michael Holloway * Michael M. * Michael Newton * Michael Walsh +* Michał Łazowik +* Michał Roszka +* Michał Zieliński * Mike Horvath +* Minh Nguyễn +* MinuteElectron +* Misza13 +* mjbmr * moejoe0000 +* Mohamed Magdy +* Molly White +* Moriel Schottlender * Mormegil +* Mr. E23 * MrBlueSky * MrPete +* Mukunda Modell +* Mwalker +* mwjames * mybugs.mail * MZMcBride +* nadeesha * Nakon +* Namit * Nathan Larson +* Nathaniel Herman +* Neil Kandalgaonkar +* Nemo bis * nephele +* Nicholas Pisarro, Jr +* Nick Jenkins +* nicoco007 +* Nicolas Dumazet +* Nicolas Weeger * Nik +* Nik Everett +* Niklas Laxström * Nikola Kovacs +* Nikola Smolenski * Nikolaos S. Karastathis +* Nimish Gautam * Nischay Nahata +* nischayn22 +* nomoa +* nullspoon +* Nuria Ruiz * Nx.devnull +* Ocean behind ears * Olaf Lenz * Olivier Finlay Beaton +* onei +* opatel99 +* Oren Held +* Ori Livneh +* oskar.jauch@gmail.com +* OverlordQ +* Owen Davis +* Paa Kwesi Imbeah +* paladox * Patricio Molina +* Patrick Reilly +* Patrick Westerhoff +* Pau Giner * Paul Copperman * Paul Oranje +* Pavel Astakhov +* Pavel Selitskas +* Pcoombe +* Perside Rosalie * Peter Gehres +* Peter Hedenskog +* Peter Potrowl +* Petr Bena +* Petr Kadlec * Petr Onderka +* Petr Pchelko +* Philip Tzou +* physikerwelt (Moritz Schubotz) * PieRRoMaN +* Pikne +* PiRSquared17 +* Platonides +* Pmlineditor +* pmolina +* prageck +* Pranav Ravichandran +* PranavK +* Prateek Saxena +* Priyanka Dhanda +* Prod +* ptarjan +* pubudu538 +* Purodha Blissenbach +* quiddity * quietust +* Quim Gil +* rahul21 +* Raimond Spekking +* Ramunas Geciauskas +* Remember the dot * René Kijewski +* Reza * rgcjonas +* Ricordisamoa +* rillke +* River Tarnell +* Roan Kattouw +* Rob Church +* Rob Lanphier * Rob Moen +* Robert Hoenig +* Robert Leverington +* Robert Rohde +* Robert Stojnić * Robert Treat +* Robert Vogel +* Robin Pepermans +* robinhood701 * RockMFR +* Rohan +* Roman Nosov +* Roman Tsukanov +* Rotem Liss +* Rowan Collins +* Russ Nelson * Russell Blau * Rusty Burchfield +* Ruud Koot +* Ryan Bies +* Ryan Finnie +* Ryan Kaldari +* Ryan Lane +* Ryan Schmidt * S Page * Salvatore Ingala +* Sam Reed +* Sam Smith * Santhosh Thottingal +* Schnark +* Scimonster +* scnd * Scott Colcord * se4598 +* Sean Colombo +* Sean Pringle +* Seb35 +* Sebastian Brückner * Sébastien Santoro +* Sergio Santoro +* Sethakill +* Shahyar +* Shane Gibbons +* Shane King +* Shinjiman +* shirayuki +* Sidhant Gupta +* Siebrand Mazeland * Simon Walker +* Smriti Singh * Solitarius +* Sorawee Porncharoenwase * Søren Løvborg * Southparkfan +* Soxred93 +* SQL * Srikanth Lakshmanan +* Stanislav Malyshev * Stefano Codari +* Steinsplitter +* Stephan Gambke +* Stephan Muggli +* Stephane Bisson +* Stephen Liang +* Steve Sanbeg +* Steven Roddis * Str4nd * Subramanya Sastry +* Sumit Asthana * svip +* Swalling +* Szymon Świerkosz +* T.D. Corell +* Tarquin +* The Discoverer * The Evil IP address +* theopolisme +* Thiemo Mättig (WMDE) +* This, that and the other +* tholam +* Thomas Arrow +* Thomas Bleher +* Thomas Dalton +* Thomas Gries +* ThomasV +* Tim Hollmann * Tim Landscheidt +* Tim Laqua +* Tim Starling +* Timo Tijhof +* Tina Johnson * Tisane +* tjlsangria +* Tjones +* TK-999 +* Tobias Gritschacher +* Tom Arrow +* Tom Gilder +* Tom Maaswinkel +* Tomasz Finc +* Tomasz W. Kozlowski +* Tomasz Wegrzanowski +* tomek +* Tony Thomas +* Tpt +* Trevor Parscal +* TyA +* Tychay +* Tyler Anthony Romeo +* Tyler Cipriani +* Tyler Romeo +* U-REDMOND\emadelw +* UltrasonicNXT * Umherirrender +* utkarsh95 * Van de Bugger +* Viačeslav +* Victor Porton +* Victor Vasiliev +* victorbarbu * Ville Stadista +* vishnu * Vitaliy Filippov * Vivek Ghaisas +* vlakoff +* Volker E * Waldir Pimenta +* wctaiwan +* Wikinaut +* Wil Mahan * William Demchick +* withoutaname +* WMDE-Fisch +* X! +* XP1 +* Yaron Koren +* Yaroslav Melnychuk +* Yesid Carrillo +* Yogesh K S +* Yongmin Hong +* yoonghm +* Yuri Astrakhan * Yusuke Matsubara * Yuvi Panda * Zachary Hauri +* Zak Greant +* Željko Filipin +* Zhaofeng Li +* Zhengzhu Feng +* Zppix +* محمد شعیب + == Translators == diff --git a/FAQ b/FAQ index cfacf14676..29017bcb5d 100644 --- a/FAQ +++ b/FAQ @@ -1,2 +1,2 @@ The MediaWiki FAQ can be found at: -https://www.mediawiki.org/wiki/Manual:FAQ +https://www.mediawiki.org/wiki/Special:MyLanguage/Manual:FAQ \ No newline at end of file diff --git a/Gruntfile.js b/Gruntfile.js index a08db5c780..55b7932f00 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,33 +1,44 @@ -/*jshint node:true */ +/* eslint-env node */ + module.exports = function ( grunt ) { - grunt.loadNpmTasks( 'grunt-contrib-copy' ); - grunt.loadNpmTasks( 'grunt-contrib-jshint' ); - grunt.loadNpmTasks( 'grunt-stylelint' ); - grunt.loadNpmTasks( 'grunt-contrib-watch' ); - grunt.loadNpmTasks( 'grunt-banana-checker' ); - grunt.loadNpmTasks( 'grunt-jscs' ); - grunt.loadNpmTasks( 'grunt-jsonlint' ); - grunt.loadNpmTasks( 'grunt-karma' ); var wgServer = process.env.MW_SERVER, wgScriptPath = process.env.MW_SCRIPT_PATH, karmaProxy = {}; + grunt.loadNpmTasks( 'grunt-banana-checker' ); + grunt.loadNpmTasks( 'grunt-contrib-copy' ); + grunt.loadNpmTasks( 'grunt-contrib-watch' ); + grunt.loadNpmTasks( 'grunt-eslint' ); + grunt.loadNpmTasks( 'grunt-jsonlint' ); + grunt.loadNpmTasks( 'grunt-karma' ); + grunt.loadNpmTasks( 'grunt-stylelint' ); + karmaProxy[ wgScriptPath ] = wgServer + wgScriptPath; grunt.initConfig( { - jshint: { - options: { - jshintrc: true - }, - all: '.' - }, - jscs: { - all: '.' + eslint: { + all: [ + '**/*.js', + '!docs/**', + '!tests/**', + '!node_modules/**', + '!resources/lib/**', + '!resources/src/jquery.tipsy/**', + '!resources/src/jquery/jquery.farbtastic.js', + '!resources/src/mediawiki.libs/**', + '!vendor/**', + // Explicitly say "**/*.js" here in case of symlinks + '!extensions/**/*.js', + '!skins/**/*.js', + // Skip functions aren't even parseable + '!resources/src/dom-level2-skip.js', + '!resources/src/es5-skip.js', + '!resources/src/mediawiki.hidpi-skip.js' + ] }, jsonlint: { all: [ - '.jscsrc', '**/*.json', '!{docs/js,extensions,node_modules,skins,vendor}/**' ] @@ -48,7 +59,7 @@ module.exports = function ( grunt ) { }, watch: { files: [ - '.{stylelintrc,jscsrc,jshintignore,jshintrc}', + '.{stylelintrc,eslintrc.json}', '**/*', '!{docs,extensions,node_modules,skins,vendor}/**' ], @@ -103,7 +114,7 @@ module.exports = function ( grunt ) { return !!( process.env.MW_SERVER && process.env.MW_SCRIPT_PATH ); } ); - grunt.registerTask( 'lint', [ 'jshint', 'jscs', 'jsonlint', 'banana', 'stylelint' ] ); + grunt.registerTask( 'lint', [ 'eslint', 'banana', 'stylelint' ] ); grunt.registerTask( 'qunit', [ 'assert-mw-env', 'karma:main' ] ); grunt.registerTask( 'test', [ 'lint' ] ); diff --git a/HISTORY b/HISTORY index 868b21af9a..28a9b869df 100644 --- a/HISTORY +++ b/HISTORY @@ -1,4 +1,328 @@ -Change notes from older releases. For current info see RELEASE-NOTES-1.28. +Change notes from older releases. For current info see RELEASE-NOTES-1.29. + +== MediaWiki 1.28 == + +=== Changes since 1.28.0-rc1 === +* (T148957) Replace wgShowExceptionDetails with wgShowDBErrorBacktrace on db + errors. +* (T148956) Only apply wgDBschema to postgres/mssql. +* (T145991) Introduce separate log action for deleting pages on move. +* (T141474) (T110464) Bypass login page if no user input is required. + +=== Changes since 1.28.0-rc0 === +* (T142210) The changes to move the parser "NewPP limit report" from a HTML + comment to a machine-readable JavaScript config option 'wgPageParseReport' + have been undone. They caused the human-readable limit report to be shown + incompletely or not at all. ParserOutput::setLimitReportData() and + getLimitReportData() behave as they did in MediaWiki 1.27 again. +* (T149510) Value of {{DISPLAYTITLE:}} parser function will not be used for + the text of subheadings on a category page when creating it. This wasn't + working correctly. +* (T106793) MediaWiki will no longer try to perform a HTTP redirect to the + canonical pretty URL when a non-pretty URL is used. It resulted in redirect + loops in some clients and in some server configurations. This undoes a change + made in MediaWiki 1.26. +* (T149759) manifest_version: 2 was removed. + +=== Configuration changes in 1.28 === +* $wgSend404Code now affects status code of action=history if the page is not there. +* BREAKING CHANGE: $wgHTTPProxy is now *required* for all external requests + made by MediaWiki via a proxy. Relying on the http_proxy environment + variable is no longer supported. +* The load.php entry point now enforces the existing policy of not allowing + access to session data, which includes the session user and the session + user's language. If such access is attempted, an exception will be thrown. +* The number of internal PBKDF2 iterations used to derive the session secret + is configurable via $wgSessionPbkdf2Iterations. +* Upload dialog's file upload log comment can now be configured separately for + local and foreign uploads. +* $wgForeignUploadTargets now defaults to `[ 'local' ]`, where `'local'` + signifies local uploads. A value of `[]` (empty array) now means that + no upload targets are allowed, effectively disabling the upload dialog. +* The deprecated $wgEditEncoding variable has been removed; it was only used + for Esperanto language character conversion. You are now recommended to use + input methods provided by the UniversalLanguageSelector extension. +* When $wgPingback is true, MediaWiki will periodically ping + https://www.mediawiki.org/beacon with basic information about the local + MediaWiki installation. This data includes, for example, the type of system, + PHP version, and chosen database backend. This behavior is off by default. +* When $wgEditSubmitButtonLabelPublish is true, MediaWiki will label the button + to store-to-database-and-show-to-others as "Publish page"/"Publish changes"; + if false, the default, they will be "Save page"/"Save changes". +* The 'editcontentmodel' permission is now granted to all logged-in users ('user'). + instead of just administrators ('sysop'). Documentation for this feature is + available at . +* $wgRevisionCacheExpiry is now set to one week by default instead of being disabled. +* Magic links are now disabled by default, and can be re-enabled by modifying the value + of $wgEnableMagicLinks. Their usage is discouraged, but if they are manually enabled, + a tracking category will be added to help identify usage and make it easier to migrate + away from. If you depend upon magic link functionality, it is requested that you comment + on and + explain your use case(s). +* New config variable $wgCSPFalsePositiveUrls to control what URLs to ignore + in upcoming Content-Security-Policy feature's reporting. + +=== New features in 1.28 === +* User::isBot() method for checking if an account is a bot role account. +* Added a new 'slideshow' mode for galleries. +* Added a new hook, 'UserIsBot', to aid in determining if a user is a bot. +* Added a new hook, 'ApiMakeParserOptions', to allow extensions to better + interact with API parsing. +* Added a new hook, 'UploadVerifyUpload', which can be used to reject a file + upload. Unlike 'UploadVerifyFile' it provides information about upload comment + and the file description page, but does not run for uploads to stash. +* (T141604) Extensions can now provide a better error message when their + maintenance scripts are run without the extension being installed. +* (T8948) Numeric sorting in categories is now supported by setting $wgCategoryCollation + to 'uca-default-u-kn' or 'uca--u-kn'. If you can't use UCA collations, + a 'numeric' collation is also available. If migrating from another + collation, you will need to run the updateCollation.php maintenance script. +* Two new codes have been added to #time parser function: "xit" for days in current + month, and "xiz" for days passed in the year, both in Iranian calendar. +* mw.Api has a new option, useUS, to use U+001F (Unit Separator) when + appropriate for sending multi-valued parameters. This defaults to true when + the mw.Api instance seems to be for the local wiki. +* After a client performs an action which alters a database that has replica databases, + MediaWiki will wait for the replica databases to synchronize with the master database + while it renders the HTML output. However, if the output is a redirect to another wiki + on the wiki farm with a different domain, MediaWiki will instead alter the redirect + URL to include a ?cpPosTime parameter that triggers the database synchronization when + the URL is followed by the client. The same-domain case uses a new cpPosTime cookie. +* Added new hooks, 'ApiQueryBaseBeforeQuery', 'ApiQueryBaseAfterQuery', and + 'ApiQueryBaseProcessRow', to make it easier for extensions to add 'prop' and + 'show' parameters to existing API query modules. + +=== External library changes in 1.28 === + +==== Upgraded external libraries ==== +* Updated es5-shim from v4.1.5 to v4.5.8 +* Updated composer/semver from v1.4.1 to v1.4.2 +* Updated wikimedia/php-session-serializer from v1.0.3 to v1.0.4 + +==== New external libraries ==== +* Added wikimedia/scoped-callback v1.0.0 +* Added wikimedia/wait-condition-loop v1.0.1 + +=== Bug fixes in 1.28 === +* (T146496) action=history pages should return 404 HTTP error code if the page does not exist +* (T137264) SECURITY: XSS in unclosed internal links +* (T133147) SECURITY: Escape '<' and ']]>' in inline ' . + "\n"; + + $hookResult = self::runHooks( $e, get_class( $e ) . 'Raw' ); + if ( $hookResult ) { + echo $hookResult; + } else { + echo self::getHTML( $e ); + } + + echo "\n"; + } + } + + /** + * If $wgShowExceptionDetails is true, return a HTML message with a + * backtrace to the error, otherwise show a message to ask to set it to true + * to show that information. + * + * @param Exception|Throwable $e + * @return string Html to output + */ + public static function getHTML( $e ) { + if ( self::showBackTrace( $e ) ) { + $html = "

" . + nl2br( htmlspecialchars( MWExceptionHandler::getLogMessage( $e ) ) ) . + '

Backtrace:

' . + nl2br( htmlspecialchars( MWExceptionHandler::getRedactedTraceAsString( $e ) ) ) . + "

\n"; + } else { + $logId = WebRequest::getRequestId(); + $html = "
" . + '[' . $logId . '] ' . + gmdate( 'Y-m-d H:i:s' ) . ": " . + self::msg( "internalerror-fatal-exception", + "Fatal exception of type $1", + get_class( $e ), + $logId, + MWExceptionHandler::getURL() + ) . "
\n" . + ""; + } + + return $html; + } + + /** + * Get a message from i18n + * + * @param string $key Message name + * @param string $fallback Default message if the message cache can't be + * called by the exception + * The function also has other parameters that are arguments for the message + * @return string Message with arguments replaced + */ + private static function msg( $key, $fallback /*[, params...] */ ) { + $args = array_slice( func_get_args(), 2 ); + try { + return wfMessage( $key, $args )->text(); + } catch ( Exception $e ) { + return wfMsgReplaceArgs( $fallback, $args ); + } + } + + /** + * @param Exception|Throwable $e + * @return string + */ + private static function getText( $e ) { + if ( self::showBackTrace( $e ) ) { + return MWExceptionHandler::getLogMessage( $e ) . + "\nBacktrace:\n" . + MWExceptionHandler::getRedactedTraceAsString( $e ) . "\n"; + } else { + return self::getShowBacktraceError( $e ); + } + } + + /** + * @param Exception|Throwable $e + * @return bool + */ + private static function showBackTrace( $e ) { + global $wgShowExceptionDetails, $wgShowDBErrorBacktrace; + + return ( + $wgShowExceptionDetails && + ( !( $e instanceof DBError ) || $wgShowDBErrorBacktrace ) + ); + } + + /** + * @param Exception|Throwable $e + * @return string + */ + private static function getShowBacktraceError( $e ) { + global $wgShowExceptionDetails, $wgShowDBErrorBacktrace; + $vars = []; + if ( !$wgShowExceptionDetails ) { + $vars[] = '$wgShowExceptionDetails = true;'; + } + if ( $e instanceof DBError && !$wgShowDBErrorBacktrace ) { + $vars[] = '$wgShowDBErrorBacktrace = true;'; + } + $vars = implode( ' and ', $vars ); + return "Set $vars at the bottom of LocalSettings.php to show detailed debugging information"; + } + + /** + * @return bool + */ + private static function isCommandLine() { + return !empty( $GLOBALS['wgCommandLineMode'] ); + } + + /** + * @param string $header + */ + private static function header( $header ) { + if ( !headers_sent() ) { + header( $header ); + } + } + + /** + * @param integer $code + */ + private static function statusHeader( $code ) { + if ( !headers_sent() ) { + HttpStatus::header( $code ); + } + } + + /** + * Print a message, if possible to STDERR. + * Use this in command line mode only (see isCommandLine) + * + * @param string $message Failure text + */ + private static function printError( $message ) { + // NOTE: STDERR may not be available, especially if php-cgi is used from the + // command line (bug #15602). Try to produce meaningful output anyway. Using + // echo may corrupt output to STDOUT though. + if ( defined( 'STDERR' ) ) { + fwrite( STDERR, $message ); + } else { + echo $message; + } + } + + /** + * @param Exception|Throwable $e + */ + private static function reportOutageHTML( $e ) { + global $wgShowDBErrorBacktrace, $wgShowHostnames, $wgShowSQLErrors; + + $sorry = htmlspecialchars( self::msg( + 'dberr-problems', + 'Sorry! This site is experiencing technical difficulties.' + ) ); + $again = htmlspecialchars( self::msg( + 'dberr-again', + 'Try waiting a few minutes and reloading.' + ) ); + + if ( $wgShowHostnames || $wgShowSQLErrors ) { + $info = str_replace( + '$1', + Html::element( 'span', [ 'dir' => 'ltr' ], htmlspecialchars( $e->getMessage() ) ), + htmlspecialchars( self::msg( 'dberr-info', '($1)' ) ) + ); + } else { + $info = htmlspecialchars( self::msg( + 'dberr-info-hidden', + '(Cannot access the database)' + ) ); + } + + MessageCache::singleton()->disable(); // no DB access + + $html = "

$sorry

$again

$info

"; + + if ( $wgShowDBErrorBacktrace ) { + $html .= '

Backtrace:

' .
+				htmlspecialchars( $e->getTraceAsString() ) . '
'; + } + + $html .= '
'; + $html .= self::googleSearchForm(); + + echo $html; + } + + /** + * @return string + */ + private static function googleSearchForm() { + global $wgSitename, $wgCanonicalServer, $wgRequest; + + $usegoogle = htmlspecialchars( self::msg( + 'dberr-usegoogle', + 'You can try searching via Google in the meantime.' + ) ); + $outofdate = htmlspecialchars( self::msg( + 'dberr-outofdate', + 'Note that their indexes of our content may be out of date.' + ) ); + $googlesearch = htmlspecialchars( self::msg( 'searchbutton', 'Search' ) ); + $search = htmlspecialchars( $wgRequest->getVal( 'search' ) ); + $server = htmlspecialchars( $wgCanonicalServer ); + $sitename = htmlspecialchars( $wgSitename ); + $trygoogle = <<$usegoogle
+$outofdate + +
+ + + + + + +

+ + +

+
+EOT; + return $trygoogle; + } +} diff --git a/includes/exception/PermissionsError.php b/includes/exception/PermissionsError.php index bd0b1204d4..e31374c2c7 100644 --- a/includes/exception/PermissionsError.php +++ b/includes/exception/PermissionsError.php @@ -29,12 +29,19 @@ class PermissionsError extends ErrorPageError { public $permission, $errors; /** - * @param string $permission A permission name. - * @param string[] $errors Error message keys + * @param string|null $permission A permission name or null if unknown + * @param array $errors Error message keys or [key, param...] arrays; must not be empty if + * $permission is null + * @throws \InvalidArgumentException */ public function __construct( $permission, $errors = [] ) { global $wgLang; + if ( $permission === null && !$errors ) { + throw new \InvalidArgumentException( __METHOD__ . + ': $permission and $errors cannot both be empty' ); + } + $this->permission = $permission; if ( !count( $errors ) ) { diff --git a/includes/exception/TimestampException.php b/includes/exception/TimestampException.php deleted file mode 100644 index b9c0c35c71..0000000000 --- a/includes/exception/TimestampException.php +++ /dev/null @@ -1,7 +0,0 @@ - "http://www.mediawiki.org/xml/export-$ver/ " . "http://www.mediawiki.org/xml/export-$ver.xsd", diff --git a/includes/externalstore/ExternalStoreDB.php b/includes/externalstore/ExternalStoreDB.php index b45457720e..7e932994e6 100644 --- a/includes/externalstore/ExternalStoreDB.php +++ b/includes/externalstore/ExternalStoreDB.php @@ -112,7 +112,7 @@ class ExternalStoreDB extends ExternalStoreMedium { } /** - * Get a slave database connection for the specified cluster + * Get a replica DB connection for the specified cluster * * @param string $cluster Cluster name * @return IDatabase @@ -130,7 +130,7 @@ class ExternalStoreDB extends ExternalStoreMedium { wfDebug( "writable external store\n" ); } - $db = $lb->getConnection( DB_SLAVE, [], $wiki ); + $db = $lb->getConnectionRef( DB_REPLICA, [], $wiki ); $db->clearFlag( DBO_TRX ); // sanity return $db; @@ -146,7 +146,7 @@ class ExternalStoreDB extends ExternalStoreMedium { $wiki = isset( $this->params['wiki'] ) ? $this->params['wiki'] : false; $lb = $this->getLoadBalancer( $cluster ); - $db = $lb->getConnection( DB_MASTER, [], $wiki ); + $db = $lb->getConnectionRef( DB_MASTER, [], $wiki ); $db->clearFlag( DBO_TRX ); // sanity return $db; @@ -264,7 +264,7 @@ class ExternalStoreDB extends ExternalStoreMedium { } /** - * Helper function for self::batchFetchBlobs for merging master/slave results + * Helper function for self::batchFetchBlobs for merging master/replica DB results * @param array &$ret Current self::batchFetchBlobs return value * @param array &$ids Map from blob_id to requested itemIDs * @param mixed $res DB result from Database::select diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php deleted file mode 100644 index 8aa11b6565..0000000000 --- a/includes/filebackend/FSFile.php +++ /dev/null @@ -1,280 +0,0 @@ -path = $path; - } - - /** - * Returns the file system path - * - * @return string - */ - public function getPath() { - return $this->path; - } - - /** - * Checks if the file exists - * - * @return bool - */ - public function exists() { - return is_file( $this->path ); - } - - /** - * Get the file size in bytes - * - * @return int|bool - */ - public function getSize() { - return filesize( $this->path ); - } - - /** - * Get the file's last-modified timestamp - * - * @return string|bool TS_MW timestamp or false on failure - */ - public function getTimestamp() { - MediaWiki\suppressWarnings(); - $timestamp = filemtime( $this->path ); - MediaWiki\restoreWarnings(); - if ( $timestamp !== false ) { - $timestamp = wfTimestamp( TS_MW, $timestamp ); - } - - return $timestamp; - } - - /** - * Guess the MIME type from the file contents alone - * - * @return string - */ - public function getMimeType() { - return MimeMagic::singleton()->guessMimeType( $this->path, false ); - } - - /** - * Get an associative array containing information about - * a file with the given storage path. - * - * Resulting array fields include: - * - fileExists - * - size (filesize in bytes) - * - mime (as major/minor) - * - media_type (value to be used with the MEDIATYPE_xxx constants) - * - metadata (handler specific) - * - sha1 (in base 36) - * - width - * - height - * - bits (bitrate) - * - file-mime - * - major_mime - * - minor_mime - * - * @param string|bool $ext The file extension, or true to extract it from the filename. - * Set it to false to ignore the extension. - * @return array - */ - public function getProps( $ext = true ) { - wfDebug( __METHOD__ . ": Getting file info for $this->path\n" ); - - $info = self::placeholderProps(); - $info['fileExists'] = $this->exists(); - - if ( $info['fileExists'] ) { - $magic = MimeMagic::singleton(); - - # get the file extension - if ( $ext === true ) { - $ext = self::extensionFromPath( $this->path ); - } - - # MIME type according to file contents - $info['file-mime'] = $this->getMimeType(); - # logical MIME type - $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext ); - - list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] ); - $info['media_type'] = $magic->getMediaType( $this->path, $info['mime'] ); - - # Get size in bytes - $info['size'] = $this->getSize(); - - # Height, width and metadata - $handler = MediaHandler::getHandler( $info['mime'] ); - if ( $handler ) { - $tempImage = (object)[]; // XXX (hack for File object) - $info['metadata'] = $handler->getMetadata( $tempImage, $this->path ); - $gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] ); - if ( is_array( $gis ) ) { - $info = $this->extractImageSizeInfo( $gis ) + $info; - } - } - $info['sha1'] = $this->getSha1Base36(); - - wfDebug( __METHOD__ . ": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n" ); - } else { - wfDebug( __METHOD__ . ": $this->path NOT FOUND!\n" ); - } - - return $info; - } - - /** - * Placeholder file properties to use for files that don't exist - * - * Resulting array fields include: - * - fileExists - * - mime (as major/minor) - * - media_type (value to be used with the MEDIATYPE_xxx constants) - * - metadata (handler specific) - * - sha1 (in base 36) - * - width - * - height - * - bits (bitrate) - * - * @return array - */ - public static function placeholderProps() { - $info = []; - $info['fileExists'] = false; - $info['mime'] = null; - $info['media_type'] = MEDIATYPE_UNKNOWN; - $info['metadata'] = ''; - $info['sha1'] = ''; - $info['width'] = 0; - $info['height'] = 0; - $info['bits'] = 0; - - return $info; - } - - /** - * Exract image size information - * - * @param array $gis - * @return array - */ - protected function extractImageSizeInfo( array $gis ) { - $info = []; - # NOTE: $gis[2] contains a code for the image type. This is no longer used. - $info['width'] = $gis[0]; - $info['height'] = $gis[1]; - if ( isset( $gis['bits'] ) ) { - $info['bits'] = $gis['bits']; - } else { - $info['bits'] = 0; - } - - return $info; - } - - /** - * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case - * encoding, zero padded to 31 digits. - * - * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 - * fairly neatly. - * - * @param bool $recache - * @return bool|string False on failure - */ - public function getSha1Base36( $recache = false ) { - if ( $this->sha1Base36 !== null && !$recache ) { - return $this->sha1Base36; - } - - MediaWiki\suppressWarnings(); - $this->sha1Base36 = sha1_file( $this->path ); - MediaWiki\restoreWarnings(); - - if ( $this->sha1Base36 !== false ) { - $this->sha1Base36 = Wikimedia\base_convert( $this->sha1Base36, 16, 36, 31 ); - } - - return $this->sha1Base36; - } - - /** - * Get the final file extension from a file system path - * - * @param string $path - * @return string - */ - public static function extensionFromPath( $path ) { - $i = strrpos( $path, '.' ); - - return strtolower( $i ? substr( $path, $i + 1 ) : '' ); - } - - /** - * Get an associative array containing information about a file in the local filesystem. - * - * @param string $path Absolute local filesystem path - * @param string|bool $ext The file extension, or true to extract it from the filename. - * Set it to false to ignore the extension. - * @return array - */ - public static function getPropsFromPath( $path, $ext = true ) { - $fsFile = new self( $path ); - - return $fsFile->getProps( $ext ); - } - - /** - * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case - * encoding, zero padded to 31 digits. - * - * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 - * fairly neatly. - * - * @param string $path - * @return bool|string False on failure - */ - public static function getSha1Base36FromPath( $path ) { - $fsFile = new self( $path ); - - return $fsFile->getSha1Base36(); - } -} diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php deleted file mode 100644 index efe78ee24b..0000000000 --- a/includes/filebackend/FSFileBackend.php +++ /dev/null @@ -1,975 +0,0 @@ -basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash - } else { - $this->basePath = null; // none; containers must have explicit paths - } - - if ( isset( $config['containerPaths'] ) ) { - $this->containerPaths = (array)$config['containerPaths']; - foreach ( $this->containerPaths as &$path ) { - $path = rtrim( $path, '/' ); // remove trailing slash - } - } - - $this->fileMode = isset( $config['fileMode'] ) ? $config['fileMode'] : 0644; - if ( isset( $config['fileOwner'] ) && function_exists( 'posix_getuid' ) ) { - $this->fileOwner = $config['fileOwner']; - // cache this, assuming it doesn't change - $this->currentUser = posix_getpwuid( posix_getuid() )['name']; - } - } - - public function getFeatures() { - return !wfIsWindows() ? FileBackend::ATTR_UNICODE_PATHS : 0; - } - - protected function resolveContainerPath( $container, $relStoragePath ) { - // Check that container has a root directory - if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) { - // Check for sane relative paths (assume the base paths are OK) - if ( $this->isLegalRelPath( $relStoragePath ) ) { - return $relStoragePath; - } - } - - return null; - } - - /** - * Sanity check a relative file system path for validity - * - * @param string $path Normalized relative path - * @return bool - */ - protected function isLegalRelPath( $path ) { - // Check for file names longer than 255 chars - if ( preg_match( '![^/]{256}!', $path ) ) { // ext3/NTFS - return false; - } - if ( wfIsWindows() ) { // NTFS - return !preg_match( '![:*?"<>|]!', $path ); - } else { - return true; - } - } - - /** - * Given the short (unresolved) and full (resolved) name of - * a container, return the file system path of the container. - * - * @param string $shortCont - * @param string $fullCont - * @return string|null - */ - protected function containerFSRoot( $shortCont, $fullCont ) { - if ( isset( $this->containerPaths[$shortCont] ) ) { - return $this->containerPaths[$shortCont]; - } elseif ( isset( $this->basePath ) ) { - return "{$this->basePath}/{$fullCont}"; - } - - return null; // no container base path defined - } - - /** - * Get the absolute file system path for a storage path - * - * @param string $storagePath Storage path - * @return string|null - */ - protected function resolveToFSPath( $storagePath ) { - list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath ); - if ( $relPath === null ) { - return null; // invalid - } - list( , $shortCont, ) = FileBackend::splitStoragePath( $storagePath ); - $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - if ( $relPath != '' ) { - $fsPath .= "/{$relPath}"; - } - - return $fsPath; - } - - public function isPathUsableInternal( $storagePath ) { - $fsPath = $this->resolveToFSPath( $storagePath ); - if ( $fsPath === null ) { - return false; // invalid - } - $parentDir = dirname( $fsPath ); - - if ( file_exists( $fsPath ) ) { - $ok = is_file( $fsPath ) && is_writable( $fsPath ); - } else { - $ok = is_dir( $parentDir ) && is_writable( $parentDir ); - } - - if ( $this->fileOwner !== null && $this->currentUser !== $this->fileOwner ) { - $ok = false; - trigger_error( __METHOD__ . ": PHP process owner is not '{$this->fileOwner}'." ); - } - - return $ok; - } - - protected function doCreateInternal( array $params ) { - $status = Status::newGood(); - - $dest = $this->resolveToFSPath( $params['dst'] ); - if ( $dest === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - if ( !empty( $params['async'] ) ) { // deferred - $tempFile = TempFSFile::factory( 'create_', 'tmp' ); - if ( !$tempFile ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - - return $status; - } - $this->trapWarnings(); - $bytes = file_put_contents( $tempFile->getPath(), $params['content'] ); - $this->untrapWarnings(); - if ( $bytes === false ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - - return $status; - } - $cmd = implode( ' ', [ - wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) - wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ), - wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) - ] ); - $handler = function ( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - }; - $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); - $tempFile->bind( $status->value ); - } else { // immediate write - $this->trapWarnings(); - $bytes = file_put_contents( $dest, $params['content'] ); - $this->untrapWarnings(); - if ( $bytes === false ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - - return $status; - } - $this->chmod( $dest ); - } - - return $status; - } - - protected function doStoreInternal( array $params ) { - $status = Status::newGood(); - - $dest = $this->resolveToFSPath( $params['dst'] ); - if ( $dest === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', [ - wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) - wfEscapeShellArg( $this->cleanPathSlashes( $params['src'] ) ), - wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) - ] ); - $handler = function ( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - }; - $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); - } else { // immediate write - $this->trapWarnings(); - $ok = copy( $params['src'], $dest ); - $this->untrapWarnings(); - // In some cases (at least over NFS), copy() returns true when it fails - if ( !$ok || ( filesize( $params['src'] ) !== filesize( $dest ) ) ) { - if ( $ok ) { // PHP bug - unlink( $dest ); // remove broken file - trigger_error( __METHOD__ . ": copy() failed but returned true." ); - } - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - - return $status; - } - $this->chmod( $dest ); - } - - return $status; - } - - protected function doCopyInternal( array $params ) { - $status = Status::newGood(); - - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - $dest = $this->resolveToFSPath( $params['dst'] ); - if ( $dest === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - if ( !is_file( $source ) ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-copy', $params['src'] ); - } - - return $status; // do nothing; either OK or bad status - } - - if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', [ - wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) - wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), - wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) - ] ); - $handler = function ( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - }; - $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest ); - } else { // immediate write - $this->trapWarnings(); - $ok = ( $source === $dest ) ? true : copy( $source, $dest ); - $this->untrapWarnings(); - // In some cases (at least over NFS), copy() returns true when it fails - if ( !$ok || ( filesize( $source ) !== filesize( $dest ) ) ) { - if ( $ok ) { // PHP bug - $this->trapWarnings(); - unlink( $dest ); // remove broken file - $this->untrapWarnings(); - trigger_error( __METHOD__ . ": copy() failed but returned true." ); - } - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - - return $status; - } - $this->chmod( $dest ); - } - - return $status; - } - - protected function doMoveInternal( array $params ) { - $status = Status::newGood(); - - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - $dest = $this->resolveToFSPath( $params['dst'] ); - if ( $dest === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - if ( !is_file( $source ) ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-move', $params['src'] ); - } - - return $status; // do nothing; either OK or bad status - } - - if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', [ - wfIsWindows() ? 'MOVE /Y' : 'mv', // (overwrite) - wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), - wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) - ] ); - $handler = function ( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - }; - $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd ); - } else { // immediate write - $this->trapWarnings(); - $ok = ( $source === $dest ) ? true : rename( $source, $dest ); - $this->untrapWarnings(); - clearstatcache(); // file no longer at source - if ( !$ok ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - - return $status; - } - } - - return $status; - } - - protected function doDeleteInternal( array $params ) { - $status = Status::newGood(); - - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - if ( !is_file( $source ) ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } - - return $status; // do nothing; either OK or bad status - } - - if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', [ - wfIsWindows() ? 'DEL' : 'unlink', - wfEscapeShellArg( $this->cleanPathSlashes( $source ) ) - ] ); - $handler = function ( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - }; - $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd ); - } else { // immediate write - $this->trapWarnings(); - $ok = unlink( $source ); - $this->untrapWarnings(); - if ( !$ok ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - - return $status; - } - } - - return $status; - } - - /** - * @param string $fullCont - * @param string $dirRel - * @param array $params - * @return Status - */ - protected function doPrepareInternal( $fullCont, $dirRel, array $params ) { - $status = Status::newGood(); - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - $existed = is_dir( $dir ); // already there? - // Create the directory and its parents as needed... - $this->trapWarnings(); - if ( !wfMkdirParents( $dir ) ) { - wfDebugLog( 'FSFileBackend', __METHOD__ . ": cannot create directory $dir" ); - $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races - } elseif ( !is_writable( $dir ) ) { - wfDebugLog( 'FSFileBackend', __METHOD__ . ": directory $dir is read-only" ); - $status->fatal( 'directoryreadonlyerror', $params['dir'] ); - } elseif ( !is_readable( $dir ) ) { - wfDebugLog( 'FSFileBackend', __METHOD__ . ": directory $dir is not readable" ); - $status->fatal( 'directorynotreadableerror', $params['dir'] ); - } - $this->untrapWarnings(); - // Respect any 'noAccess' or 'noListing' flags... - if ( is_dir( $dir ) && !$existed ) { - $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) ); - } - - return $status; - } - - protected function doSecureInternal( $fullCont, $dirRel, array $params ) { - $status = Status::newGood(); - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - // Seed new directories with a blank index.html, to prevent crawling... - if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) { - $this->trapWarnings(); - $bytes = file_put_contents( "{$dir}/index.html", $this->indexHtmlPrivate() ); - $this->untrapWarnings(); - if ( $bytes === false ) { - $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' ); - } - } - // Add a .htaccess file to the root of the container... - if ( !empty( $params['noAccess'] ) && !file_exists( "{$contRoot}/.htaccess" ) ) { - $this->trapWarnings(); - $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() ); - $this->untrapWarnings(); - if ( $bytes === false ) { - $storeDir = "mwstore://{$this->name}/{$shortCont}"; - $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" ); - } - } - - return $status; - } - - protected function doPublishInternal( $fullCont, $dirRel, array $params ) { - $status = Status::newGood(); - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - // Unseed new directories with a blank index.html, to allow crawling... - if ( !empty( $params['listing'] ) && is_file( "{$dir}/index.html" ) ) { - $exists = ( file_get_contents( "{$dir}/index.html" ) === $this->indexHtmlPrivate() ); - $this->trapWarnings(); - if ( $exists && !unlink( "{$dir}/index.html" ) ) { // reverse secure() - $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' ); - } - $this->untrapWarnings(); - } - // Remove the .htaccess file from the root of the container... - if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) { - $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() ); - $this->trapWarnings(); - if ( $exists && !unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure() - $storeDir = "mwstore://{$this->name}/{$shortCont}"; - $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" ); - } - $this->untrapWarnings(); - } - - return $status; - } - - protected function doCleanInternal( $fullCont, $dirRel, array $params ) { - $status = Status::newGood(); - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - $this->trapWarnings(); - if ( is_dir( $dir ) ) { - rmdir( $dir ); // remove directory if empty - } - $this->untrapWarnings(); - - return $status; - } - - protected function doGetFileStat( array $params ) { - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - return false; // invalid storage path - } - - $this->trapWarnings(); // don't trust 'false' if there were errors - $stat = is_file( $source ) ? stat( $source ) : false; // regular files only - $hadError = $this->untrapWarnings(); - - if ( $stat ) { - return [ - 'mtime' => wfTimestamp( TS_MW, $stat['mtime'] ), - 'size' => $stat['size'] - ]; - } elseif ( !$hadError ) { - return false; // file does not exist - } else { - return null; // failure - } - } - - protected function doClearCache( array $paths = null ) { - clearstatcache(); // clear the PHP file stat cache - } - - protected function doDirectoryExists( $fullCont, $dirRel, array $params ) { - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - - $this->trapWarnings(); // don't trust 'false' if there were errors - $exists = is_dir( $dir ); - $hadError = $this->untrapWarnings(); - - return $hadError ? null : $exists; - } - - /** - * @see FileBackendStore::getDirectoryListInternal() - * @param string $fullCont - * @param string $dirRel - * @param array $params - * @return array|null - */ - public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) { - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - $exists = is_dir( $dir ); - if ( !$exists ) { - wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" ); - - return []; // nothing under this dir - } elseif ( !is_readable( $dir ) ) { - wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" ); - - return null; // bad permissions? - } - - return new FSFileBackendDirList( $dir, $params ); - } - - /** - * @see FileBackendStore::getFileListInternal() - * @param string $fullCont - * @param string $dirRel - * @param array $params - * @return array|FSFileBackendFileList|null - */ - public function getFileListInternal( $fullCont, $dirRel, array $params ) { - list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); - $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid - $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - $exists = is_dir( $dir ); - if ( !$exists ) { - wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" ); - - return []; // nothing under this dir - } elseif ( !is_readable( $dir ) ) { - wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" ); - - return null; // bad permissions? - } - - return new FSFileBackendFileList( $dir, $params ); - } - - protected function doGetLocalReferenceMulti( array $params ) { - $fsFiles = []; // (path => FSFile) - - foreach ( $params['srcs'] as $src ) { - $source = $this->resolveToFSPath( $src ); - if ( $source === null || !is_file( $source ) ) { - $fsFiles[$src] = null; // invalid path or file does not exist - } else { - $fsFiles[$src] = new FSFile( $source ); - } - } - - return $fsFiles; - } - - protected function doGetLocalCopyMulti( array $params ) { - $tmpFiles = []; // (path => TempFSFile) - - foreach ( $params['srcs'] as $src ) { - $source = $this->resolveToFSPath( $src ); - if ( $source === null ) { - $tmpFiles[$src] = null; // invalid path - } else { - // Create a new temporary file with the same extension... - $ext = FileBackend::extensionFromPath( $src ); - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( !$tmpFile ) { - $tmpFiles[$src] = null; - } else { - $tmpPath = $tmpFile->getPath(); - // Copy the source file over the temp file - $this->trapWarnings(); - $ok = copy( $source, $tmpPath ); - $this->untrapWarnings(); - if ( !$ok ) { - $tmpFiles[$src] = null; - } else { - $this->chmod( $tmpPath ); - $tmpFiles[$src] = $tmpFile; - } - } - } - } - - return $tmpFiles; - } - - protected function directoriesAreVirtual() { - return false; - } - - /** - * @param FSFileOpHandle[] $fileOpHandles - * - * @return Status[] - */ - protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { - $statuses = []; - - $pipes = []; - foreach ( $fileOpHandles as $index => $fileOpHandle ) { - $pipes[$index] = popen( "{$fileOpHandle->cmd} 2>&1", 'r' ); - } - - $errs = []; - foreach ( $pipes as $index => $pipe ) { - // Result will be empty on success in *NIX. On Windows, - // it may be something like " 1 file(s) [copied|moved].". - $errs[$index] = stream_get_contents( $pipe ); - fclose( $pipe ); - } - - foreach ( $fileOpHandles as $index => $fileOpHandle ) { - $status = Status::newGood(); - $function = $fileOpHandle->call; - $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd ); - $statuses[$index] = $status; - if ( $status->isOK() && $fileOpHandle->chmodPath ) { - $this->chmod( $fileOpHandle->chmodPath ); - } - } - - clearstatcache(); // files changed - return $statuses; - } - - /** - * Chmod a file, suppressing the warnings - * - * @param string $path Absolute file system path - * @return bool Success - */ - protected function chmod( $path ) { - $this->trapWarnings(); - $ok = chmod( $path, $this->fileMode ); - $this->untrapWarnings(); - - return $ok; - } - - /** - * Return the text of an index.html file to hide directory listings - * - * @return string - */ - protected function indexHtmlPrivate() { - return ''; - } - - /** - * Return the text of a .htaccess file to make a directory private - * - * @return string - */ - protected function htaccessPrivate() { - return "Deny from all\n"; - } - - /** - * Clean up directory separators for the given OS - * - * @param string $path FS path - * @return string - */ - protected function cleanPathSlashes( $path ) { - return wfIsWindows() ? strtr( $path, '/', '\\' ) : $path; - } - - /** - * Listen for E_WARNING errors and track whether any happen - */ - protected function trapWarnings() { - $this->hadWarningErrors[] = false; // push to stack - set_error_handler( [ $this, 'handleWarning' ], E_WARNING ); - } - - /** - * Stop listening for E_WARNING errors and return true if any happened - * - * @return bool - */ - protected function untrapWarnings() { - restore_error_handler(); // restore previous handler - return array_pop( $this->hadWarningErrors ); // pop from stack - } - - /** - * @param int $errno - * @param string $errstr - * @return bool - * @access private - */ - public function handleWarning( $errno, $errstr ) { - wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging - $this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true; - - return true; // suppress from PHP handler - } -} - -/** - * @see FileBackendStoreOpHandle - */ -class FSFileOpHandle extends FileBackendStoreOpHandle { - public $cmd; // string; shell command - public $chmodPath; // string; file to chmod - - /** - * @param FSFileBackend $backend - * @param array $params - * @param callable $call - * @param string $cmd - * @param int|null $chmodPath - */ - public function __construct( - FSFileBackend $backend, array $params, $call, $cmd, $chmodPath = null - ) { - $this->backend = $backend; - $this->params = $params; - $this->call = $call; - $this->cmd = $cmd; - $this->chmodPath = $chmodPath; - } -} - -/** - * Wrapper around RecursiveDirectoryIterator/DirectoryIterator that - * catches exception or does any custom behavoir that we may want. - * Do not use this class from places outside FSFileBackend. - * - * @ingroup FileBackend - */ -abstract class FSFileBackendList implements Iterator { - /** @var Iterator */ - protected $iter; - - /** @var int */ - protected $suffixStart; - - /** @var int */ - protected $pos = 0; - - /** @var array */ - protected $params = []; - - /** - * @param string $dir File system directory - * @param array $params - */ - public function __construct( $dir, array $params ) { - $path = realpath( $dir ); // normalize - if ( $path === false ) { - $path = $dir; - } - $this->suffixStart = strlen( $path ) + 1; // size of "path/to/dir/" - $this->params = $params; - - try { - $this->iter = $this->initIterator( $path ); - } catch ( UnexpectedValueException $e ) { - $this->iter = null; // bad permissions? deleted? - } - } - - /** - * Return an appropriate iterator object to wrap - * - * @param string $dir File system directory - * @return Iterator - */ - protected function initIterator( $dir ) { - if ( !empty( $this->params['topOnly'] ) ) { // non-recursive - # Get an iterator that will get direct sub-nodes - return new DirectoryIterator( $dir ); - } else { // recursive - # Get an iterator that will return leaf nodes (non-directories) - # RecursiveDirectoryIterator extends FilesystemIterator. - # FilesystemIterator::SKIP_DOTS default is inconsistent in PHP 5.3.x. - $flags = FilesystemIterator::CURRENT_AS_SELF | FilesystemIterator::SKIP_DOTS; - - return new RecursiveIteratorIterator( - new RecursiveDirectoryIterator( $dir, $flags ), - RecursiveIteratorIterator::CHILD_FIRST // include dirs - ); - } - } - - /** - * @see Iterator::key() - * @return int - */ - public function key() { - return $this->pos; - } - - /** - * @see Iterator::current() - * @return string|bool String or false - */ - public function current() { - return $this->getRelPath( $this->iter->current()->getPathname() ); - } - - /** - * @see Iterator::next() - * @throws FileBackendError - */ - public function next() { - try { - $this->iter->next(); - $this->filterViaNext(); - } catch ( UnexpectedValueException $e ) { // bad permissions? deleted? - throw new FileBackendError( "File iterator gave UnexpectedValueException." ); - } - ++$this->pos; - } - - /** - * @see Iterator::rewind() - * @throws FileBackendError - */ - public function rewind() { - $this->pos = 0; - try { - $this->iter->rewind(); - $this->filterViaNext(); - } catch ( UnexpectedValueException $e ) { // bad permissions? deleted? - throw new FileBackendError( "File iterator gave UnexpectedValueException." ); - } - } - - /** - * @see Iterator::valid() - * @return bool - */ - public function valid() { - return $this->iter && $this->iter->valid(); - } - - /** - * Filter out items by advancing to the next ones - */ - protected function filterViaNext() { - } - - /** - * Return only the relative path and normalize slashes to FileBackend-style. - * Uses the "real path" since the suffix is based upon that. - * - * @param string $dir - * @return string - */ - protected function getRelPath( $dir ) { - $path = realpath( $dir ); - if ( $path === false ) { - $path = $dir; - } - - return strtr( substr( $path, $this->suffixStart ), '\\', '/' ); - } -} - -class FSFileBackendDirList extends FSFileBackendList { - protected function filterViaNext() { - while ( $this->iter->valid() ) { - if ( $this->iter->current()->isDot() || !$this->iter->current()->isDir() ) { - $this->iter->next(); // skip non-directories and dot files - } else { - break; - } - } - } -} - -class FSFileBackendFileList extends FSFileBackendList { - protected function filterViaNext() { - while ( $this->iter->valid() ) { - if ( !$this->iter->current()->isFile() ) { - $this->iter->next(); // skip non-files and dot files - } else { - break; - } - } - } -} diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php deleted file mode 100644 index 10183f490d..0000000000 --- a/includes/filebackend/FileBackend.php +++ /dev/null @@ -1,1551 +0,0 @@ -//". - * The "backend" portion is unique name for MediaWiki to refer to a backend, while - * the "container" portion is a top-level directory of the backend. The "path" portion - * is a relative path that uses UNIX file system (FS) notation, though any particular - * backend may not actually be using a local filesystem. Therefore, the relative paths - * are only virtual. - * - * Backend contents are stored under wiki-specific container names by default. - * Global (qualified) backends are achieved by configuring the "wiki ID" to a constant. - * For legacy reasons, the FSFileBackend class allows manually setting the paths of - * containers to ones that do not respect the "wiki ID". - * - * In key/value (object) stores, containers are the only hierarchy (the rest is emulated). - * FS-based backends are somewhat more restrictive due to the existence of real - * directory files; a regular file cannot have the same name as a directory. Other - * backends with virtual directories may not have this limitation. Callers should - * store files in such a way that no files and directories are under the same path. - * - * In general, this class allows for callers to access storage through the same - * interface, without regard to the underlying storage system. However, calling code - * must follow certain patterns and be aware of certain things to ensure compatibility: - * - a) Always call prepare() on the parent directory before trying to put a file there; - * key/value stores only need the container to exist first, but filesystems need - * all the parent directories to exist first (prepare() is aware of all this) - * - b) Always call clean() on a directory when it might become empty to avoid empty - * directory buildup on filesystems; key/value stores never have empty directories, - * so doing this helps preserve consistency in both cases - * - c) Likewise, do not rely on the existence of empty directories for anything; - * calling directoryExists() on a path that prepare() was previously called on - * will return false for key/value stores if there are no files under that path - * - d) Never alter the resulting FSFile returned from getLocalReference(), as it could - * either be a copy of the source file in /tmp or the original source file itself - * - e) Use a file layout that results in never attempting to store files over directories - * or directories over files; key/value stores allow this but filesystems do not - * - f) Use ASCII file names (e.g. base32, IDs, hashes) to avoid Unicode issues in Windows - * - g) Do not assume that move operations are atomic (difficult with key/value stores) - * - h) Do not assume that file stat or read operations always have immediate consistency; - * various methods have a "latest" flag that should always be used if up-to-date - * information is required (this trades performance for correctness as needed) - * - i) Do not assume that directory listings have immediate consistency - * - * Methods of subclasses should avoid throwing exceptions at all costs. - * As a corollary, external dependencies should be kept to a minimum. - * - * @ingroup FileBackend - * @since 1.19 - */ -abstract class FileBackend { - /** @var string Unique backend name */ - protected $name; - - /** @var string Unique wiki name */ - protected $wikiId; - - /** @var string Read-only explanation message */ - protected $readOnly; - - /** @var string When to do operations in parallel */ - protected $parallelize; - - /** @var int How many operations can be done in parallel */ - protected $concurrency; - - /** @var LockManager */ - protected $lockManager; - - /** @var FileJournal */ - protected $fileJournal; - - /** Bitfield flags for supported features */ - const ATTR_HEADERS = 1; // files can be tagged with standard HTTP headers - const ATTR_METADATA = 2; // files can be stored with metadata key/values - const ATTR_UNICODE_PATHS = 4; // files can have Unicode paths (not just ASCII) - - /** - * Create a new backend instance from configuration. - * This should only be called from within FileBackendGroup. - * - * @param array $config Parameters include: - * - name : The unique name of this backend. - * This should consist of alphanumberic, '-', and '_' characters. - * This name should not be changed after use (e.g. with journaling). - * Note that the name is *not* used in actual container names. - * - wikiId : Prefix to container names that is unique to this backend. - * It should only consist of alphanumberic, '-', and '_' characters. - * This ID is what avoids collisions if multiple logical backends - * use the same storage system, so this should be set carefully. - * - lockManager : LockManager object to use for any file locking. - * If not provided, then no file locking will be enforced. - * - fileJournal : FileJournal object to use for logging changes to files. - * If not provided, then change journaling will be disabled. - * - readOnly : Write operations are disallowed if this is a non-empty string. - * It should be an explanation for the backend being read-only. - * - parallelize : When to do file operations in parallel (when possible). - * Allowed values are "implicit", "explicit" and "off". - * - concurrency : How many file operations can be done in parallel. - * @throws FileBackendException - */ - public function __construct( array $config ) { - $this->name = $config['name']; - $this->wikiId = $config['wikiId']; // e.g. "my_wiki-en_" - if ( !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) { - throw new FileBackendException( "Backend name '{$this->name}' is invalid." ); - } elseif ( !is_string( $this->wikiId ) ) { - throw new FileBackendException( "Backend wiki ID not provided for '{$this->name}'." ); - } - $this->lockManager = isset( $config['lockManager'] ) - ? $config['lockManager'] - : new NullLockManager( [] ); - $this->fileJournal = isset( $config['fileJournal'] ) - ? $config['fileJournal'] - : FileJournal::factory( [ 'class' => 'NullFileJournal' ], $this->name ); - $this->readOnly = isset( $config['readOnly'] ) - ? (string)$config['readOnly'] - : ''; - $this->parallelize = isset( $config['parallelize'] ) - ? (string)$config['parallelize'] - : 'off'; - $this->concurrency = isset( $config['concurrency'] ) - ? (int)$config['concurrency'] - : 50; - } - - /** - * Get the unique backend name. - * We may have multiple different backends of the same type. - * For example, we can have two Swift backends using different proxies. - * - * @return string - */ - final public function getName() { - return $this->name; - } - - /** - * Get the wiki identifier used for this backend (possibly empty). - * Note that this might *not* be in the same format as wfWikiID(). - * - * @return string - * @since 1.20 - */ - final public function getWikiId() { - return $this->wikiId; - } - - /** - * Check if this backend is read-only - * - * @return bool - */ - final public function isReadOnly() { - return ( $this->readOnly != '' ); - } - - /** - * Get an explanatory message if this backend is read-only - * - * @return string|bool Returns false if the backend is not read-only - */ - final public function getReadOnlyReason() { - return ( $this->readOnly != '' ) ? $this->readOnly : false; - } - - /** - * Get the a bitfield of extra features supported by the backend medium - * - * @return int Bitfield of FileBackend::ATTR_* flags - * @since 1.23 - */ - public function getFeatures() { - return self::ATTR_UNICODE_PATHS; - } - - /** - * Check if the backend medium supports a field of extra features - * - * @param int $bitfield Bitfield of FileBackend::ATTR_* flags - * @return bool - * @since 1.23 - */ - final public function hasFeatures( $bitfield ) { - return ( $this->getFeatures() & $bitfield ) === $bitfield; - } - - /** - * This is the main entry point into the backend for write operations. - * Callers supply an ordered list of operations to perform as a transaction. - * Files will be locked, the stat cache cleared, and then the operations attempted. - * If any serious errors occur, all attempted operations will be rolled back. - * - * $ops is an array of arrays. The outer array holds a list of operations. - * Each inner array is a set of key value pairs that specify an operation. - * - * Supported operations and their parameters. The supported actions are: - * - create - * - store - * - copy - * - move - * - delete - * - describe (since 1.21) - * - null - * - * FSFile/TempFSFile object support was added in 1.27. - * - * a) Create a new file in storage with the contents of a string - * @code - * array( - * 'op' => 'create', - * 'dst' => , - * 'content' => , - * 'overwrite' => , - * 'overwriteSame' => , - * 'headers' => # since 1.21 - * ); - * @endcode - * - * b) Copy a file system file into storage - * @code - * array( - * 'op' => 'store', - * 'src' => , - * 'dst' => , - * 'overwrite' => , - * 'overwriteSame' => , - * 'headers' => # since 1.21 - * ) - * @endcode - * - * c) Copy a file within storage - * @code - * array( - * 'op' => 'copy', - * 'src' => , - * 'dst' => , - * 'overwrite' => , - * 'overwriteSame' => , - * 'ignoreMissingSource' => , # since 1.21 - * 'headers' => # since 1.21 - * ) - * @endcode - * - * d) Move a file within storage - * @code - * array( - * 'op' => 'move', - * 'src' => , - * 'dst' => , - * 'overwrite' => , - * 'overwriteSame' => , - * 'ignoreMissingSource' => , # since 1.21 - * 'headers' => # since 1.21 - * ) - * @endcode - * - * e) Delete a file within storage - * @code - * array( - * 'op' => 'delete', - * 'src' => , - * 'ignoreMissingSource' => - * ) - * @endcode - * - * f) Update metadata for a file within storage - * @code - * array( - * 'op' => 'describe', - * 'src' => , - * 'headers' => - * ) - * @endcode - * - * g) Do nothing (no-op) - * @code - * array( - * 'op' => 'null', - * ) - * @endcode - * - * Boolean flags for operations (operation-specific): - * - ignoreMissingSource : The operation will simply succeed and do - * nothing if the source file does not exist. - * - overwrite : Any destination file will be overwritten. - * - overwriteSame : If a file already exists at the destination with the - * same contents, then do nothing to the destination file - * instead of giving an error. This does not compare headers. - * This option is ignored if 'overwrite' is already provided. - * - headers : If supplied, the result of merging these headers with any - * existing source file headers (replacing conflicting ones) - * will be set as the destination file headers. Headers are - * deleted if their value is set to the empty string. When a - * file has headers they are included in responses to GET and - * HEAD requests to the backing store for that file. - * Header values should be no larger than 255 bytes, except for - * Content-Disposition. The system might ignore or truncate any - * headers that are too long to store (exact limits will vary). - * Backends that don't support metadata ignore this. (since 1.21) - * - * $opts is an associative of boolean flags, including: - * - force : Operation precondition errors no longer trigger an abort. - * Any remaining operations are still attempted. Unexpected - * failures may still cause remaining operations to be aborted. - * - nonLocking : No locks are acquired for the operations. - * This can increase performance for non-critical writes. - * This has no effect unless the 'force' flag is set. - * - nonJournaled : Don't log this operation batch in the file journal. - * This limits the ability of recovery scripts. - * - parallelize : Try to do operations in parallel when possible. - * - bypassReadOnly : Allow writes in read-only mode. (since 1.20) - * - preserveCache : Don't clear the process cache before checking files. - * This should only be used if all entries in the process - * cache were added after the files were already locked. (since 1.20) - * - * @remarks Remarks on locking: - * File system paths given to operations should refer to files that are - * already locked or otherwise safe from modification from other processes. - * Normally these files will be new temp files, which should be adequate. - * - * @par Return value: - * - * This returns a Status, which contains all warnings and fatals that occurred - * during the operation. The 'failCount', 'successCount', and 'success' members - * will reflect each operation attempted. - * - * The status will be "OK" unless: - * - a) unexpected operation errors occurred (network partitions, disk full...) - * - b) significant operation errors occurred and 'force' was not set - * - * @param array $ops List of operations to execute in order - * @param array $opts Batch operation options - * @return Status - */ - final public function doOperations( array $ops, array $opts = [] ) { - if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - if ( !count( $ops ) ) { - return Status::newGood(); // nothing to do - } - - $ops = $this->resolveFSFileObjects( $ops ); - if ( empty( $opts['force'] ) ) { // sanity - unset( $opts['nonLocking'] ); - } - - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - - return $this->doOperationsInternal( $ops, $opts ); - } - - /** - * @see FileBackend::doOperations() - * @param array $ops - * @param array $opts - */ - abstract protected function doOperationsInternal( array $ops, array $opts ); - - /** - * Same as doOperations() except it takes a single operation. - * If you are doing a batch of operations that should either - * all succeed or all fail, then use that function instead. - * - * @see FileBackend::doOperations() - * - * @param array $op Operation - * @param array $opts Operation options - * @return Status - */ - final public function doOperation( array $op, array $opts = [] ) { - return $this->doOperations( [ $op ], $opts ); - } - - /** - * Performs a single create operation. - * This sets $params['op'] to 'create' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - */ - final public function create( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'create' ] + $params, $opts ); - } - - /** - * Performs a single store operation. - * This sets $params['op'] to 'store' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - */ - final public function store( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'store' ] + $params, $opts ); - } - - /** - * Performs a single copy operation. - * This sets $params['op'] to 'copy' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - */ - final public function copy( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'copy' ] + $params, $opts ); - } - - /** - * Performs a single move operation. - * This sets $params['op'] to 'move' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - */ - final public function move( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'move' ] + $params, $opts ); - } - - /** - * Performs a single delete operation. - * This sets $params['op'] to 'delete' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - */ - final public function delete( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'delete' ] + $params, $opts ); - } - - /** - * Performs a single describe operation. - * This sets $params['op'] to 'describe' and passes it to doOperation(). - * - * @see FileBackend::doOperation() - * - * @param array $params Operation parameters - * @param array $opts Operation options - * @return Status - * @since 1.21 - */ - final public function describe( array $params, array $opts = [] ) { - return $this->doOperation( [ 'op' => 'describe' ] + $params, $opts ); - } - - /** - * Perform a set of independent file operations on some files. - * - * This does no locking, nor journaling, and possibly no stat calls. - * Any destination files that already exist will be overwritten. - * This should *only* be used on non-original files, like cache files. - * - * Supported operations and their parameters: - * - create - * - store - * - copy - * - move - * - delete - * - describe (since 1.21) - * - null - * - * FSFile/TempFSFile object support was added in 1.27. - * - * a) Create a new file in storage with the contents of a string - * @code - * array( - * 'op' => 'create', - * 'dst' => , - * 'content' => , - * 'headers' => # since 1.21 - * ) - * @endcode - * - * b) Copy a file system file into storage - * @code - * array( - * 'op' => 'store', - * 'src' => , - * 'dst' => , - * 'headers' => # since 1.21 - * ) - * @endcode - * - * c) Copy a file within storage - * @code - * array( - * 'op' => 'copy', - * 'src' => , - * 'dst' => , - * 'ignoreMissingSource' => , # since 1.21 - * 'headers' => # since 1.21 - * ) - * @endcode - * - * d) Move a file within storage - * @code - * array( - * 'op' => 'move', - * 'src' => , - * 'dst' => , - * 'ignoreMissingSource' => , # since 1.21 - * 'headers' => # since 1.21 - * ) - * @endcode - * - * e) Delete a file within storage - * @code - * array( - * 'op' => 'delete', - * 'src' => , - * 'ignoreMissingSource' => - * ) - * @endcode - * - * f) Update metadata for a file within storage - * @code - * array( - * 'op' => 'describe', - * 'src' => , - * 'headers' => - * ) - * @endcode - * - * g) Do nothing (no-op) - * @code - * array( - * 'op' => 'null', - * ) - * @endcode - * - * @par Boolean flags for operations (operation-specific): - * - ignoreMissingSource : The operation will simply succeed and do - * nothing if the source file does not exist. - * - headers : If supplied with a header name/value map, the backend will - * reply with these headers when GETs/HEADs of the destination - * file are made. Header values should be smaller than 256 bytes. - * Content-Disposition headers can be longer, though the system - * might ignore or truncate ones that are too long to store. - * Existing headers will remain, but these will replace any - * conflicting previous headers, and headers will be removed - * if they are set to an empty string. - * Backends that don't support metadata ignore this. (since 1.21) - * - * $opts is an associative of boolean flags, including: - * - bypassReadOnly : Allow writes in read-only mode (since 1.20) - * - * @par Return value: - * This returns a Status, which contains all warnings and fatals that occurred - * during the operation. The 'failCount', 'successCount', and 'success' members - * will reflect each operation attempted for the given files. The status will be - * considered "OK" as long as no fatal errors occurred. - * - * @param array $ops Set of operations to execute - * @param array $opts Batch operation options - * @return Status - * @since 1.20 - */ - final public function doQuickOperations( array $ops, array $opts = [] ) { - if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - if ( !count( $ops ) ) { - return Status::newGood(); // nothing to do - } - - $ops = $this->resolveFSFileObjects( $ops ); - foreach ( $ops as &$op ) { - $op['overwrite'] = true; // avoids RTTs in key/value stores - } - - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - - return $this->doQuickOperationsInternal( $ops ); - } - - /** - * @see FileBackend::doQuickOperations() - * @param array $ops - * @since 1.20 - */ - abstract protected function doQuickOperationsInternal( array $ops ); - - /** - * Same as doQuickOperations() except it takes a single operation. - * If you are doing a batch of operations, then use that function instead. - * - * @see FileBackend::doQuickOperations() - * - * @param array $op Operation - * @return Status - * @since 1.20 - */ - final public function doQuickOperation( array $op ) { - return $this->doQuickOperations( [ $op ] ); - } - - /** - * Performs a single quick create operation. - * This sets $params['op'] to 'create' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.20 - */ - final public function quickCreate( array $params ) { - return $this->doQuickOperation( [ 'op' => 'create' ] + $params ); - } - - /** - * Performs a single quick store operation. - * This sets $params['op'] to 'store' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.20 - */ - final public function quickStore( array $params ) { - return $this->doQuickOperation( [ 'op' => 'store' ] + $params ); - } - - /** - * Performs a single quick copy operation. - * This sets $params['op'] to 'copy' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.20 - */ - final public function quickCopy( array $params ) { - return $this->doQuickOperation( [ 'op' => 'copy' ] + $params ); - } - - /** - * Performs a single quick move operation. - * This sets $params['op'] to 'move' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.20 - */ - final public function quickMove( array $params ) { - return $this->doQuickOperation( [ 'op' => 'move' ] + $params ); - } - - /** - * Performs a single quick delete operation. - * This sets $params['op'] to 'delete' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.20 - */ - final public function quickDelete( array $params ) { - return $this->doQuickOperation( [ 'op' => 'delete' ] + $params ); - } - - /** - * Performs a single quick describe operation. - * This sets $params['op'] to 'describe' and passes it to doQuickOperation(). - * - * @see FileBackend::doQuickOperation() - * - * @param array $params Operation parameters - * @return Status - * @since 1.21 - */ - final public function quickDescribe( array $params ) { - return $this->doQuickOperation( [ 'op' => 'describe' ] + $params ); - } - - /** - * Concatenate a list of storage files into a single file system file. - * The target path should refer to a file that is already locked or - * otherwise safe from modification from other processes. Normally, - * the file will be a new temp file, which should be adequate. - * - * @param array $params Operation parameters, include: - * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) - * - dst : file system path to 0-byte temp file - * - parallelize : try to do operations in parallel when possible - * @return Status - */ - abstract public function concatenate( array $params ); - - /** - * Prepare a storage directory for usage. - * This will create any required containers and parent directories. - * Backends using key/value stores only need to create the container. - * - * The 'noAccess' and 'noListing' parameters works the same as in secure(), - * except they are only applied *if* the directory/container had to be created. - * These flags should always be set for directories that have private files. - * However, setting them is not guaranteed to actually do anything. - * Additional server configuration may be needed to achieve the desired effect. - * - * @param array $params Parameters include: - * - dir : storage directory - * - noAccess : try to deny file access (since 1.20) - * - noListing : try to deny file listing (since 1.20) - * - bypassReadOnly : allow writes in read-only mode (since 1.20) - * @return Status - */ - final public function prepare( array $params ) { - if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - return $this->doPrepare( $params ); - } - - /** - * @see FileBackend::prepare() - * @param array $params - */ - abstract protected function doPrepare( array $params ); - - /** - * Take measures to block web access to a storage directory and - * the container it belongs to. FS backends might add .htaccess - * files whereas key/value store backends might revoke container - * access to the storage user representing end-users in web requests. - * - * This is not guaranteed to actually make files or listings publically hidden. - * Additional server configuration may be needed to achieve the desired effect. - * - * @param array $params Parameters include: - * - dir : storage directory - * - noAccess : try to deny file access - * - noListing : try to deny file listing - * - bypassReadOnly : allow writes in read-only mode (since 1.20) - * @return Status - */ - final public function secure( array $params ) { - if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - return $this->doSecure( $params ); - } - - /** - * @see FileBackend::secure() - * @param array $params - */ - abstract protected function doSecure( array $params ); - - /** - * Remove measures to block web access to a storage directory and - * the container it belongs to. FS backends might remove .htaccess - * files whereas key/value store backends might grant container - * access to the storage user representing end-users in web requests. - * This essentially can undo the result of secure() calls. - * - * This is not guaranteed to actually make files or listings publically viewable. - * Additional server configuration may be needed to achieve the desired effect. - * - * @param array $params Parameters include: - * - dir : storage directory - * - access : try to allow file access - * - listing : try to allow file listing - * - bypassReadOnly : allow writes in read-only mode (since 1.20) - * @return Status - * @since 1.20 - */ - final public function publish( array $params ) { - if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - return $this->doPublish( $params ); - } - - /** - * @see FileBackend::publish() - * @param array $params - */ - abstract protected function doPublish( array $params ); - - /** - * Delete a storage directory if it is empty. - * Backends using key/value stores may do nothing unless the directory - * is that of an empty container, in which case it will be deleted. - * - * @param array $params Parameters include: - * - dir : storage directory - * - recursive : recursively delete empty subdirectories first (since 1.20) - * - bypassReadOnly : allow writes in read-only mode (since 1.20) - * @return Status - */ - final public function clean( array $params ) { - if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { - return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); - } - /** @noinspection PhpUnusedLocalVariableInspection */ - $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts - return $this->doClean( $params ); - } - - /** - * @see FileBackend::clean() - * @param array $params - */ - abstract protected function doClean( array $params ); - - /** - * Enter file operation scope. - * This just makes PHP ignore user aborts/disconnects until the return - * value leaves scope. This returns null and does nothing in CLI mode. - * - * @return ScopedCallback|null - */ - final protected function getScopedPHPBehaviorForOps() { - if ( PHP_SAPI != 'cli' ) { // http://bugs.php.net/bug.php?id=47540 - $old = ignore_user_abort( true ); // avoid half-finished operations - return new ScopedCallback( function () use ( $old ) { - ignore_user_abort( $old ); - } ); - } - - return null; - } - - /** - * Check if a file exists at a storage path in the backend. - * This returns false if only a directory exists at the path. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return bool|null Returns null on failure - */ - abstract public function fileExists( array $params ); - - /** - * Get the last-modified timestamp of the file at a storage path. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return string|bool TS_MW timestamp or false on failure - */ - abstract public function getFileTimestamp( array $params ); - - /** - * Get the contents of a file at a storage path in the backend. - * This should be avoided for potentially large files. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return string|bool Returns false on failure - */ - final public function getFileContents( array $params ) { - $contents = $this->getFileContentsMulti( - [ 'srcs' => [ $params['src'] ] ] + $params ); - - return $contents[$params['src']]; - } - - /** - * Like getFileContents() except it takes an array of storage paths - * and returns a map of storage paths to strings (or null on failure). - * The map keys (paths) are in the same order as the provided list of paths. - * - * @see FileBackend::getFileContents() - * - * @param array $params Parameters include: - * - srcs : list of source storage paths - * - latest : use the latest available data - * - parallelize : try to do operations in parallel when possible - * @return array Map of (path name => string or false on failure) - * @since 1.20 - */ - abstract public function getFileContentsMulti( array $params ); - - /** - * Get metadata about a file at a storage path in the backend. - * If the file does not exist, then this returns false. - * Otherwise, the result is an associative array that includes: - * - headers : map of HTTP headers used for GET/HEAD requests (name => value) - * - metadata : map of file metadata (name => value) - * Metadata keys and headers names will be returned in all lower-case. - * Additional values may be included for internal use only. - * - * Use FileBackend::hasFeatures() to check how well this is supported. - * - * @param array $params - * $params include: - * - src : source storage path - * - latest : use the latest available data - * @return array|bool Returns false on failure - * @since 1.23 - */ - abstract public function getFileXAttributes( array $params ); - - /** - * Get the size (bytes) of a file at a storage path in the backend. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return int|bool Returns false on failure - */ - abstract public function getFileSize( array $params ); - - /** - * Get quick information about a file at a storage path in the backend. - * If the file does not exist, then this returns false. - * Otherwise, the result is an associative array that includes: - * - mtime : the last-modified timestamp (TS_MW) - * - size : the file size (bytes) - * Additional values may be included for internal use only. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return array|bool|null Returns null on failure - */ - abstract public function getFileStat( array $params ); - - /** - * Get a SHA-1 hash of the file at a storage path in the backend. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return string|bool Hash string or false on failure - */ - abstract public function getFileSha1Base36( array $params ); - - /** - * Get the properties of the file at a storage path in the backend. - * This gives the result of FSFile::getProps() on a local copy of the file. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return array Returns FSFile::placeholderProps() on failure - */ - abstract public function getFileProps( array $params ); - - /** - * Stream the file at a storage path in the backend. - * - * If the file does not exists, an HTTP 404 error will be given. - * Appropriate HTTP headers (Status, Content-Type, Content-Length) - * will be sent if streaming began, while none will be sent otherwise. - * Implementations should flush the output buffer before sending data. - * - * @param array $params Parameters include: - * - src : source storage path - * - headers : list of additional HTTP headers to send if the file exists - * - options : HTTP request header map with lower case keys (since 1.28). Supports: - * range : format is "bytes=(\d*-\d*)" - * if-modified-since : format is an HTTP date - * - headless : only include the body (and headers from "headers") (since 1.28) - * - latest : use the latest available data - * - allowOB : preserve any output buffers (since 1.28) - * @return Status - */ - abstract public function streamFile( array $params ); - - /** - * Returns a file system file, identical to the file at a storage path. - * The file returned is either: - * - a) A local copy of the file at a storage path in the backend. - * The temporary copy will have the same extension as the source. - * - b) An original of the file at a storage path in the backend. - * Temporary files may be purged when the file object falls out of scope. - * - * Write operations should *never* be done on this file as some backends - * may do internal tracking or may be instances of FileBackendMultiWrite. - * In that later case, there are copies of the file that must stay in sync. - * Additionally, further calls to this function may return the same file. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return FSFile|null Returns null on failure - */ - final public function getLocalReference( array $params ) { - $fsFiles = $this->getLocalReferenceMulti( - [ 'srcs' => [ $params['src'] ] ] + $params ); - - return $fsFiles[$params['src']]; - } - - /** - * Like getLocalReference() except it takes an array of storage paths - * and returns a map of storage paths to FSFile objects (or null on failure). - * The map keys (paths) are in the same order as the provided list of paths. - * - * @see FileBackend::getLocalReference() - * - * @param array $params Parameters include: - * - srcs : list of source storage paths - * - latest : use the latest available data - * - parallelize : try to do operations in parallel when possible - * @return array Map of (path name => FSFile or null on failure) - * @since 1.20 - */ - abstract public function getLocalReferenceMulti( array $params ); - - /** - * Get a local copy on disk of the file at a storage path in the backend. - * The temporary copy will have the same file extension as the source. - * Temporary files may be purged when the file object falls out of scope. - * - * @param array $params Parameters include: - * - src : source storage path - * - latest : use the latest available data - * @return TempFSFile|null Returns null on failure - */ - final public function getLocalCopy( array $params ) { - $tmpFiles = $this->getLocalCopyMulti( - [ 'srcs' => [ $params['src'] ] ] + $params ); - - return $tmpFiles[$params['src']]; - } - - /** - * Like getLocalCopy() except it takes an array of storage paths and - * returns a map of storage paths to TempFSFile objects (or null on failure). - * The map keys (paths) are in the same order as the provided list of paths. - * - * @see FileBackend::getLocalCopy() - * - * @param array $params Parameters include: - * - srcs : list of source storage paths - * - latest : use the latest available data - * - parallelize : try to do operations in parallel when possible - * @return array Map of (path name => TempFSFile or null on failure) - * @since 1.20 - */ - abstract public function getLocalCopyMulti( array $params ); - - /** - * Return an HTTP URL to a given file that requires no authentication to use. - * The URL may be pre-authenticated (via some token in the URL) and temporary. - * This will return null if the backend cannot make an HTTP URL for the file. - * - * This is useful for key/value stores when using scripts that seek around - * large files and those scripts (and the backend) support HTTP Range headers. - * Otherwise, one would need to use getLocalReference(), which involves loading - * the entire file on to local disk. - * - * @param array $params Parameters include: - * - src : source storage path - * - ttl : lifetime (seconds) if pre-authenticated; default is 1 day - * @return string|null - * @since 1.21 - */ - abstract public function getFileHttpUrl( array $params ); - - /** - * Check if a directory exists at a given storage path. - * Backends using key/value stores will check if the path is a - * virtual directory, meaning there are files under the given directory. - * - * Storage backends with eventual consistency might return stale data. - * - * @param array $params Parameters include: - * - dir : storage directory - * @return bool|null Returns null on failure - * @since 1.20 - */ - abstract public function directoryExists( array $params ); - - /** - * Get an iterator to list *all* directories under a storage directory. - * If the directory is of the form "mwstore://backend/container", - * then all directories in the container will be listed. - * If the directory is of form "mwstore://backend/container/dir", - * then all directories directly under that directory will be listed. - * Results will be storage directories relative to the given directory. - * - * Storage backends with eventual consistency might return stale data. - * - * Failures during iteration can result in FileBackendError exceptions (since 1.22). - * - * @param array $params Parameters include: - * - dir : storage directory - * - topOnly : only return direct child dirs of the directory - * @return Traversable|array|null Returns null on failure - * @since 1.20 - */ - abstract public function getDirectoryList( array $params ); - - /** - * Same as FileBackend::getDirectoryList() except only lists - * directories that are immediately under the given directory. - * - * Storage backends with eventual consistency might return stale data. - * - * Failures during iteration can result in FileBackendError exceptions (since 1.22). - * - * @param array $params Parameters include: - * - dir : storage directory - * @return Traversable|array|null Returns null on failure - * @since 1.20 - */ - final public function getTopDirectoryList( array $params ) { - return $this->getDirectoryList( [ 'topOnly' => true ] + $params ); - } - - /** - * Get an iterator to list *all* stored files under a storage directory. - * If the directory is of the form "mwstore://backend/container", - * then all files in the container will be listed. - * If the directory is of form "mwstore://backend/container/dir", - * then all files under that directory will be listed. - * Results will be storage paths relative to the given directory. - * - * Storage backends with eventual consistency might return stale data. - * - * Failures during iteration can result in FileBackendError exceptions (since 1.22). - * - * @param array $params Parameters include: - * - dir : storage directory - * - topOnly : only return direct child files of the directory (since 1.20) - * - adviseStat : set to true if stat requests will be made on the files (since 1.22) - * @return Traversable|array|null Returns null on failure - */ - abstract public function getFileList( array $params ); - - /** - * Same as FileBackend::getFileList() except only lists - * files that are immediately under the given directory. - * - * Storage backends with eventual consistency might return stale data. - * - * Failures during iteration can result in FileBackendError exceptions (since 1.22). - * - * @param array $params Parameters include: - * - dir : storage directory - * - adviseStat : set to true if stat requests will be made on the files (since 1.22) - * @return Traversable|array|null Returns null on failure - * @since 1.20 - */ - final public function getTopFileList( array $params ) { - return $this->getFileList( [ 'topOnly' => true ] + $params ); - } - - /** - * Preload persistent file stat cache and property cache into in-process cache. - * This should be used when stat calls will be made on a known list of a many files. - * - * @see FileBackend::getFileStat() - * - * @param array $paths Storage paths - */ - abstract public function preloadCache( array $paths ); - - /** - * Invalidate any in-process file stat and property cache. - * If $paths is given, then only the cache for those files will be cleared. - * - * @see FileBackend::getFileStat() - * - * @param array $paths Storage paths (optional) - */ - abstract public function clearCache( array $paths = null ); - - /** - * Preload file stat information (concurrently if possible) into in-process cache. - * - * This should be used when stat calls will be made on a known list of a many files. - * This does not make use of the persistent file stat cache. - * - * @see FileBackend::getFileStat() - * - * @param array $params Parameters include: - * - srcs : list of source storage paths - * - latest : use the latest available data - * @return bool All requests proceeded without I/O errors (since 1.24) - * @since 1.23 - */ - abstract public function preloadFileStat( array $params ); - - /** - * Lock the files at the given storage paths in the backend. - * This will either lock all the files or none (on failure). - * - * Callers should consider using getScopedFileLocks() instead. - * - * @param array $paths Storage paths - * @param int $type LockManager::LOCK_* constant - * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24) - * @return Status - */ - final public function lockFiles( array $paths, $type, $timeout = 0 ) { - $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); - - return $this->lockManager->lock( $paths, $type, $timeout ); - } - - /** - * Unlock the files at the given storage paths in the backend. - * - * @param array $paths Storage paths - * @param int $type LockManager::LOCK_* constant - * @return Status - */ - final public function unlockFiles( array $paths, $type ) { - $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); - - return $this->lockManager->unlock( $paths, $type ); - } - - /** - * Lock the files at the given storage paths in the backend. - * This will either lock all the files or none (on failure). - * On failure, the status object will be updated with errors. - * - * Once the return value goes out scope, the locks will be released and - * the status updated. Unlock fatals will not change the status "OK" value. - * - * @see ScopedLock::factory() - * - * @param array $paths List of storage paths or map of lock types to path lists - * @param int|string $type LockManager::LOCK_* constant or "mixed" - * @param Status $status Status to update on lock/unlock - * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24) - * @return ScopedLock|null Returns null on failure - */ - final public function getScopedFileLocks( array $paths, $type, Status $status, $timeout = 0 ) { - if ( $type === 'mixed' ) { - foreach ( $paths as &$typePaths ) { - $typePaths = array_map( 'FileBackend::normalizeStoragePath', $typePaths ); - } - } else { - $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); - } - - return ScopedLock::factory( $this->lockManager, $paths, $type, $status, $timeout ); - } - - /** - * Get an array of scoped locks needed for a batch of file operations. - * - * Normally, FileBackend::doOperations() handles locking, unless - * the 'nonLocking' param is passed in. This function is useful if you - * want the files to be locked for a broader scope than just when the - * files are changing. For example, if you need to update DB metadata, - * you may want to keep the files locked until finished. - * - * @see FileBackend::doOperations() - * - * @param array $ops List of file operations to FileBackend::doOperations() - * @param Status $status Status to update on lock/unlock - * @return ScopedLock|null - * @since 1.20 - */ - abstract public function getScopedLocksForOps( array $ops, Status $status ); - - /** - * Get the root storage path of this backend. - * All container paths are "subdirectories" of this path. - * - * @return string Storage path - * @since 1.20 - */ - final public function getRootStoragePath() { - return "mwstore://{$this->name}"; - } - - /** - * Get the storage path for the given container for this backend - * - * @param string $container Container name - * @return string Storage path - * @since 1.21 - */ - final public function getContainerStoragePath( $container ) { - return $this->getRootStoragePath() . "/{$container}"; - } - - /** - * Get the file journal object for this backend - * - * @return FileJournal - */ - final public function getJournal() { - return $this->fileJournal; - } - - /** - * Convert FSFile 'src' paths to string paths (with an 'srcRef' field set to the FSFile) - * - * The 'srcRef' field keeps any TempFSFile objects in scope for the backend to have it - * around as long it needs (which may vary greatly depending on configuration) - * - * @param array $ops File operation batch for FileBaclend::doOperations() - * @return array File operation batch - */ - protected function resolveFSFileObjects( array $ops ) { - foreach ( $ops as &$op ) { - $src = isset( $op['src'] ) ? $op['src'] : null; - if ( $src instanceof FSFile ) { - $op['srcRef'] = $src; - $op['src'] = $src->getPath(); - } - } - unset( $op ); - - return $ops; - } - - /** - * Check if a given path is a "mwstore://" path. - * This does not do any further validation or any existence checks. - * - * @param string $path - * @return bool - */ - final public static function isStoragePath( $path ) { - return ( strpos( $path, 'mwstore://' ) === 0 ); - } - - /** - * Split a storage path into a backend name, a container name, - * and a relative file path. The relative path may be the empty string. - * This does not do any path normalization or traversal checks. - * - * @param string $storagePath - * @return array (backend, container, rel object) or (null, null, null) - */ - final public static function splitStoragePath( $storagePath ) { - if ( self::isStoragePath( $storagePath ) ) { - // Remove the "mwstore://" prefix and split the path - $parts = explode( '/', substr( $storagePath, 10 ), 3 ); - if ( count( $parts ) >= 2 && $parts[0] != '' && $parts[1] != '' ) { - if ( count( $parts ) == 3 ) { - return $parts; // e.g. "backend/container/path" - } else { - return [ $parts[0], $parts[1], '' ]; // e.g. "backend/container" - } - } - } - - return [ null, null, null ]; - } - - /** - * Normalize a storage path by cleaning up directory separators. - * Returns null if the path is not of the format of a valid storage path. - * - * @param string $storagePath - * @return string|null - */ - final public static function normalizeStoragePath( $storagePath ) { - list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); - if ( $relPath !== null ) { // must be for this backend - $relPath = self::normalizeContainerPath( $relPath ); - if ( $relPath !== null ) { - return ( $relPath != '' ) - ? "mwstore://{$backend}/{$container}/{$relPath}" - : "mwstore://{$backend}/{$container}"; - } - } - - return null; - } - - /** - * Get the parent storage directory of a storage path. - * This returns a path like "mwstore://backend/container", - * "mwstore://backend/container/...", or null if there is no parent. - * - * @param string $storagePath - * @return string|null - */ - final public static function parentStoragePath( $storagePath ) { - $storagePath = dirname( $storagePath ); - list( , , $rel ) = self::splitStoragePath( $storagePath ); - - return ( $rel === null ) ? null : $storagePath; - } - - /** - * Get the final extension from a storage or FS path - * - * @param string $path - * @param string $case One of (rawcase, uppercase, lowercase) (since 1.24) - * @return string - */ - final public static function extensionFromPath( $path, $case = 'lowercase' ) { - $i = strrpos( $path, '.' ); - $ext = $i ? substr( $path, $i + 1 ) : ''; - - if ( $case === 'lowercase' ) { - $ext = strtolower( $ext ); - } elseif ( $case === 'uppercase' ) { - $ext = strtoupper( $ext ); - } - - return $ext; - } - - /** - * Check if a relative path has no directory traversals - * - * @param string $path - * @return bool - * @since 1.20 - */ - final public static function isPathTraversalFree( $path ) { - return ( self::normalizeContainerPath( $path ) !== null ); - } - - /** - * Build a Content-Disposition header value per RFC 6266. - * - * @param string $type One of (attachment, inline) - * @param string $filename Suggested file name (should not contain slashes) - * @throws FileBackendError - * @return string - * @since 1.20 - */ - final public static function makeContentDisposition( $type, $filename = '' ) { - $parts = []; - - $type = strtolower( $type ); - if ( !in_array( $type, [ 'inline', 'attachment' ] ) ) { - throw new FileBackendError( "Invalid Content-Disposition type '$type'." ); - } - $parts[] = $type; - - if ( strlen( $filename ) ) { - $parts[] = "filename*=UTF-8''" . rawurlencode( basename( $filename ) ); - } - - return implode( ';', $parts ); - } - - /** - * Validate and normalize a relative storage path. - * Null is returned if the path involves directory traversal. - * Traversal is insecure for FS backends and broken for others. - * - * This uses the same traversal protection as Title::secureAndSplit(). - * - * @param string $path Storage path relative to a container - * @return string|null - */ - final protected static function normalizeContainerPath( $path ) { - // Normalize directory separators - $path = strtr( $path, '\\', '/' ); - // Collapse any consecutive directory separators - $path = preg_replace( '![/]{2,}!', '/', $path ); - // Remove any leading directory separator - $path = ltrim( $path, '/' ); - // Use the same traversal protection as Title::secureAndSplit() - if ( strpos( $path, '.' ) !== false ) { - if ( - $path === '.' || - $path === '..' || - strpos( $path, './' ) === 0 || - strpos( $path, '../' ) === 0 || - strpos( $path, '/./' ) !== false || - strpos( $path, '/../' ) !== false - ) { - return null; - } - } - - return $path; - } -} - -/** - * Generic file backend exception for checked and unexpected (e.g. config) exceptions - * - * @ingroup FileBackend - * @since 1.23 - */ -class FileBackendException extends Exception { -} - -/** - * File backend exception for checked exceptions (e.g. I/O errors) - * - * @ingroup FileBackend - * @since 1.22 - */ -class FileBackendError extends FileBackendException { -} diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php index 57461a48ea..e65a5945ff 100644 --- a/includes/filebackend/FileBackendGroup.php +++ b/includes/filebackend/FileBackendGroup.php @@ -21,6 +21,8 @@ * @ingroup FileBackend * @author Aaron Schulz */ +use \MediaWiki\Logger\LoggerFactory; +use MediaWiki\MediaWikiServices; /** * Class to handle file backend registration @@ -61,7 +63,7 @@ class FileBackendGroup { * Register file backends from the global variables */ protected function initFromGlobals() { - global $wgLocalFileRepo, $wgForeignFileRepos, $wgFileBackends; + global $wgLocalFileRepo, $wgForeignFileRepos, $wgFileBackends, $wgDirectoryMode; // Register explicitly defined backends $this->register( $wgFileBackends, wfConfiguredReadOnlyReason() ); @@ -86,9 +88,6 @@ class FileBackendGroup { $transcodedDir = isset( $info['transcodedDir'] ) ? $info['transcodedDir'] : "{$directory}/transcoded"; - $fileMode = isset( $info['fileMode'] ) - ? $info['fileMode'] - : 0644; // Get the FS backend configuration $autoBackends[] = [ 'name' => $backendName, @@ -101,7 +100,8 @@ class FileBackendGroup { "{$repoName}-deleted" => $deletedDir, "{$repoName}-temp" => "{$directory}/temp" ], - 'fileMode' => $fileMode, + 'fileMode' => isset( $info['fileMode'] ) ? $info['fileMode'] : 0644, + 'directoryMode' => $wgDirectoryMode, ]; } @@ -114,18 +114,18 @@ class FileBackendGroup { * * @param array $configs * @param string|null $readOnlyReason - * @throws FileBackendException + * @throws InvalidArgumentException */ protected function register( array $configs, $readOnlyReason = null ) { foreach ( $configs as $config ) { if ( !isset( $config['name'] ) ) { - throw new FileBackendException( "Cannot register a backend with no name." ); + throw new InvalidArgumentException( "Cannot register a backend with no name." ); } $name = $config['name']; if ( isset( $this->backends[$name] ) ) { - throw new FileBackendException( "Backend with name `{$name}` already registered." ); + throw new LogicException( "Backend with name `{$name}` already registered." ); } elseif ( !isset( $config['class'] ) ) { - throw new FileBackendException( "Backend with name `{$name}` has no class." ); + throw new InvalidArgumentException( "Backend with name `{$name}` has no class." ); } $class = $config['class']; @@ -147,26 +147,23 @@ class FileBackendGroup { * * @param string $name * @return FileBackend - * @throws FileBackendException + * @throws InvalidArgumentException */ public function get( $name ) { - if ( !isset( $this->backends[$name] ) ) { - throw new FileBackendException( "No backend defined with the name `$name`." ); - } // Lazy-load the actual backend instance if ( !isset( $this->backends[$name]['instance'] ) ) { - $class = $this->backends[$name]['class']; - $config = $this->backends[$name]['config']; - $config['wikiId'] = isset( $config['wikiId'] ) - ? $config['wikiId'] - : wfWikiID(); // e.g. "my_wiki-en_" - $config['lockManager'] = - LockManagerGroup::singleton( $config['wikiId'] )->get( $config['lockManager'] ); - $config['fileJournal'] = isset( $config['fileJournal'] ) - ? FileJournal::factory( $config['fileJournal'], $name ) - : FileJournal::factory( [ 'class' => 'NullFileJournal' ], $name ); - $config['wanCache'] = ObjectCache::getMainWANInstance(); - $config['mimeCallback'] = [ $this, 'guessMimeInternal' ]; + $config = $this->config( $name ); + + $class = $config['class']; + if ( $class === 'FileBackendMultiWrite' ) { + foreach ( $config['backends'] as $index => $beConfig ) { + if ( isset( $beConfig['template'] ) ) { + // Config is just a modified version of a registered backend's. + // This should only be used when that config is used only by this backend. + $config['backends'][$index] += $this->config( $beConfig['template'] ); + } + } + } $this->backends[$name]['instance'] = new $class( $config ); } @@ -178,16 +175,36 @@ class FileBackendGroup { * Get the config array for a backend object with a given name * * @param string $name - * @return array - * @throws FileBackendException + * @return array Parameters to FileBackend::__construct() + * @throws InvalidArgumentException */ public function config( $name ) { if ( !isset( $this->backends[$name] ) ) { - throw new FileBackendException( "No backend defined with the name `$name`." ); + throw new InvalidArgumentException( "No backend defined with the name `$name`." ); } $class = $this->backends[$name]['class']; - return [ 'class' => $class ] + $this->backends[$name]['config']; + $config = $this->backends[$name]['config']; + $config['class'] = $class; + $config += [ // set defaults + 'wikiId' => wfWikiID(), // e.g. "my_wiki-en_" + 'mimeCallback' => [ $this, 'guessMimeInternal' ], + 'obResetFunc' => 'wfResetOutputBuffers', + 'streamMimeFunc' => [ 'StreamFile', 'contentTypeFromPath' ], + 'tmpDirectory' => wfTempDir(), + 'statusWrapper' => [ 'Status', 'wrap' ], + 'wanCache' => MediaWikiServices::getInstance()->getMainWANObjectCache(), + 'srvCache' => ObjectCache::getLocalServerInstance( 'hash' ), + 'logger' => LoggerFactory::getInstance( 'FileOperation' ), + 'profiler' => Profiler::instance() + ]; + $config['lockManager'] = + LockManagerGroup::singleton( $config['wikiId'] )->get( $config['lockManager'] ); + $config['fileJournal'] = isset( $config['fileJournal'] ) + ? FileJournal::factory( $config['fileJournal'], $name ) + : FileJournal::factory( [ 'class' => 'NullFileJournal' ], $name ); + + return $config; } /** @@ -221,7 +238,7 @@ class FileBackendGroup { if ( !$type && $fsPath ) { $type = $magic->guessMimeType( $fsPath, false ); } elseif ( !$type && strlen( $content ) ) { - $tmpFile = TempFSFile::factory( 'mime_' ); + $tmpFile = TempFSFile::factory( 'mime_', '', wfTempDir() ); file_put_contents( $tmpFile->getPath(), $content ); $type = $magic->guessMimeType( $tmpFile->getPath(), false ); } diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php deleted file mode 100644 index 3b2004827d..0000000000 --- a/includes/filebackend/FileBackendMultiWrite.php +++ /dev/null @@ -1,761 +0,0 @@ -syncChecks = isset( $config['syncChecks'] ) - ? $config['syncChecks'] - : self::CHECK_SIZE; - $this->autoResync = isset( $config['autoResync'] ) - ? $config['autoResync'] - : false; - $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async'; - // Construct backends here rather than via registration - // to keep these backends hidden from outside the proxy. - $namesUsed = []; - foreach ( $config['backends'] as $index => $config ) { - if ( isset( $config['template'] ) ) { - // Config is just a modified version of a registered backend's. - // This should only be used when that config is used only by this backend. - $config = $config + FileBackendGroup::singleton()->config( $config['template'] ); - } - $name = $config['name']; - if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates - throw new FileBackendError( "Two or more backends defined with the name $name." ); - } - $namesUsed[$name] = 1; - // Alter certain sub-backend settings for sanity - unset( $config['readOnly'] ); // use proxy backend setting - unset( $config['fileJournal'] ); // use proxy backend journal - unset( $config['lockManager'] ); // lock under proxy backend - $config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID - if ( !empty( $config['isMultiMaster'] ) ) { - if ( $this->masterIndex >= 0 ) { - throw new FileBackendError( 'More than one master backend defined.' ); - } - $this->masterIndex = $index; // this is the "master" - $config['fileJournal'] = $this->fileJournal; // log under proxy backend - } - if ( !empty( $config['readAffinity'] ) ) { - $this->readIndex = $index; // prefer this for reads - } - // Create sub-backend object - if ( !isset( $config['class'] ) ) { - throw new FileBackendError( 'No class given for a backend config.' ); - } - $class = $config['class']; - $this->backends[$index] = new $class( $config ); - } - if ( $this->masterIndex < 0 ) { // need backends and must have a master - throw new FileBackendError( 'No master backend defined.' ); - } - if ( $this->readIndex < 0 ) { - $this->readIndex = $this->masterIndex; // default - } - } - - final protected function doOperationsInternal( array $ops, array $opts ) { - $status = Status::newGood(); - - $mbe = $this->backends[$this->masterIndex]; // convenience - - // Try to lock those files for the scope of this function... - $scopeLock = null; - if ( empty( $opts['nonLocking'] ) ) { - // Try to lock those files for the scope of this function... - /** @noinspection PhpUnusedLocalVariableInspection */ - $scopeLock = $this->getScopedLocksForOps( $ops, $status ); - if ( !$status->isOK() ) { - return $status; // abort - } - } - // Clear any cache entries (after locks acquired) - $this->clearCache(); - $opts['preserveCache'] = true; // only locked files are cached - // Get the list of paths to read/write... - $relevantPaths = $this->fileStoragePathsForOps( $ops ); - // Check if the paths are valid and accessible on all backends... - $status->merge( $this->accessibilityCheck( $relevantPaths ) ); - if ( !$status->isOK() ) { - return $status; // abort - } - // Do a consistency check to see if the backends are consistent... - $syncStatus = $this->consistencyCheck( $relevantPaths ); - if ( !$syncStatus->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . - " failed sync check: " . FormatJson::encode( $relevantPaths ) ); - // Try to resync the clone backends to the master on the spot... - if ( $this->autoResync === false - || !$this->resyncFiles( $relevantPaths, $this->autoResync )->isOK() - ) { - $status->merge( $syncStatus ); - - return $status; // abort - } - } - // Actually attempt the operation batch on the master backend... - $realOps = $this->substOpBatchPaths( $ops, $mbe ); - $masterStatus = $mbe->doOperations( $realOps, $opts ); - $status->merge( $masterStatus ); - // Propagate the operations to the clone backends if there were no unexpected errors - // and if there were either no expected errors or if the 'force' option was used. - // However, if nothing succeeded at all, then don't replicate any of the operations. - // If $ops only had one operation, this might avoid backend sync inconsistencies. - if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) { - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // done already - } - - $realOps = $this->substOpBatchPaths( $ops, $backend ); - if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { - // Bind $scopeLock to the callback to preserve locks - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $realOps, $opts, $scopeLock, $relevantPaths ) { - wfDebugLog( 'FileOperationReplication', - "'{$backend->getName()}' async replication; paths: " . - FormatJson::encode( $relevantPaths ) ); - $backend->doOperations( $realOps, $opts ); - } - ); - } else { - wfDebugLog( 'FileOperationReplication', - "'{$backend->getName()}' sync replication; paths: " . - FormatJson::encode( $relevantPaths ) ); - $status->merge( $backend->doOperations( $realOps, $opts ) ); - } - } - } - // Make 'success', 'successCount', and 'failCount' fields reflect - // the overall operation, rather than all the batches for each backend. - // Do this by only using success values from the master backend's batch. - $status->success = $masterStatus->success; - $status->successCount = $masterStatus->successCount; - $status->failCount = $masterStatus->failCount; - - return $status; - } - - /** - * Check that a set of files are consistent across all internal backends - * - * @param array $paths List of storage paths - * @return Status - */ - public function consistencyCheck( array $paths ) { - $status = Status::newGood(); - if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) { - return $status; // skip checks - } - - // Preload all of the stat info in as few round trips as possible... - foreach ( $this->backends as $backend ) { - $realPaths = $this->substPaths( $paths, $backend ); - $backend->preloadFileStat( [ 'srcs' => $realPaths, 'latest' => true ] ); - } - - $mBackend = $this->backends[$this->masterIndex]; - foreach ( $paths as $path ) { - $params = [ 'src' => $path, 'latest' => true ]; - $mParams = $this->substOpPaths( $params, $mBackend ); - // Stat the file on the 'master' backend - $mStat = $mBackend->getFileStat( $mParams ); - if ( $this->syncChecks & self::CHECK_SHA1 ) { - $mSha1 = $mBackend->getFileSha1Base36( $mParams ); - } else { - $mSha1 = false; - } - // Check if all clone backends agree with the master... - foreach ( $this->backends as $index => $cBackend ) { - if ( $index === $this->masterIndex ) { - continue; // master - } - $cParams = $this->substOpPaths( $params, $cBackend ); - $cStat = $cBackend->getFileStat( $cParams ); - if ( $mStat ) { // file is in master - if ( !$cStat ) { // file should exist - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - if ( $this->syncChecks & self::CHECK_SIZE ) { - if ( $cStat['size'] != $mStat['size'] ) { // wrong size - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - if ( $this->syncChecks & self::CHECK_TIME ) { - $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] ); - $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] ); - if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - if ( $this->syncChecks & self::CHECK_SHA1 ) { - if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1 - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - } else { // file is not in master - if ( $cStat ) { // file should not exist - $status->fatal( 'backend-fail-synced', $path ); - } - } - } - } - - return $status; - } - - /** - * Check that a set of file paths are usable across all internal backends - * - * @param array $paths List of storage paths - * @return Status - */ - public function accessibilityCheck( array $paths ) { - $status = Status::newGood(); - if ( count( $this->backends ) <= 1 ) { - return $status; // skip checks - } - - foreach ( $paths as $path ) { - foreach ( $this->backends as $backend ) { - $realPath = $this->substPaths( $path, $backend ); - if ( !$backend->isPathUsableInternal( $realPath ) ) { - $status->fatal( 'backend-fail-usable', $path ); - } - } - } - - return $status; - } - - /** - * Check that a set of files are consistent across all internal backends - * and re-synchronize those files against the "multi master" if needed. - * - * @param array $paths List of storage paths - * @param string|bool $resyncMode False, True, or "conservative"; see __construct() - * @return Status - */ - public function resyncFiles( array $paths, $resyncMode = true ) { - $status = Status::newGood(); - - $mBackend = $this->backends[$this->masterIndex]; - foreach ( $paths as $path ) { - $mPath = $this->substPaths( $path, $mBackend ); - $mSha1 = $mBackend->getFileSha1Base36( [ 'src' => $mPath, 'latest' => true ] ); - $mStat = $mBackend->getFileStat( [ 'src' => $mPath, 'latest' => true ] ); - if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'FileOperation', __METHOD__ - . ': File is not available on the master backend' ); - continue; // file is not available on the master backend... - } - // Check of all clone backends agree with the master... - foreach ( $this->backends as $index => $cBackend ) { - if ( $index === $this->masterIndex ) { - continue; // master - } - $cPath = $this->substPaths( $path, $cBackend ); - $cSha1 = $cBackend->getFileSha1Base36( [ 'src' => $cPath, 'latest' => true ] ); - $cStat = $cBackend->getFileStat( [ 'src' => $cPath, 'latest' => true ] ); - if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity - $status->fatal( 'backend-fail-internal', $cBackend->getName() ); - wfDebugLog( 'FileOperation', __METHOD__ . - ': File is not available on the clone backend' ); - continue; // file is not available on the clone backend... - } - if ( $mSha1 === $cSha1 ) { - // already synced; nothing to do - } elseif ( $mSha1 !== false ) { // file is in master - if ( $resyncMode === 'conservative' - && $cStat && $cStat['mtime'] > $mStat['mtime'] - ) { - $status->fatal( 'backend-fail-synced', $path ); - continue; // don't rollback data - } - $fsFile = $mBackend->getLocalReference( - [ 'src' => $mPath, 'latest' => true ] ); - $status->merge( $cBackend->quickStore( - [ 'src' => $fsFile->getPath(), 'dst' => $cPath ] - ) ); - } elseif ( $mStat === false ) { // file is not in master - if ( $resyncMode === 'conservative' ) { - $status->fatal( 'backend-fail-synced', $path ); - continue; // don't delete data - } - $status->merge( $cBackend->quickDelete( [ 'src' => $cPath ] ) ); - } - } - } - - if ( !$status->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . - " failed to resync: " . FormatJson::encode( $paths ) ); - } - - return $status; - } - - /** - * Get a list of file storage paths to read or write for a list of operations - * - * @param array $ops Same format as doOperations() - * @return array List of storage paths to files (does not include directories) - */ - protected function fileStoragePathsForOps( array $ops ) { - $paths = []; - foreach ( $ops as $op ) { - if ( isset( $op['src'] ) ) { - // For things like copy/move/delete with "ignoreMissingSource" and there - // is no source file, nothing should happen and there should be no errors. - if ( empty( $op['ignoreMissingSource'] ) - || $this->fileExists( [ 'src' => $op['src'] ] ) - ) { - $paths[] = $op['src']; - } - } - if ( isset( $op['srcs'] ) ) { - $paths = array_merge( $paths, $op['srcs'] ); - } - if ( isset( $op['dst'] ) ) { - $paths[] = $op['dst']; - } - } - - return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) ); - } - - /** - * Substitute the backend name in storage path parameters - * for a set of operations with that of a given internal backend. - * - * @param array $ops List of file operation arrays - * @param FileBackendStore $backend - * @return array - */ - protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) { - $newOps = []; // operations - foreach ( $ops as $op ) { - $newOp = $op; // operation - foreach ( [ 'src', 'srcs', 'dst', 'dir' ] as $par ) { - if ( isset( $newOp[$par] ) ) { // string or array - $newOp[$par] = $this->substPaths( $newOp[$par], $backend ); - } - } - $newOps[] = $newOp; - } - - return $newOps; - } - - /** - * Same as substOpBatchPaths() but for a single operation - * - * @param array $ops File operation array - * @param FileBackendStore $backend - * @return array - */ - protected function substOpPaths( array $ops, FileBackendStore $backend ) { - $newOps = $this->substOpBatchPaths( [ $ops ], $backend ); - - return $newOps[0]; - } - - /** - * Substitute the backend of storage paths with an internal backend's name - * - * @param array|string $paths List of paths or single string path - * @param FileBackendStore $backend - * @return array|string - */ - protected function substPaths( $paths, FileBackendStore $backend ) { - return preg_replace( - '!^mwstore://' . preg_quote( $this->name, '!' ) . '/!', - StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ), - $paths // string or array - ); - } - - /** - * Substitute the backend of internal storage paths with the proxy backend's name - * - * @param array|string $paths List of paths or single string path - * @return array|string - */ - protected function unsubstPaths( $paths ) { - return preg_replace( - '!^mwstore://([^/]+)!', - StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ), - $paths // string or array - ); - } - - /** - * @param array $ops File operations for FileBackend::doOperations() - * @return bool Whether there are file path sources with outside lifetime/ownership - */ - protected function hasVolatileSources( array $ops ) { - foreach ( $ops as $op ) { - if ( $op['op'] === 'store' && !isset( $op['srcRef'] ) ) { - return true; // source file might be deleted anytime after do*Operations() - } - } - - return false; - } - - protected function doQuickOperationsInternal( array $ops ) { - $status = Status::newGood(); - // Do the operations on the master backend; setting Status fields... - $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); - $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps ); - $status->merge( $masterStatus ); - // Propagate the operations to the clone backends... - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // done already - } - - $realOps = $this->substOpBatchPaths( $ops, $backend ); - if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $realOps ) { - $backend->doQuickOperations( $realOps ); - } - ); - } else { - $status->merge( $backend->doQuickOperations( $realOps ) ); - } - } - // Make 'success', 'successCount', and 'failCount' fields reflect - // the overall operation, rather than all the batches for each backend. - // Do this by only using success values from the master backend's batch. - $status->success = $masterStatus->success; - $status->successCount = $masterStatus->successCount; - $status->failCount = $masterStatus->failCount; - - return $status; - } - - protected function doPrepare( array $params ) { - return $this->doDirectoryOp( 'prepare', $params ); - } - - protected function doSecure( array $params ) { - return $this->doDirectoryOp( 'secure', $params ); - } - - protected function doPublish( array $params ) { - return $this->doDirectoryOp( 'publish', $params ); - } - - protected function doClean( array $params ) { - return $this->doDirectoryOp( 'clean', $params ); - } - - /** - * @param string $method One of (doPrepare,doSecure,doPublish,doClean) - * @param array $params Method arguments - * @return Status - */ - protected function doDirectoryOp( $method, array $params ) { - $status = Status::newGood(); - - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams ); - $status->merge( $masterStatus ); - - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // already done - } - - $realParams = $this->substOpPaths( $params, $backend ); - if ( $this->asyncWrites ) { - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $method, $realParams ) { - $backend->$method( $realParams ); - } - ); - } else { - $status->merge( $backend->$method( $realParams ) ); - } - } - - return $status; - } - - public function concatenate( array $params ) { - // We are writing to an FS file, so we don't need to do this per-backend - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->concatenate( $realParams ); - } - - public function fileExists( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->fileExists( $realParams ); - } - - public function getFileTimestamp( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileTimestamp( $realParams ); - } - - public function getFileSize( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileSize( $realParams ); - } - - public function getFileStat( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileStat( $realParams ); - } - - public function getFileXAttributes( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileXAttributes( $realParams ); - } - - public function getFileContentsMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams ); - - $contents = []; // (path => FSFile) mapping using the proxy backend's name - foreach ( $contentsM as $path => $data ) { - $contents[$this->unsubstPaths( $path )] = $data; - } - - return $contents; - } - - public function getFileSha1Base36( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileSha1Base36( $realParams ); - } - - public function getFileProps( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileProps( $realParams ); - } - - public function streamFile( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->streamFile( $realParams ); - } - - public function getLocalReferenceMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams ); - - $fsFiles = []; // (path => FSFile) mapping using the proxy backend's name - foreach ( $fsFilesM as $path => $fsFile ) { - $fsFiles[$this->unsubstPaths( $path )] = $fsFile; - } - - return $fsFiles; - } - - public function getLocalCopyMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams ); - - $tempFiles = []; // (path => TempFSFile) mapping using the proxy backend's name - foreach ( $tempFilesM as $path => $tempFile ) { - $tempFiles[$this->unsubstPaths( $path )] = $tempFile; - } - - return $tempFiles; - } - - public function getFileHttpUrl( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileHttpUrl( $realParams ); - } - - public function directoryExists( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->directoryExists( $realParams ); - } - - public function getDirectoryList( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->getDirectoryList( $realParams ); - } - - public function getFileList( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->getFileList( $realParams ); - } - - public function getFeatures() { - return $this->backends[$this->masterIndex]->getFeatures(); - } - - public function clearCache( array $paths = null ) { - foreach ( $this->backends as $backend ) { - $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; - $backend->clearCache( $realPaths ); - } - } - - public function preloadCache( array $paths ) { - $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] ); - $this->backends[$this->readIndex]->preloadCache( $realPaths ); - } - - public function preloadFileStat( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->preloadFileStat( $realParams ); - } - - public function getScopedLocksForOps( array $ops, Status $status ) { - $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); - $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps ); - // Get the paths to lock from the master backend - $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps ); - // Get the paths under the proxy backend's name - $pbPaths = [ - LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ), - LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] ) - ]; - - // Actually acquire the locks - return $this->getScopedFileLocks( $pbPaths, 'mixed', $status ); - } - - /** - * @param array $params - * @return int The master or read affinity backend index, based on $params['latest'] - */ - protected function getReadIndexFromParams( array $params ) { - return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex; - } -} diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php deleted file mode 100644 index a29119c9cf..0000000000 --- a/includes/filebackend/FileBackendStore.php +++ /dev/null @@ -1,1971 +0,0 @@ -mimeCallback = isset( $config['mimeCallback'] ) - ? $config['mimeCallback'] - : null; - $this->memCache = WANObjectCache::newEmpty(); // disabled by default - $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); - $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); - } - - /** - * Get the maximum allowable file size given backend - * medium restrictions and basic performance constraints. - * Do not call this function from places outside FileBackend and FileOp. - * - * @return int Bytes - */ - final public function maxFileSizeInternal() { - return $this->maxFileSize; - } - - /** - * Check if a file can be created or changed at a given storage path. - * FS backends should check if the parent directory exists, files can be - * written under it, and that any file already there is writable. - * Backends using key/value stores should check if the container exists. - * - * @param string $storagePath - * @return bool - */ - abstract public function isPathUsableInternal( $storagePath ); - - /** - * Create a file in the backend with the given contents. - * This will overwrite any file that exists at the destination. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - content : the raw file contents - * - dst : destination storage path - * - headers : HTTP header name/value map - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - dstExists : Whether a file exists at the destination (optimization). - * Callers can use "false" if no existing file is being changed. - * - * @param array $params - * @return Status - */ - final public function createInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { - $status = Status::newFatal( 'backend-fail-maxsize', - $params['dst'], $this->maxFileSizeInternal() ); - } else { - $status = $this->doCreateInternal( $params ); - $this->clearCache( [ $params['dst'] ] ); - if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { - $this->deleteFileCache( $params['dst'] ); // persistent cache - } - } - - return $status; - } - - /** - * @see FileBackendStore::createInternal() - * @param array $params - * @return Status - */ - abstract protected function doCreateInternal( array $params ); - - /** - * Store a file into the backend from a file on disk. - * This will overwrite any file that exists at the destination. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - src : source path on disk - * - dst : destination storage path - * - headers : HTTP header name/value map - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - dstExists : Whether a file exists at the destination (optimization). - * Callers can use "false" if no existing file is being changed. - * - * @param array $params - * @return Status - */ - final public function storeInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { - $status = Status::newFatal( 'backend-fail-maxsize', - $params['dst'], $this->maxFileSizeInternal() ); - } else { - $status = $this->doStoreInternal( $params ); - $this->clearCache( [ $params['dst'] ] ); - if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { - $this->deleteFileCache( $params['dst'] ); // persistent cache - } - } - - return $status; - } - - /** - * @see FileBackendStore::storeInternal() - * @param array $params - * @return Status - */ - abstract protected function doStoreInternal( array $params ); - - /** - * Copy a file from one storage path to another in the backend. - * This will overwrite any file that exists at the destination. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - src : source storage path - * - dst : destination storage path - * - ignoreMissingSource : do nothing if the source file does not exist - * - headers : HTTP header name/value map - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - dstExists : Whether a file exists at the destination (optimization). - * Callers can use "false" if no existing file is being changed. - * - * @param array $params - * @return Status - */ - final public function copyInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = $this->doCopyInternal( $params ); - $this->clearCache( [ $params['dst'] ] ); - if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { - $this->deleteFileCache( $params['dst'] ); // persistent cache - } - - return $status; - } - - /** - * @see FileBackendStore::copyInternal() - * @param array $params - * @return Status - */ - abstract protected function doCopyInternal( array $params ); - - /** - * Delete a file at the storage path. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - src : source storage path - * - ignoreMissingSource : do nothing if the source file does not exist - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - * @param array $params - * @return Status - */ - final public function deleteInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = $this->doDeleteInternal( $params ); - $this->clearCache( [ $params['src'] ] ); - $this->deleteFileCache( $params['src'] ); // persistent cache - return $status; - } - - /** - * @see FileBackendStore::deleteInternal() - * @param array $params - * @return Status - */ - abstract protected function doDeleteInternal( array $params ); - - /** - * Move a file from one storage path to another in the backend. - * This will overwrite any file that exists at the destination. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - src : source storage path - * - dst : destination storage path - * - ignoreMissingSource : do nothing if the source file does not exist - * - headers : HTTP header name/value map - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - dstExists : Whether a file exists at the destination (optimization). - * Callers can use "false" if no existing file is being changed. - * - * @param array $params - * @return Status - */ - final public function moveInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = $this->doMoveInternal( $params ); - $this->clearCache( [ $params['src'], $params['dst'] ] ); - $this->deleteFileCache( $params['src'] ); // persistent cache - if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { - $this->deleteFileCache( $params['dst'] ); // persistent cache - } - - return $status; - } - - /** - * @see FileBackendStore::moveInternal() - * @param array $params - * @return Status - */ - protected function doMoveInternal( array $params ) { - unset( $params['async'] ); // two steps, won't work here :) - $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); - $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); - // Copy source to dest - $status = $this->copyInternal( $params ); - if ( $nsrc !== $ndst && $status->isOK() ) { - // Delete source (only fails due to races or network problems) - $status->merge( $this->deleteInternal( [ 'src' => $params['src'] ] ) ); - $status->setResult( true, $status->value ); // ignore delete() errors - } - - return $status; - } - - /** - * Alter metadata for a file at the storage path. - * Do not call this function from places outside FileBackend and FileOp. - * - * $params include: - * - src : source storage path - * - headers : HTTP header name/value map - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. - * - * @param array $params - * @return Status - */ - final public function describeInternal( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - if ( count( $params['headers'] ) ) { - $status = $this->doDescribeInternal( $params ); - $this->clearCache( [ $params['src'] ] ); - $this->deleteFileCache( $params['src'] ); // persistent cache - } else { - $status = Status::newGood(); // nothing to do - } - - return $status; - } - - /** - * @see FileBackendStore::describeInternal() - * @param array $params - * @return Status - */ - protected function doDescribeInternal( array $params ) { - return Status::newGood(); - } - - /** - * No-op file operation that does nothing. - * Do not call this function from places outside FileBackend and FileOp. - * - * @param array $params - * @return Status - */ - final public function nullInternal( array $params ) { - return Status::newGood(); - } - - final public function concatenate( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - // Try to lock the source files for the scope of this function - $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); - if ( $status->isOK() ) { - // Actually do the file concatenation... - $start_time = microtime( true ); - $status->merge( $this->doConcatenate( $params ) ); - $sec = microtime( true ) - $start_time; - if ( !$status->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name}" . - " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" ); - } - } - - return $status; - } - - /** - * @see FileBackendStore::concatenate() - * @param array $params - * @return Status - */ - protected function doConcatenate( array $params ) { - $status = Status::newGood(); - $tmpPath = $params['dst']; // convenience - unset( $params['latest'] ); // sanity - - // Check that the specified temp file is valid... - MediaWiki\suppressWarnings(); - $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 ); - MediaWiki\restoreWarnings(); - if ( !$ok ) { // not present or not empty - $status->fatal( 'backend-fail-opentemp', $tmpPath ); - - return $status; - } - - // Get local FS versions of the chunks needed for the concatenation... - $fsFiles = $this->getLocalReferenceMulti( $params ); - foreach ( $fsFiles as $path => &$fsFile ) { - if ( !$fsFile ) { // chunk failed to download? - $fsFile = $this->getLocalReference( [ 'src' => $path ] ); - if ( !$fsFile ) { // retry failed? - $status->fatal( 'backend-fail-read', $path ); - - return $status; - } - } - } - unset( $fsFile ); // unset reference so we can reuse $fsFile - - // Get a handle for the destination temp file - $tmpHandle = fopen( $tmpPath, 'ab' ); - if ( $tmpHandle === false ) { - $status->fatal( 'backend-fail-opentemp', $tmpPath ); - - return $status; - } - - // Build up the temp file using the source chunks (in order)... - foreach ( $fsFiles as $virtualSource => $fsFile ) { - // Get a handle to the local FS version - $sourceHandle = fopen( $fsFile->getPath(), 'rb' ); - if ( $sourceHandle === false ) { - fclose( $tmpHandle ); - $status->fatal( 'backend-fail-read', $virtualSource ); - - return $status; - } - // Append chunk to file (pass chunk size to avoid magic quotes) - if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { - fclose( $sourceHandle ); - fclose( $tmpHandle ); - $status->fatal( 'backend-fail-writetemp', $tmpPath ); - - return $status; - } - fclose( $sourceHandle ); - } - if ( !fclose( $tmpHandle ) ) { - $status->fatal( 'backend-fail-closetemp', $tmpPath ); - - return $status; - } - - clearstatcache(); // temp file changed - - return $status; - } - - final protected function doPrepare( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - - return $status; // invalid storage path - } - - if ( $shard !== null ) { // confined to a single container/shard - $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); - } else { // directory is on several shards - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { - $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); - } - } - - return $status; - } - - /** - * @see FileBackendStore::doPrepare() - * @param string $container - * @param string $dir - * @param array $params - * @return Status - */ - protected function doPrepareInternal( $container, $dir, array $params ) { - return Status::newGood(); - } - - final protected function doSecure( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - - return $status; // invalid storage path - } - - if ( $shard !== null ) { // confined to a single container/shard - $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); - } else { // directory is on several shards - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { - $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); - } - } - - return $status; - } - - /** - * @see FileBackendStore::doSecure() - * @param string $container - * @param string $dir - * @param array $params - * @return Status - */ - protected function doSecureInternal( $container, $dir, array $params ) { - return Status::newGood(); - } - - final protected function doPublish( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - - return $status; // invalid storage path - } - - if ( $shard !== null ) { // confined to a single container/shard - $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); - } else { // directory is on several shards - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { - $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) ); - } - } - - return $status; - } - - /** - * @see FileBackendStore::doPublish() - * @param string $container - * @param string $dir - * @param array $params - * @return Status - */ - protected function doPublishInternal( $container, $dir, array $params ) { - return Status::newGood(); - } - - final protected function doClean( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - // Recursive: first delete all empty subdirs recursively - if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) { - $subDirsRel = $this->getTopDirectoryList( [ 'dir' => $params['dir'] ] ); - if ( $subDirsRel !== null ) { // no errors - foreach ( $subDirsRel as $subDirRel ) { - $subDir = $params['dir'] . "/{$subDirRel}"; // full path - $status->merge( $this->doClean( [ 'dir' => $subDir ] + $params ) ); - } - unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends) - } - } - - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - - return $status; // invalid storage path - } - - // Attempt to lock this directory... - $filesLockEx = [ $params['dir'] ]; - $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); - if ( !$status->isOK() ) { - return $status; // abort - } - - if ( $shard !== null ) { // confined to a single container/shard - $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); - $this->deleteContainerCache( $fullCont ); // purge cache - } else { // directory is on several shards - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { - $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); - $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache - } - } - - return $status; - } - - /** - * @see FileBackendStore::doClean() - * @param string $container - * @param string $dir - * @param array $params - * @return Status - */ - protected function doCleanInternal( $container, $dir, array $params ) { - return Status::newGood(); - } - - final public function fileExists( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $stat = $this->getFileStat( $params ); - - return ( $stat === null ) ? null : (bool)$stat; // null => failure - } - - final public function getFileTimestamp( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $stat = $this->getFileStat( $params ); - - return $stat ? $stat['mtime'] : false; - } - - final public function getFileSize( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $stat = $this->getFileStat( $params ); - - return $stat ? $stat['size'] : false; - } - - final public function getFileStat( array $params ) { - $path = self::normalizeStoragePath( $params['src'] ); - if ( $path === null ) { - return false; // invalid storage path - } - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $latest = !empty( $params['latest'] ); // use latest data? - if ( !$latest && !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { - $this->primeFileCache( [ $path ] ); // check persistent cache - } - if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { - $stat = $this->cheapCache->get( $path, 'stat' ); - // If we want the latest data, check that this cached - // value was in fact fetched with the latest available data. - if ( is_array( $stat ) ) { - if ( !$latest || $stat['latest'] ) { - return $stat; - } - } elseif ( in_array( $stat, [ 'NOT_EXIST', 'NOT_EXIST_LATEST' ] ) ) { - if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { - return false; - } - } - } - $stat = $this->doGetFileStat( $params ); - if ( is_array( $stat ) ) { // file exists - // Strongly consistent backends can automatically set "latest" - $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; - $this->cheapCache->set( $path, 'stat', $stat ); - $this->setFileCache( $path, $stat ); // update persistent cache - if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata - $this->cheapCache->set( $path, 'sha1', - [ 'hash' => $stat['sha1'], 'latest' => $latest ] ); - } - if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata - $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); - $this->cheapCache->set( $path, 'xattr', - [ 'map' => $stat['xattr'], 'latest' => $latest ] ); - } - } elseif ( $stat === false ) { // file does not exist - $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); - $this->cheapCache->set( $path, 'xattr', [ 'map' => false, 'latest' => $latest ] ); - $this->cheapCache->set( $path, 'sha1', [ 'hash' => false, 'latest' => $latest ] ); - wfDebug( __METHOD__ . ": File $path does not exist.\n" ); - } else { // an error occurred - wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); - } - - return $stat; - } - - /** - * @see FileBackendStore::getFileStat() - */ - abstract protected function doGetFileStat( array $params ); - - public function getFileContentsMulti( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $params = $this->setConcurrencyFlags( $params ); - $contents = $this->doGetFileContentsMulti( $params ); - - return $contents; - } - - /** - * @see FileBackendStore::getFileContentsMulti() - * @param array $params - * @return array - */ - protected function doGetFileContentsMulti( array $params ) { - $contents = []; - foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { - MediaWiki\suppressWarnings(); - $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false; - MediaWiki\restoreWarnings(); - } - - return $contents; - } - - final public function getFileXAttributes( array $params ) { - $path = self::normalizeStoragePath( $params['src'] ); - if ( $path === null ) { - return false; // invalid storage path - } - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $latest = !empty( $params['latest'] ); // use latest data? - if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) { - $stat = $this->cheapCache->get( $path, 'xattr' ); - // If we want the latest data, check that this cached - // value was in fact fetched with the latest available data. - if ( !$latest || $stat['latest'] ) { - return $stat['map']; - } - } - $fields = $this->doGetFileXAttributes( $params ); - $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false; - $this->cheapCache->set( $path, 'xattr', [ 'map' => $fields, 'latest' => $latest ] ); - - return $fields; - } - - /** - * @see FileBackendStore::getFileXAttributes() - * @return bool|string - */ - protected function doGetFileXAttributes( array $params ) { - return [ 'headers' => [], 'metadata' => [] ]; // not supported - } - - final public function getFileSha1Base36( array $params ) { - $path = self::normalizeStoragePath( $params['src'] ); - if ( $path === null ) { - return false; // invalid storage path - } - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $latest = !empty( $params['latest'] ); // use latest data? - if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { - $stat = $this->cheapCache->get( $path, 'sha1' ); - // If we want the latest data, check that this cached - // value was in fact fetched with the latest available data. - if ( !$latest || $stat['latest'] ) { - return $stat['hash']; - } - } - $hash = $this->doGetFileSha1Base36( $params ); - $this->cheapCache->set( $path, 'sha1', [ 'hash' => $hash, 'latest' => $latest ] ); - - return $hash; - } - - /** - * @see FileBackendStore::getFileSha1Base36() - * @param array $params - * @return bool|string - */ - protected function doGetFileSha1Base36( array $params ) { - $fsFile = $this->getLocalReference( $params ); - if ( !$fsFile ) { - return false; - } else { - return $fsFile->getSha1Base36(); - } - } - - final public function getFileProps( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $fsFile = $this->getLocalReference( $params ); - $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); - - return $props; - } - - final public function getLocalReferenceMulti( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $params = $this->setConcurrencyFlags( $params ); - - $fsFiles = []; // (path => FSFile) - $latest = !empty( $params['latest'] ); // use latest data? - // Reuse any files already in process cache... - foreach ( $params['srcs'] as $src ) { - $path = self::normalizeStoragePath( $src ); - if ( $path === null ) { - $fsFiles[$src] = null; // invalid storage path - } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) { - $val = $this->expensiveCache->get( $path, 'localRef' ); - // If we want the latest data, check that this cached - // value was in fact fetched with the latest available data. - if ( !$latest || $val['latest'] ) { - $fsFiles[$src] = $val['object']; - } - } - } - // Fetch local references of any remaning files... - $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) ); - foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { - $fsFiles[$path] = $fsFile; - if ( $fsFile ) { // update the process cache... - $this->expensiveCache->set( $path, 'localRef', - [ 'object' => $fsFile, 'latest' => $latest ] ); - } - } - - return $fsFiles; - } - - /** - * @see FileBackendStore::getLocalReferenceMulti() - * @param array $params - * @return array - */ - protected function doGetLocalReferenceMulti( array $params ) { - return $this->doGetLocalCopyMulti( $params ); - } - - final public function getLocalCopyMulti( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $params = $this->setConcurrencyFlags( $params ); - $tmpFiles = $this->doGetLocalCopyMulti( $params ); - - return $tmpFiles; - } - - /** - * @see FileBackendStore::getLocalCopyMulti() - * @param array $params - * @return array - */ - abstract protected function doGetLocalCopyMulti( array $params ); - - /** - * @see FileBackend::getFileHttpUrl() - * @param array $params - * @return string|null - */ - public function getFileHttpUrl( array $params ) { - return null; // not supported - } - - final public function streamFile( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - // Always set some fields for subclass convenience - $params['options'] = isset( $params['options'] ) ? $params['options'] : []; - $params['headers'] = isset( $params['headers'] ) ? $params['headers'] : []; - - // Don't stream it out as text/html if there was a PHP error - if ( ( empty( $params['headless'] ) || $params['headers'] ) && headers_sent() ) { - print "Headers already sent, terminating.\n"; - $status->fatal( 'backend-fail-stream', $params['src'] ); - return $status; - } - - $status->merge( $this->doStreamFile( $params ) ); - - return $status; - } - - /** - * @see FileBackendStore::streamFile() - * @param array $params - * @return Status - */ - protected function doStreamFile( array $params ) { - $status = Status::newGood(); - - $flags = 0; - $flags |= !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0; - $flags |= !empty( $params['allowOB'] ) ? StreamFile::STREAM_ALLOW_OB : 0; - - $fsFile = $this->getLocalReference( $params ); - - if ( $fsFile ) { - $res = StreamFile::stream( $fsFile->getPath(), - $params['headers'], true, $params['options'], $flags ); - } else { - $res = false; - StreamFile::send404Message( $params['src'], $flags ); - } - - if ( !$res ) { - $status->fatal( 'backend-fail-stream', $params['src'] ); - } - - return $status; - } - - final public function directoryExists( array $params ) { - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - return false; // invalid storage path - } - if ( $shard !== null ) { // confined to a single container/shard - return $this->doDirectoryExists( $fullCont, $dir, $params ); - } else { // directory is on several shards - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - $res = false; // response - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { - $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params ); - if ( $exists ) { - $res = true; - break; // found one! - } elseif ( $exists === null ) { // error? - $res = null; // if we don't find anything, it is indeterminate - } - } - - return $res; - } - } - - /** - * @see FileBackendStore::directoryExists() - * - * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param array $params - * @return bool|null - */ - abstract protected function doDirectoryExists( $container, $dir, array $params ); - - final public function getDirectoryList( array $params ) { - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { // invalid storage path - return null; - } - if ( $shard !== null ) { - // File listing is confined to a single container/shard - return $this->getDirectoryListInternal( $fullCont, $dir, $params ); - } else { - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - // File listing spans multiple containers/shards - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - - return new FileBackendStoreShardDirIterator( $this, - $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); - } - } - - /** - * Do not call this function from places outside FileBackend - * - * @see FileBackendStore::getDirectoryList() - * - * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param array $params - * @return Traversable|array|null Returns null on failure - */ - abstract public function getDirectoryListInternal( $container, $dir, array $params ); - - final public function getFileList( array $params ) { - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { // invalid storage path - return null; - } - if ( $shard !== null ) { - // File listing is confined to a single container/shard - return $this->getFileListInternal( $fullCont, $dir, $params ); - } else { - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - // File listing spans multiple containers/shards - list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); - - return new FileBackendStoreShardFileIterator( $this, - $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); - } - } - - /** - * Do not call this function from places outside FileBackend - * - * @see FileBackendStore::getFileList() - * - * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param array $params - * @return Traversable|array|null Returns null on failure - */ - abstract public function getFileListInternal( $container, $dir, array $params ); - - /** - * Return a list of FileOp objects from a list of operations. - * Do not call this function from places outside FileBackend. - * - * The result must have the same number of items as the input. - * An exception is thrown if an unsupported operation is requested. - * - * @param array $ops Same format as doOperations() - * @return array List of FileOp objects - * @throws FileBackendError - */ - final public function getOperationsInternal( array $ops ) { - $supportedOps = [ - 'store' => 'StoreFileOp', - 'copy' => 'CopyFileOp', - 'move' => 'MoveFileOp', - 'delete' => 'DeleteFileOp', - 'create' => 'CreateFileOp', - 'describe' => 'DescribeFileOp', - 'null' => 'NullFileOp' - ]; - - $performOps = []; // array of FileOp objects - // Build up ordered array of FileOps... - foreach ( $ops as $operation ) { - $opName = $operation['op']; - if ( isset( $supportedOps[$opName] ) ) { - $class = $supportedOps[$opName]; - // Get params for this operation - $params = $operation; - // Append the FileOp class - $performOps[] = new $class( $this, $params ); - } else { - throw new FileBackendError( "Operation '$opName' is not supported." ); - } - } - - return $performOps; - } - - /** - * Get a list of storage paths to lock for a list of operations - * Returns an array with LockManager::LOCK_UW (shared locks) and - * LockManager::LOCK_EX (exclusive locks) keys, each corresponding - * to a list of storage paths to be locked. All returned paths are - * normalized. - * - * @param array $performOps List of FileOp objects - * @return array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) - */ - final public function getPathsToLockForOpsInternal( array $performOps ) { - // Build up a list of files to lock... - $paths = [ 'sh' => [], 'ex' => [] ]; - foreach ( $performOps as $fileOp ) { - $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() ); - $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() ); - } - // Optimization: if doing an EX lock anyway, don't also set an SH one - $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] ); - // Get a shared lock on the parent directory of each path changed - $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); - - return [ - LockManager::LOCK_UW => $paths['sh'], - LockManager::LOCK_EX => $paths['ex'] - ]; - } - - public function getScopedLocksForOps( array $ops, Status $status ) { - $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); - - return $this->getScopedFileLocks( $paths, 'mixed', $status ); - } - - final protected function doOperationsInternal( array $ops, array $opts ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - // Fix up custom header name/value pairs... - $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops ); - - // Build up a list of FileOps... - $performOps = $this->getOperationsInternal( $ops ); - - // Acquire any locks as needed... - if ( empty( $opts['nonLocking'] ) ) { - // Build up a list of files to lock... - $paths = $this->getPathsToLockForOpsInternal( $performOps ); - // Try to lock those files for the scope of this function... - - $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status ); - if ( !$status->isOK() ) { - return $status; // abort - } - } - - // Clear any file cache entries (after locks acquired) - if ( empty( $opts['preserveCache'] ) ) { - $this->clearCache(); - } - - // Build the list of paths involved - $paths = []; - foreach ( $performOps as $op ) { - $paths = array_merge( $paths, $op->storagePathsRead() ); - $paths = array_merge( $paths, $op->storagePathsChanged() ); - } - - // Enlarge the cache to fit the stat entries of these files - $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) ); - - // Load from the persistent container caches - $this->primeContainerCache( $paths ); - // Get the latest stat info for all the files (having locked them) - $ok = $this->preloadFileStat( [ 'srcs' => $paths, 'latest' => true ] ); - - if ( $ok ) { - // Actually attempt the operation batch... - $opts = $this->setConcurrencyFlags( $opts ); - $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); - } else { - // If we could not even stat some files, then bail out... - $subStatus = Status::newFatal( 'backend-fail-internal', $this->name ); - foreach ( $ops as $i => $op ) { // mark each op as failed - $subStatus->success[$i] = false; - ++$subStatus->failCount; - } - wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name} " . - " stat failure; aborted operations: " . FormatJson::encode( $ops ) ); - } - - // Merge errors into status fields - $status->merge( $subStatus ); - $status->success = $subStatus->success; // not done in merge() - - // Shrink the stat cache back to normal size - $this->cheapCache->resize( self::CACHE_CHEAP_SIZE ); - - return $status; - } - - final protected function doQuickOperationsInternal( array $ops ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $status = Status::newGood(); - - // Fix up custom header name/value pairs... - $ops = array_map( [ $this, 'sanitizeOpHeaders' ], $ops ); - - // Clear any file cache entries - $this->clearCache(); - - $supportedOps = [ 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' ]; - // Parallel ops may be disabled in config due to dependencies (e.g. needing popen()) - $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); - $maxConcurrency = $this->concurrency; // throttle - - $statuses = []; // array of (index => Status) - $fileOpHandles = []; // list of (index => handle) arrays - $curFileOpHandles = []; // current handle batch - // Perform the sync-only ops and build up op handles for the async ops... - foreach ( $ops as $index => $params ) { - if ( !in_array( $params['op'], $supportedOps ) ) { - throw new FileBackendError( "Operation '{$params['op']}' is not supported." ); - } - $method = $params['op'] . 'Internal'; // e.g. "storeInternal" - $subStatus = $this->$method( [ 'async' => $async ] + $params ); - if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async - if ( count( $curFileOpHandles ) >= $maxConcurrency ) { - $fileOpHandles[] = $curFileOpHandles; // push this batch - $curFileOpHandles = []; - } - $curFileOpHandles[$index] = $subStatus->value; // keep index - } else { // error or completed - $statuses[$index] = $subStatus; // keep index - } - } - if ( count( $curFileOpHandles ) ) { - $fileOpHandles[] = $curFileOpHandles; // last batch - } - // Do all the async ops that can be done concurrently... - foreach ( $fileOpHandles as $fileHandleBatch ) { - $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch ); - } - // Marshall and merge all the responses... - foreach ( $statuses as $index => $subStatus ) { - $status->merge( $subStatus ); - if ( $subStatus->isOK() ) { - $status->success[$index] = true; - ++$status->successCount; - } else { - $status->success[$index] = false; - ++$status->failCount; - } - } - - return $status; - } - - /** - * Execute a list of FileBackendStoreOpHandle handles in parallel. - * The resulting Status object fields will correspond - * to the order in which the handles where given. - * - * @param FileBackendStoreOpHandle[] $fileOpHandles - * - * @throws FileBackendError - * @return array Map of Status objects - */ - final public function executeOpHandlesInternal( array $fileOpHandles ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - foreach ( $fileOpHandles as $fileOpHandle ) { - if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { - throw new FileBackendError( "Given a non-FileBackendStoreOpHandle object." ); - } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) { - throw new FileBackendError( "Given a FileBackendStoreOpHandle for the wrong backend." ); - } - } - $res = $this->doExecuteOpHandlesInternal( $fileOpHandles ); - foreach ( $fileOpHandles as $fileOpHandle ) { - $fileOpHandle->closeResources(); - } - - return $res; - } - - /** - * @see FileBackendStore::executeOpHandlesInternal() - * - * @param FileBackendStoreOpHandle[] $fileOpHandles - * - * @throws FileBackendError - * @return Status[] List of corresponding Status objects - */ - protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { - if ( count( $fileOpHandles ) ) { - throw new FileBackendError( "This backend supports no asynchronous operations." ); - } - - return []; - } - - /** - * Normalize and filter HTTP headers from a file operation - * - * This normalizes and strips long HTTP headers from a file operation. - * Most headers are just numbers, but some are allowed to be long. - * This function is useful for cleaning up headers and avoiding backend - * specific errors, especially in the middle of batch file operations. - * - * @param array $op Same format as doOperation() - * @return array - */ - protected function sanitizeOpHeaders( array $op ) { - static $longs = [ 'content-disposition' ]; - - if ( isset( $op['headers'] ) ) { // op sets HTTP headers - $newHeaders = []; - foreach ( $op['headers'] as $name => $value ) { - $name = strtolower( $name ); - $maxHVLen = in_array( $name, $longs ) ? INF : 255; - if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { - trigger_error( "Header '$name: $value' is too long." ); - } else { - $newHeaders[$name] = strlen( $value ) ? $value : ''; // null/false => "" - } - } - $op['headers'] = $newHeaders; - } - - return $op; - } - - final public function preloadCache( array $paths ) { - $fullConts = []; // full container names - foreach ( $paths as $path ) { - list( $fullCont, , ) = $this->resolveStoragePath( $path ); - $fullConts[] = $fullCont; - } - // Load from the persistent file and container caches - $this->primeContainerCache( $fullConts ); - $this->primeFileCache( $paths ); - } - - final public function clearCache( array $paths = null ) { - if ( is_array( $paths ) ) { - $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); - $paths = array_filter( $paths, 'strlen' ); // remove nulls - } - if ( $paths === null ) { - $this->cheapCache->clear(); - $this->expensiveCache->clear(); - } else { - foreach ( $paths as $path ) { - $this->cheapCache->clear( $path ); - $this->expensiveCache->clear( $path ); - } - } - $this->doClearCache( $paths ); - } - - /** - * Clears any additional stat caches for storage paths - * - * @see FileBackend::clearCache() - * - * @param array $paths Storage paths (optional) - */ - protected function doClearCache( array $paths = null ) { - } - - final public function preloadFileStat( array $params ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - $success = true; // no network errors - - $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1; - $stats = $this->doGetFileStatMulti( $params ); - if ( $stats === null ) { - return true; // not supported - } - - $latest = !empty( $params['latest'] ); // use latest data? - foreach ( $stats as $path => $stat ) { - $path = FileBackend::normalizeStoragePath( $path ); - if ( $path === null ) { - continue; // this shouldn't happen - } - if ( is_array( $stat ) ) { // file exists - // Strongly consistent backends can automatically set "latest" - $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest; - $this->cheapCache->set( $path, 'stat', $stat ); - $this->setFileCache( $path, $stat ); // update persistent cache - if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata - $this->cheapCache->set( $path, 'sha1', - [ 'hash' => $stat['sha1'], 'latest' => $latest ] ); - } - if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata - $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); - $this->cheapCache->set( $path, 'xattr', - [ 'map' => $stat['xattr'], 'latest' => $latest ] ); - } - } elseif ( $stat === false ) { // file does not exist - $this->cheapCache->set( $path, 'stat', - $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); - $this->cheapCache->set( $path, 'xattr', - [ 'map' => false, 'latest' => $latest ] ); - $this->cheapCache->set( $path, 'sha1', - [ 'hash' => false, 'latest' => $latest ] ); - wfDebug( __METHOD__ . ": File $path does not exist.\n" ); - } else { // an error occurred - $success = false; - wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); - } - } - - return $success; - } - - /** - * Get file stat information (concurrently if possible) for several files - * - * @see FileBackend::getFileStat() - * - * @param array $params Parameters include: - * - srcs : list of source storage paths - * - latest : use the latest available data - * @return array|null Map of storage paths to array|bool|null (returns null if not supported) - * @since 1.23 - */ - protected function doGetFileStatMulti( array $params ) { - return null; // not supported - } - - /** - * Is this a key/value store where directories are just virtual? - * Virtual directories exists in so much as files exists that are - * prefixed with the directory path followed by a forward slash. - * - * @return bool - */ - abstract protected function directoriesAreVirtual(); - - /** - * Check if a short container name is valid - * - * This checks for length and illegal characters. - * This may disallow certain characters that can appear - * in the prefix used to make the full container name. - * - * @param string $container - * @return bool - */ - final protected static function isValidShortContainerName( $container ) { - // Suffixes like '.xxx' (hex shard chars) or '.seg' (file segments) - // might be used by subclasses. Reserve the dot character for sanity. - // The only way dots end up in containers (e.g. resolveStoragePath) - // is due to the wikiId container prefix or the above suffixes. - return self::isValidContainerName( $container ) && !preg_match( '/[.]/', $container ); - } - - /** - * Check if a full container name is valid - * - * This checks for length and illegal characters. - * Limiting the characters makes migrations to other stores easier. - * - * @param string $container - * @return bool - */ - final protected static function isValidContainerName( $container ) { - // This accounts for NTFS, Swift, and Ceph restrictions - // and disallows directory separators or traversal characters. - // Note that matching strings URL encode to the same string; - // in Swift/Ceph, the length restriction is *after* URL encoding. - return (bool)preg_match( '/^[a-z0-9][a-z0-9-_.]{0,199}$/i', $container ); - } - - /** - * Splits a storage path into an internal container name, - * an internal relative file name, and a container shard suffix. - * Any shard suffix is already appended to the internal container name. - * This also checks that the storage path is valid and within this backend. - * - * If the container is sharded but a suffix could not be determined, - * this means that the path can only refer to a directory and can only - * be scanned by looking in all the container shards. - * - * @param string $storagePath - * @return array (container, path, container suffix) or (null, null, null) if invalid - */ - final protected function resolveStoragePath( $storagePath ) { - list( $backend, $shortCont, $relPath ) = self::splitStoragePath( $storagePath ); - if ( $backend === $this->name ) { // must be for this backend - $relPath = self::normalizeContainerPath( $relPath ); - if ( $relPath !== null && self::isValidShortContainerName( $shortCont ) ) { - // Get shard for the normalized path if this container is sharded - $cShard = $this->getContainerShard( $shortCont, $relPath ); - // Validate and sanitize the relative path (backend-specific) - $relPath = $this->resolveContainerPath( $shortCont, $relPath ); - if ( $relPath !== null ) { - // Prepend any wiki ID prefix to the container name - $container = $this->fullContainerName( $shortCont ); - if ( self::isValidContainerName( $container ) ) { - // Validate and sanitize the container name (backend-specific) - $container = $this->resolveContainerName( "{$container}{$cShard}" ); - if ( $container !== null ) { - return [ $container, $relPath, $cShard ]; - } - } - } - } - } - - return [ null, null, null ]; - } - - /** - * Like resolveStoragePath() except null values are returned if - * the container is sharded and the shard could not be determined - * or if the path ends with '/'. The later case is illegal for FS - * backends and can confuse listings for object store backends. - * - * This function is used when resolving paths that must be valid - * locations for files. Directory and listing functions should - * generally just use resolveStoragePath() instead. - * - * @see FileBackendStore::resolveStoragePath() - * - * @param string $storagePath - * @return array (container, path) or (null, null) if invalid - */ - final protected function resolveStoragePathReal( $storagePath ) { - list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); - if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { - return [ $container, $relPath ]; - } - - return [ null, null ]; - } - - /** - * Get the container name shard suffix for a given path. - * Any empty suffix means the container is not sharded. - * - * @param string $container Container name - * @param string $relPath Storage path relative to the container - * @return string|null Returns null if shard could not be determined - */ - final protected function getContainerShard( $container, $relPath ) { - list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); - if ( $levels == 1 || $levels == 2 ) { - // Hash characters are either base 16 or 36 - $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; - // Get a regex that represents the shard portion of paths. - // The concatenation of the captures gives us the shard. - if ( $levels === 1 ) { // 16 or 36 shards per container - $hashDirRegex = '(' . $char . ')'; - } else { // 256 or 1296 shards per container - if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") - $hashDirRegex = $char . '/(' . $char . '{2})'; - } else { // short hash dir format (e.g. "a/b/c") - $hashDirRegex = '(' . $char . ')/(' . $char . ')'; - } - } - // Allow certain directories to be above the hash dirs so as - // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). - // They must be 2+ chars to avoid any hash directory ambiguity. - $m = []; - if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { - return '.' . implode( '', array_slice( $m, 1 ) ); - } - - return null; // failed to match - } - - return ''; // no sharding - } - - /** - * Check if a storage path maps to a single shard. - * Container dirs like "a", where the container shards on "x/xy", - * can reside on several shards. Such paths are tricky to handle. - * - * @param string $storagePath Storage path - * @return bool - */ - final public function isSingleShardPathInternal( $storagePath ) { - list( , , $shard ) = $this->resolveStoragePath( $storagePath ); - - return ( $shard !== null ); - } - - /** - * Get the sharding config for a container. - * If greater than 0, then all file storage paths within - * the container are required to be hashed accordingly. - * - * @param string $container - * @return array (integer levels, integer base, repeat flag) or (0, 0, false) - */ - final protected function getContainerHashLevels( $container ) { - if ( isset( $this->shardViaHashLevels[$container] ) ) { - $config = $this->shardViaHashLevels[$container]; - $hashLevels = (int)$config['levels']; - if ( $hashLevels == 1 || $hashLevels == 2 ) { - $hashBase = (int)$config['base']; - if ( $hashBase == 16 || $hashBase == 36 ) { - return [ $hashLevels, $hashBase, $config['repeat'] ]; - } - } - } - - return [ 0, 0, false ]; // no sharding - } - - /** - * Get a list of full container shard suffixes for a container - * - * @param string $container - * @return array - */ - final protected function getContainerSuffixes( $container ) { - $shards = []; - list( $digits, $base ) = $this->getContainerHashLevels( $container ); - if ( $digits > 0 ) { - $numShards = pow( $base, $digits ); - for ( $index = 0; $index < $numShards; $index++ ) { - $shards[] = '.' . Wikimedia\base_convert( $index, 10, $base, $digits ); - } - } - - return $shards; - } - - /** - * Get the full container name, including the wiki ID prefix - * - * @param string $container - * @return string - */ - final protected function fullContainerName( $container ) { - if ( $this->wikiId != '' ) { - return "{$this->wikiId}-$container"; - } else { - return $container; - } - } - - /** - * Resolve a container name, checking if it's allowed by the backend. - * This is intended for internal use, such as encoding illegal chars. - * Subclasses can override this to be more restrictive. - * - * @param string $container - * @return string|null - */ - protected function resolveContainerName( $container ) { - return $container; - } - - /** - * Resolve a relative storage path, checking if it's allowed by the backend. - * This is intended for internal use, such as encoding illegal chars or perhaps - * getting absolute paths (e.g. FS based backends). Note that the relative path - * may be the empty string (e.g. the path is simply to the container). - * - * @param string $container Container name - * @param string $relStoragePath Storage path relative to the container - * @return string|null Path or null if not valid - */ - protected function resolveContainerPath( $container, $relStoragePath ) { - return $relStoragePath; - } - - /** - * Get the cache key for a container - * - * @param string $container Resolved container name - * @return string - */ - private function containerCacheKey( $container ) { - return "filebackend:{$this->name}:{$this->wikiId}:container:{$container}"; - } - - /** - * Set the cached info for a container - * - * @param string $container Resolved container name - * @param array $val Information to cache - */ - final protected function setContainerCache( $container, array $val ) { - $this->memCache->set( $this->containerCacheKey( $container ), $val, 14 * 86400 ); - } - - /** - * Delete the cached info for a container. - * The cache key is salted for a while to prevent race conditions. - * - * @param string $container Resolved container name - */ - final protected function deleteContainerCache( $container ) { - if ( !$this->memCache->delete( $this->containerCacheKey( $container ), 300 ) ) { - trigger_error( "Unable to delete stat cache for container $container." ); - } - } - - /** - * Do a batch lookup from cache for container stats for all containers - * used in a list of container names or storage paths objects. - * This loads the persistent cache values into the process cache. - * - * @param array $items - */ - final protected function primeContainerCache( array $items ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $paths = []; // list of storage paths - $contNames = []; // (cache key => resolved container name) - // Get all the paths/containers from the items... - foreach ( $items as $item ) { - if ( self::isStoragePath( $item ) ) { - $paths[] = $item; - } elseif ( is_string( $item ) ) { // full container name - $contNames[$this->containerCacheKey( $item )] = $item; - } - } - // Get all the corresponding cache keys for paths... - foreach ( $paths as $path ) { - list( $fullCont, , ) = $this->resolveStoragePath( $path ); - if ( $fullCont !== null ) { // valid path for this backend - $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; - } - } - - $contInfo = []; // (resolved container name => cache value) - // Get all cache entries for these container cache keys... - $values = $this->memCache->getMulti( array_keys( $contNames ) ); - foreach ( $values as $cacheKey => $val ) { - $contInfo[$contNames[$cacheKey]] = $val; - } - - // Populate the container process cache for the backend... - $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); - } - - /** - * Fill the backend-specific process cache given an array of - * resolved container names and their corresponding cached info. - * Only containers that actually exist should appear in the map. - * - * @param array $containerInfo Map of resolved container names to cached info - */ - protected function doPrimeContainerCache( array $containerInfo ) { - } - - /** - * Get the cache key for a file path - * - * @param string $path Normalized storage path - * @return string - */ - private function fileCacheKey( $path ) { - return "filebackend:{$this->name}:{$this->wikiId}:file:" . sha1( $path ); - } - - /** - * Set the cached stat info for a file path. - * Negatives (404s) are not cached. By not caching negatives, we can skip cache - * salting for the case when a file is created at a path were there was none before. - * - * @param string $path Storage path - * @param array $val Stat information to cache - */ - final protected function setFileCache( $path, array $val ) { - $path = FileBackend::normalizeStoragePath( $path ); - if ( $path === null ) { - return; // invalid storage path - } - $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); - $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); - $key = $this->fileCacheKey( $path ); - // Set the cache unless it is currently salted. - $this->memCache->set( $key, $val, $ttl ); - } - - /** - * Delete the cached stat info for a file path. - * The cache key is salted for a while to prevent race conditions. - * Since negatives (404s) are not cached, this does not need to be called when - * a file is created at a path were there was none before. - * - * @param string $path Storage path - */ - final protected function deleteFileCache( $path ) { - $path = FileBackend::normalizeStoragePath( $path ); - if ( $path === null ) { - return; // invalid storage path - } - if ( !$this->memCache->delete( $this->fileCacheKey( $path ), 300 ) ) { - trigger_error( "Unable to delete stat cache for file $path." ); - } - } - - /** - * Do a batch lookup from cache for file stats for all paths - * used in a list of storage paths or FileOp objects. - * This loads the persistent cache values into the process cache. - * - * @param array $items List of storage paths - */ - final protected function primeFileCache( array $items ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $paths = []; // list of storage paths - $pathNames = []; // (cache key => storage path) - // Get all the paths/containers from the items... - foreach ( $items as $item ) { - if ( self::isStoragePath( $item ) ) { - $paths[] = FileBackend::normalizeStoragePath( $item ); - } - } - // Get rid of any paths that failed normalization... - $paths = array_filter( $paths, 'strlen' ); // remove nulls - // Get all the corresponding cache keys for paths... - foreach ( $paths as $path ) { - list( , $rel, ) = $this->resolveStoragePath( $path ); - if ( $rel !== null ) { // valid path for this backend - $pathNames[$this->fileCacheKey( $path )] = $path; - } - } - // Get all cache entries for these file cache keys... - $values = $this->memCache->getMulti( array_keys( $pathNames ) ); - foreach ( $values as $cacheKey => $val ) { - $path = $pathNames[$cacheKey]; - if ( is_array( $val ) ) { - $val['latest'] = false; // never completely trust cache - $this->cheapCache->set( $path, 'stat', $val ); - if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata - $this->cheapCache->set( $path, 'sha1', - [ 'hash' => $val['sha1'], 'latest' => false ] ); - } - if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata - $val['xattr'] = self::normalizeXAttributes( $val['xattr'] ); - $this->cheapCache->set( $path, 'xattr', - [ 'map' => $val['xattr'], 'latest' => false ] ); - } - } - } - } - - /** - * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format - * - * @param array $xattr - * @return array - * @since 1.22 - */ - final protected static function normalizeXAttributes( array $xattr ) { - $newXAttr = [ 'headers' => [], 'metadata' => [] ]; - - foreach ( $xattr['headers'] as $name => $value ) { - $newXAttr['headers'][strtolower( $name )] = $value; - } - - foreach ( $xattr['metadata'] as $name => $value ) { - $newXAttr['metadata'][strtolower( $name )] = $value; - } - - return $newXAttr; - } - - /** - * Set the 'concurrency' option from a list of operation options - * - * @param array $opts Map of operation options - * @return array - */ - final protected function setConcurrencyFlags( array $opts ) { - $opts['concurrency'] = 1; // off - if ( $this->parallelize === 'implicit' ) { - if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) { - $opts['concurrency'] = $this->concurrency; - } - } elseif ( $this->parallelize === 'explicit' ) { - if ( !empty( $opts['parallelize'] ) ) { - $opts['concurrency'] = $this->concurrency; - } - } - - return $opts; - } - - /** - * Get the content type to use in HEAD/GET requests for a file - * - * @param string $storagePath - * @param string|null $content File data - * @param string|null $fsPath File system path - * @return string MIME type - */ - protected function getContentType( $storagePath, $content, $fsPath ) { - if ( $this->mimeCallback ) { - return call_user_func_array( $this->mimeCallback, func_get_args() ); - } - - $mime = null; - if ( $fsPath !== null && function_exists( 'finfo_file' ) ) { - $finfo = finfo_open( FILEINFO_MIME_TYPE ); - $mime = finfo_file( $finfo, $fsPath ); - finfo_close( $finfo ); - } - - return is_string( $mime ) ? $mime : 'unknown/unknown'; - } -} - -/** - * FileBackendStore helper class for performing asynchronous file operations. - * - * For example, calling FileBackendStore::createInternal() with the "async" - * param flag may result in a Status that contains this object as a value. - * This class is largely backend-specific and is mostly just "magic" to be - * passed to FileBackendStore::executeOpHandlesInternal(). - */ -abstract class FileBackendStoreOpHandle { - /** @var array */ - public $params = []; // params to caller functions - /** @var FileBackendStore */ - public $backend; - /** @var array */ - public $resourcesToClose = []; - - public $call; // string; name that identifies the function called - - /** - * Close all open file handles - */ - public function closeResources() { - array_map( 'fclose', $this->resourcesToClose ); - } -} - -/** - * FileBackendStore helper function to handle listings that span container shards. - * Do not use this class from places outside of FileBackendStore. - * - * @ingroup FileBackend - */ -abstract class FileBackendStoreShardListIterator extends FilterIterator { - /** @var FileBackendStore */ - protected $backend; - - /** @var array */ - protected $params; - - /** @var string Full container name */ - protected $container; - - /** @var string Resolved relative path */ - protected $directory; - - /** @var array */ - protected $multiShardPaths = []; // (rel path => 1) - - /** - * @param FileBackendStore $backend - * @param string $container Full storage container name - * @param string $dir Storage directory relative to container - * @param array $suffixes List of container shard suffixes - * @param array $params - */ - public function __construct( - FileBackendStore $backend, $container, $dir, array $suffixes, array $params - ) { - $this->backend = $backend; - $this->container = $container; - $this->directory = $dir; - $this->params = $params; - - $iter = new AppendIterator(); - foreach ( $suffixes as $suffix ) { - $iter->append( $this->listFromShard( $this->container . $suffix ) ); - } - - parent::__construct( $iter ); - } - - public function accept() { - $rel = $this->getInnerIterator()->current(); // path relative to given directory - $path = $this->params['dir'] . "/{$rel}"; // full storage path - if ( $this->backend->isSingleShardPathInternal( $path ) ) { - return true; // path is only on one shard; no issue with duplicates - } elseif ( isset( $this->multiShardPaths[$rel] ) ) { - // Don't keep listing paths that are on multiple shards - return false; - } else { - $this->multiShardPaths[$rel] = 1; - - return true; - } - } - - public function rewind() { - parent::rewind(); - $this->multiShardPaths = []; - } - - /** - * Get the list for a given container shard - * - * @param string $container Resolved container name - * @return Iterator - */ - abstract protected function listFromShard( $container ); -} - -/** - * Iterator for listing directories - */ -class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { - protected function listFromShard( $container ) { - $list = $this->backend->getDirectoryListInternal( - $container, $this->directory, $this->params ); - if ( $list === null ) { - return new ArrayIterator( [] ); - } else { - return is_array( $list ) ? new ArrayIterator( $list ) : $list; - } - } -} - -/** - * Iterator for listing regular files - */ -class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { - protected function listFromShard( $container ) { - $list = $this->backend->getFileListInternal( - $container, $this->directory, $this->params ); - if ( $list === null ) { - return new ArrayIterator( [] ); - } else { - return is_array( $list ) ? new ArrayIterator( $list ) : $list; - } - } -} diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php deleted file mode 100644 index 56a40738e6..0000000000 --- a/includes/filebackend/FileOp.php +++ /dev/null @@ -1,848 +0,0 @@ -backend = $backend; - list( $required, $optional, $paths ) = $this->allowedParams(); - foreach ( $required as $name ) { - if ( isset( $params[$name] ) ) { - $this->params[$name] = $params[$name]; - } else { - throw new FileBackendError( "File operation missing parameter '$name'." ); - } - } - foreach ( $optional as $name ) { - if ( isset( $params[$name] ) ) { - $this->params[$name] = $params[$name]; - } - } - foreach ( $paths as $name ) { - if ( isset( $this->params[$name] ) ) { - // Normalize paths so the paths to the same file have the same string - $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] ); - } - } - } - - /** - * Normalize a string if it is a valid storage path - * - * @param string $path - * @return string - */ - protected static function normalizeIfValidStoragePath( $path ) { - if ( FileBackend::isStoragePath( $path ) ) { - $res = FileBackend::normalizeStoragePath( $path ); - - return ( $res !== null ) ? $res : $path; - } - - return $path; - } - - /** - * Set the batch UUID this operation belongs to - * - * @param string $batchId - */ - final public function setBatchId( $batchId ) { - $this->batchId = $batchId; - } - - /** - * Get the value of the parameter with the given name - * - * @param string $name - * @return mixed Returns null if the parameter is not set - */ - final public function getParam( $name ) { - return isset( $this->params[$name] ) ? $this->params[$name] : null; - } - - /** - * Check if this operation failed precheck() or attempt() - * - * @return bool - */ - final public function failed() { - return $this->failed; - } - - /** - * Get a new empty predicates array for precheck() - * - * @return array - */ - final public static function newPredicates() { - return [ 'exists' => [], 'sha1' => [] ]; - } - - /** - * Get a new empty dependency tracking array for paths read/written to - * - * @return array - */ - final public static function newDependencies() { - return [ 'read' => [], 'write' => [] ]; - } - - /** - * Update a dependency tracking array to account for this operation - * - * @param array $deps Prior path reads/writes; format of FileOp::newPredicates() - * @return array - */ - final public function applyDependencies( array $deps ) { - $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); - $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 ); - - return $deps; - } - - /** - * Check if this operation changes files listed in $paths - * - * @param array $deps Prior path reads/writes; format of FileOp::newPredicates() - * @return bool - */ - final public function dependsOn( array $deps ) { - foreach ( $this->storagePathsChanged() as $path ) { - if ( isset( $deps['read'][$path] ) || isset( $deps['write'][$path] ) ) { - return true; // "output" or "anti" dependency - } - } - foreach ( $this->storagePathsRead() as $path ) { - if ( isset( $deps['write'][$path] ) ) { - return true; // "flow" dependency - } - } - - return false; - } - - /** - * Get the file journal entries for this file operation - * - * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates) - * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates) - * @return array - */ - final public function getJournalEntries( array $oPredicates, array $nPredicates ) { - if ( !$this->doOperation ) { - return []; // this is a no-op - } - $nullEntries = []; - $updateEntries = []; - $deleteEntries = []; - $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() ); - foreach ( array_unique( $pathsUsed ) as $path ) { - $nullEntries[] = [ // assertion for recovery - 'op' => 'null', - 'path' => $path, - 'newSha1' => $this->fileSha1( $path, $oPredicates ) - ]; - } - foreach ( $this->storagePathsChanged() as $path ) { - if ( $nPredicates['sha1'][$path] === false ) { // deleted - $deleteEntries[] = [ - 'op' => 'delete', - 'path' => $path, - 'newSha1' => '' - ]; - } else { // created/updated - $updateEntries[] = [ - 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create', - 'path' => $path, - 'newSha1' => $nPredicates['sha1'][$path] - ]; - } - } - - return array_merge( $nullEntries, $updateEntries, $deleteEntries ); - } - - /** - * Check preconditions of the operation without writing anything. - * This must update $predicates for each path that the op can change - * except when a failing status object is returned. - * - * @param array $predicates - * @return Status - */ - final public function precheck( array &$predicates ) { - if ( $this->state !== self::STATE_NEW ) { - return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state ); - } - $this->state = self::STATE_CHECKED; - $status = $this->doPrecheck( $predicates ); - if ( !$status->isOK() ) { - $this->failed = true; - } - - return $status; - } - - /** - * @param array $predicates - * @return Status - */ - protected function doPrecheck( array &$predicates ) { - return Status::newGood(); - } - - /** - * Attempt the operation - * - * @return Status - */ - final public function attempt() { - if ( $this->state !== self::STATE_CHECKED ) { - return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state ); - } elseif ( $this->failed ) { // failed precheck - return Status::newFatal( 'fileop-fail-attempt-precheck' ); - } - $this->state = self::STATE_ATTEMPTED; - if ( $this->doOperation ) { - $status = $this->doAttempt(); - if ( !$status->isOK() ) { - $this->failed = true; - $this->logFailure( 'attempt' ); - } - } else { // no-op - $status = Status::newGood(); - } - - return $status; - } - - /** - * @return Status - */ - protected function doAttempt() { - return Status::newGood(); - } - - /** - * Attempt the operation in the background - * - * @return Status - */ - final public function attemptAsync() { - $this->async = true; - $result = $this->attempt(); - $this->async = false; - - return $result; - } - - /** - * Get the file operation parameters - * - * @return array (required params list, optional params list, list of params that are paths) - */ - protected function allowedParams() { - return [ [], [], [] ]; - } - - /** - * Adjust params to FileBackendStore internal file calls - * - * @param array $params - * @return array (required params list, optional params list) - */ - protected function setFlags( array $params ) { - return [ 'async' => $this->async ] + $params; - } - - /** - * Get a list of storage paths read from for this operation - * - * @return array - */ - public function storagePathsRead() { - return []; - } - - /** - * Get a list of storage paths written to for this operation - * - * @return array - */ - public function storagePathsChanged() { - return []; - } - - /** - * Check for errors with regards to the destination file already existing. - * Also set the destExists, overwriteSameCase and sourceSha1 member variables. - * A bad status will be returned if there is no chance it can be overwritten. - * - * @param array $predicates - * @return Status - */ - protected function precheckDestExistence( array $predicates ) { - $status = Status::newGood(); - // Get hash of source file/string and the destination file - $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string - if ( $this->sourceSha1 === null ) { // file in storage? - $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates ); - } - $this->overwriteSameCase = false; - $this->destExists = $this->fileExists( $this->params['dst'], $predicates ); - if ( $this->destExists ) { - if ( $this->getParam( 'overwrite' ) ) { - return $status; // OK - } elseif ( $this->getParam( 'overwriteSame' ) ) { - $dhash = $this->fileSha1( $this->params['dst'], $predicates ); - // Check if hashes are valid and match each other... - if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) { - $status->fatal( 'backend-fail-hashes' ); - } elseif ( $this->sourceSha1 !== $dhash ) { - // Give an error if the files are not identical - $status->fatal( 'backend-fail-notsame', $this->params['dst'] ); - } else { - $this->overwriteSameCase = true; // OK - } - - return $status; // do nothing; either OK or bad status - } else { - $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); - - return $status; - } - } - - return $status; - } - - /** - * precheckDestExistence() helper function to get the source file SHA-1. - * Subclasses should overwride this if the source is not in storage. - * - * @return string|bool Returns false on failure - */ - protected function getSourceSha1Base36() { - return null; // N/A - } - - /** - * Check if a file will exist in storage when this operation is attempted - * - * @param string $source Storage path - * @param array $predicates - * @return bool - */ - final protected function fileExists( $source, array $predicates ) { - if ( isset( $predicates['exists'][$source] ) ) { - return $predicates['exists'][$source]; // previous op assures this - } else { - $params = [ 'src' => $source, 'latest' => true ]; - - return $this->backend->fileExists( $params ); - } - } - - /** - * Get the SHA-1 of a file in storage when this operation is attempted - * - * @param string $source Storage path - * @param array $predicates - * @return string|bool False on failure - */ - final protected function fileSha1( $source, array $predicates ) { - if ( isset( $predicates['sha1'][$source] ) ) { - return $predicates['sha1'][$source]; // previous op assures this - } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) { - return false; // previous op assures this - } else { - $params = [ 'src' => $source, 'latest' => true ]; - - return $this->backend->getFileSha1Base36( $params ); - } - } - - /** - * Get the backend this operation is for - * - * @return FileBackendStore - */ - public function getBackend() { - return $this->backend; - } - - /** - * Log a file operation failure and preserve any temp files - * - * @param string $action - */ - final public function logFailure( $action ) { - $params = $this->params; - $params['failedAction'] = $action; - try { - wfDebugLog( 'FileOperation', get_class( $this ) . - " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) ); - } catch ( Exception $e ) { - // bad config? debug log error? - } - } -} - -/** - * Create a file in the backend with the given content. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class CreateFileOp extends FileOp { - protected function allowedParams() { - return [ - [ 'content', 'dst' ], - [ 'overwrite', 'overwriteSame', 'headers' ], - [ 'dst' ] - ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source data is too big - if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) { - $status->fatal( 'backend-fail-maxsize', - $this->params['dst'], $this->backend->maxFileSizeInternal() ); - $status->fatal( 'backend-fail-create', $this->params['dst'] ); - - return $status; - // Check if a file can be placed/changed at the destination - } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-create', $this->params['dst'] ); - - return $status; - } - // Check if destination file exists - $status->merge( $this->precheckDestExistence( $predicates ) ); - $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() - if ( $status->isOK() ) { - // Update file existence predicates - $predicates['exists'][$this->params['dst']] = true; - $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; - } - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - if ( !$this->overwriteSameCase ) { - // Create the file at the destination - return $this->backend->createInternal( $this->setFlags( $this->params ) ); - } - - return Status::newGood(); - } - - protected function getSourceSha1Base36() { - return Wikimedia\base_convert( sha1( $this->params['content'] ), 16, 36, 31 ); - } - - public function storagePathsChanged() { - return [ $this->params['dst'] ]; - } -} - -/** - * Store a file into the backend from a file on the file system. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class StoreFileOp extends FileOp { - protected function allowedParams() { - return [ - [ 'src', 'dst' ], - [ 'overwrite', 'overwriteSame', 'headers' ], - [ 'src', 'dst' ] - ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source file exists on the file system - if ( !is_file( $this->params['src'] ) ) { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - - return $status; - // Check if the source file is too big - } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) { - $status->fatal( 'backend-fail-maxsize', - $this->params['dst'], $this->backend->maxFileSizeInternal() ); - $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); - - return $status; - // Check if a file can be placed/changed at the destination - } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); - - return $status; - } - // Check if destination file exists - $status->merge( $this->precheckDestExistence( $predicates ) ); - $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() - if ( $status->isOK() ) { - // Update file existence predicates - $predicates['exists'][$this->params['dst']] = true; - $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; - } - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - if ( !$this->overwriteSameCase ) { - // Store the file at the destination - return $this->backend->storeInternal( $this->setFlags( $this->params ) ); - } - - return Status::newGood(); - } - - protected function getSourceSha1Base36() { - MediaWiki\suppressWarnings(); - $hash = sha1_file( $this->params['src'] ); - MediaWiki\restoreWarnings(); - if ( $hash !== false ) { - $hash = Wikimedia\base_convert( $hash, 16, 36, 31 ); - } - - return $hash; - } - - public function storagePathsChanged() { - return [ $this->params['dst'] ]; - } -} - -/** - * Copy a file from one storage path to another in the backend. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class CopyFileOp extends FileOp { - protected function allowedParams() { - return [ - [ 'src', 'dst' ], - [ 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ], - [ 'src', 'dst' ] - ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source file exists - if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - if ( $this->getParam( 'ignoreMissingSource' ) ) { - $this->doOperation = false; // no-op - // Update file existence predicates (cache 404s) - $predicates['exists'][$this->params['src']] = false; - $predicates['sha1'][$this->params['src']] = false; - - return $status; // nothing to do - } else { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - - return $status; - } - // Check if a file can be placed/changed at the destination - } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] ); - - return $status; - } - // Check if destination file exists - $status->merge( $this->precheckDestExistence( $predicates ) ); - $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() - if ( $status->isOK() ) { - // Update file existence predicates - $predicates['exists'][$this->params['dst']] = true; - $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; - } - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - if ( $this->overwriteSameCase ) { - $status = Status::newGood(); // nothing to do - } elseif ( $this->params['src'] === $this->params['dst'] ) { - // Just update the destination file headers - $headers = $this->getParam( 'headers' ) ?: []; - $status = $this->backend->describeInternal( $this->setFlags( [ - 'src' => $this->params['dst'], 'headers' => $headers - ] ) ); - } else { - // Copy the file to the destination - $status = $this->backend->copyInternal( $this->setFlags( $this->params ) ); - } - - return $status; - } - - public function storagePathsRead() { - return [ $this->params['src'] ]; - } - - public function storagePathsChanged() { - return [ $this->params['dst'] ]; - } -} - -/** - * Move a file from one storage path to another in the backend. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class MoveFileOp extends FileOp { - protected function allowedParams() { - return [ - [ 'src', 'dst' ], - [ 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ], - [ 'src', 'dst' ] - ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source file exists - if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - if ( $this->getParam( 'ignoreMissingSource' ) ) { - $this->doOperation = false; // no-op - // Update file existence predicates (cache 404s) - $predicates['exists'][$this->params['src']] = false; - $predicates['sha1'][$this->params['src']] = false; - - return $status; // nothing to do - } else { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - - return $status; - } - // Check if a file can be placed/changed at the destination - } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] ); - - return $status; - } - // Check if destination file exists - $status->merge( $this->precheckDestExistence( $predicates ) ); - $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() - if ( $status->isOK() ) { - // Update file existence predicates - $predicates['exists'][$this->params['src']] = false; - $predicates['sha1'][$this->params['src']] = false; - $predicates['exists'][$this->params['dst']] = true; - $predicates['sha1'][$this->params['dst']] = $this->sourceSha1; - } - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - if ( $this->overwriteSameCase ) { - if ( $this->params['src'] === $this->params['dst'] ) { - // Do nothing to the destination (which is also the source) - $status = Status::newGood(); - } else { - // Just delete the source as the destination file needs no changes - $status = $this->backend->deleteInternal( $this->setFlags( - [ 'src' => $this->params['src'] ] - ) ); - } - } elseif ( $this->params['src'] === $this->params['dst'] ) { - // Just update the destination file headers - $headers = $this->getParam( 'headers' ) ?: []; - $status = $this->backend->describeInternal( $this->setFlags( - [ 'src' => $this->params['dst'], 'headers' => $headers ] - ) ); - } else { - // Move the file to the destination - $status = $this->backend->moveInternal( $this->setFlags( $this->params ) ); - } - - return $status; - } - - public function storagePathsRead() { - return [ $this->params['src'] ]; - } - - public function storagePathsChanged() { - return [ $this->params['src'], $this->params['dst'] ]; - } -} - -/** - * Delete a file at the given storage path from the backend. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class DeleteFileOp extends FileOp { - protected function allowedParams() { - return [ [ 'src' ], [ 'ignoreMissingSource' ], [ 'src' ] ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source file exists - if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - if ( $this->getParam( 'ignoreMissingSource' ) ) { - $this->doOperation = false; // no-op - // Update file existence predicates (cache 404s) - $predicates['exists'][$this->params['src']] = false; - $predicates['sha1'][$this->params['src']] = false; - - return $status; // nothing to do - } else { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - - return $status; - } - // Check if a file can be placed/changed at the source - } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['src'] ); - $status->fatal( 'backend-fail-delete', $this->params['src'] ); - - return $status; - } - // Update file existence predicates - $predicates['exists'][$this->params['src']] = false; - $predicates['sha1'][$this->params['src']] = false; - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - // Delete the source file - return $this->backend->deleteInternal( $this->setFlags( $this->params ) ); - } - - public function storagePathsChanged() { - return [ $this->params['src'] ]; - } -} - -/** - * Change metadata for a file at the given storage path in the backend. - * Parameters for this operation are outlined in FileBackend::doOperations(). - */ -class DescribeFileOp extends FileOp { - protected function allowedParams() { - return [ [ 'src' ], [ 'headers' ], [ 'src' ] ]; - } - - protected function doPrecheck( array &$predicates ) { - $status = Status::newGood(); - // Check if the source file exists - if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - - return $status; - // Check if a file can be placed/changed at the source - } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { - $status->fatal( 'backend-fail-usable', $this->params['src'] ); - $status->fatal( 'backend-fail-describe', $this->params['src'] ); - - return $status; - } - // Update file existence predicates - $predicates['exists'][$this->params['src']] = - $this->fileExists( $this->params['src'], $predicates ); - $predicates['sha1'][$this->params['src']] = - $this->fileSha1( $this->params['src'], $predicates ); - - return $status; // safe to call attempt() - } - - protected function doAttempt() { - // Update the source file's metadata - return $this->backend->describeInternal( $this->setFlags( $this->params ) ); - } - - public function storagePathsChanged() { - return [ $this->params['src'] ]; - } -} - -/** - * Placeholder operation that has no params and does nothing - */ -class NullFileOp extends FileOp { -} diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php deleted file mode 100644 index 78209d8bf8..0000000000 --- a/includes/filebackend/FileOpBatch.php +++ /dev/null @@ -1,202 +0,0 @@ - self::MAX_BATCH_SIZE ) { - $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE ); - - return $status; - } - - $batchId = $journal->getTimestampedUUID(); - $ignoreErrors = !empty( $opts['force'] ); - $journaled = empty( $opts['nonJournaled'] ); - $maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1; - - $entries = []; // file journal entry list - $predicates = FileOp::newPredicates(); // account for previous ops in prechecks - $curBatch = []; // concurrent FileOp sub-batch accumulation - $curBatchDeps = FileOp::newDependencies(); // paths used in FileOp sub-batch - $pPerformOps = []; // ordered list of concurrent FileOp sub-batches - $lastBackend = null; // last op backend name - // Do pre-checks for each operation; abort on failure... - foreach ( $performOps as $index => $fileOp ) { - $backendName = $fileOp->getBackend()->getName(); - $fileOp->setBatchId( $batchId ); // transaction ID - // Decide if this op can be done concurrently within this sub-batch - // or if a new concurrent sub-batch must be started after this one... - if ( $fileOp->dependsOn( $curBatchDeps ) - || count( $curBatch ) >= $maxConcurrency - || ( $backendName !== $lastBackend && count( $curBatch ) ) - ) { - $pPerformOps[] = $curBatch; // push this batch - $curBatch = []; // start a new sub-batch - $curBatchDeps = FileOp::newDependencies(); - } - $lastBackend = $backendName; - $curBatch[$index] = $fileOp; // keep index - // Update list of affected paths in this batch - $curBatchDeps = $fileOp->applyDependencies( $curBatchDeps ); - // Simulate performing the operation... - $oldPredicates = $predicates; - $subStatus = $fileOp->precheck( $predicates ); // updates $predicates - $status->merge( $subStatus ); - if ( $subStatus->isOK() ) { - if ( $journaled ) { // journal log entries - $entries = array_merge( $entries, - $fileOp->getJournalEntries( $oldPredicates, $predicates ) ); - } - } else { // operation failed? - $status->success[$index] = false; - ++$status->failCount; - if ( !$ignoreErrors ) { - return $status; // abort - } - } - } - // Push the last sub-batch - if ( count( $curBatch ) ) { - $pPerformOps[] = $curBatch; - } - - // Log the operations in the file journal... - if ( count( $entries ) ) { - $subStatus = $journal->logChangeBatch( $entries, $batchId ); - if ( !$subStatus->isOK() ) { - return $subStatus; // abort - } - } - - if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings - $status->setResult( true, $status->value ); - } - - // Attempt each operation (in parallel if allowed and possible)... - self::runParallelBatches( $pPerformOps, $status ); - - return $status; - } - - /** - * Attempt a list of file operations sub-batches in series. - * - * The operations *in* each sub-batch will be done in parallel. - * The caller is responsible for making sure the operations - * within any given sub-batch do not depend on each other. - * This will abort remaining ops on failure. - * - * @param array $pPerformOps Batches of file ops (batches use original indexes) - * @param Status $status - */ - protected static function runParallelBatches( array $pPerformOps, Status $status ) { - $aborted = false; // set to true on unexpected errors - foreach ( $pPerformOps as $performOpsBatch ) { - /** @var FileOp[] $performOpsBatch */ - if ( $aborted ) { // check batch op abort flag... - // We can't continue (even with $ignoreErrors) as $predicates is wrong. - // Log the remaining ops as failed for recovery... - foreach ( $performOpsBatch as $i => $fileOp ) { - $status->success[$i] = false; - ++$status->failCount; - $performOpsBatch[$i]->logFailure( 'attempt_aborted' ); - } - continue; - } - /** @var Status[] $statuses */ - $statuses = []; - $opHandles = []; - // Get the backend; all sub-batch ops belong to a single backend - $backend = reset( $performOpsBatch )->getBackend(); - // Get the operation handles or actually do it if there is just one. - // If attemptAsync() returns a Status, it was either due to an error - // or the backend does not support async ops and did it synchronously. - foreach ( $performOpsBatch as $i => $fileOp ) { - if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() - // Parallel ops may be disabled in config due to missing dependencies, - // (e.g. needing popen()). When they are, $performOpsBatch has size 1. - $subStatus = ( count( $performOpsBatch ) > 1 ) - ? $fileOp->attemptAsync() - : $fileOp->attempt(); - if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { - $opHandles[$i] = $subStatus->value; // deferred - } else { - $statuses[$i] = $subStatus; // done already - } - } - } - // Try to do all the operations concurrently... - $statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles ); - // Marshall and merge all the responses (blocking)... - foreach ( $performOpsBatch as $i => $fileOp ) { - if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck() - $subStatus = $statuses[$i]; - $status->merge( $subStatus ); - if ( $subStatus->isOK() ) { - $status->success[$i] = true; - ++$status->successCount; - } else { - $status->success[$i] = false; - ++$status->failCount; - $aborted = true; // set abort flag; we can't continue - } - } - } - } - } -} diff --git a/includes/filebackend/MemoryFileBackend.php b/includes/filebackend/MemoryFileBackend.php deleted file mode 100644 index e2c1ede46d..0000000000 --- a/includes/filebackend/MemoryFileBackend.php +++ /dev/null @@ -1,263 +0,0 @@ - (data,mtime) */ - protected $files = []; - - public function getFeatures() { - return self::ATTR_UNICODE_PATHS; - } - - public function isPathUsableInternal( $storagePath ) { - return true; - } - - protected function doCreateInternal( array $params ) { - $status = Status::newGood(); - - $dst = $this->resolveHashKey( $params['dst'] ); - if ( $dst === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - $this->files[$dst] = [ - 'data' => $params['content'], - 'mtime' => wfTimestamp( TS_MW, time() ) - ]; - - return $status; - } - - protected function doStoreInternal( array $params ) { - $status = Status::newGood(); - - $dst = $this->resolveHashKey( $params['dst'] ); - if ( $dst === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - MediaWiki\suppressWarnings(); - $data = file_get_contents( $params['src'] ); - MediaWiki\restoreWarnings(); - if ( $data === false ) { // source doesn't exist? - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - - return $status; - } - - $this->files[$dst] = [ - 'data' => $data, - 'mtime' => wfTimestamp( TS_MW, time() ) - ]; - - return $status; - } - - protected function doCopyInternal( array $params ) { - $status = Status::newGood(); - - $src = $this->resolveHashKey( $params['src'] ); - if ( $src === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - $dst = $this->resolveHashKey( $params['dst'] ); - if ( $dst === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - if ( !isset( $this->files[$src] ) ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } - - return $status; - } - - $this->files[$dst] = [ - 'data' => $this->files[$src]['data'], - 'mtime' => wfTimestamp( TS_MW, time() ) - ]; - - return $status; - } - - protected function doDeleteInternal( array $params ) { - $status = Status::newGood(); - - $src = $this->resolveHashKey( $params['src'] ); - if ( $src === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - if ( !isset( $this->files[$src] ) ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } - - return $status; - } - - unset( $this->files[$src] ); - - return $status; - } - - protected function doGetFileStat( array $params ) { - $src = $this->resolveHashKey( $params['src'] ); - if ( $src === null ) { - return null; - } - - if ( isset( $this->files[$src] ) ) { - return [ - 'mtime' => $this->files[$src]['mtime'], - 'size' => strlen( $this->files[$src]['data'] ), - ]; - } - - return false; - } - - protected function doGetLocalCopyMulti( array $params ) { - $tmpFiles = []; // (path => TempFSFile) - foreach ( $params['srcs'] as $srcPath ) { - $src = $this->resolveHashKey( $srcPath ); - if ( $src === null || !isset( $this->files[$src] ) ) { - $fsFile = null; - } else { - // Create a new temporary file with the same extension... - $ext = FileBackend::extensionFromPath( $src ); - $fsFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( $fsFile ) { - $bytes = file_put_contents( $fsFile->getPath(), $this->files[$src]['data'] ); - if ( $bytes !== strlen( $this->files[$src]['data'] ) ) { - $fsFile = null; - } - } - } - $tmpFiles[$srcPath] = $fsFile; - } - - return $tmpFiles; - } - - protected function doDirectoryExists( $container, $dir, array $params ) { - $prefix = rtrim( "$container/$dir", '/' ) . '/'; - foreach ( $this->files as $path => $data ) { - if ( strpos( $path, $prefix ) === 0 ) { - return true; - } - } - - return false; - } - - public function getDirectoryListInternal( $container, $dir, array $params ) { - $dirs = []; - $prefix = rtrim( "$container/$dir", '/' ) . '/'; - $prefixLen = strlen( $prefix ); - foreach ( $this->files as $path => $data ) { - if ( strpos( $path, $prefix ) === 0 ) { - $relPath = substr( $path, $prefixLen ); - if ( $relPath === false ) { - continue; - } elseif ( strpos( $relPath, '/' ) === false ) { - continue; // just a file - } - $parts = array_slice( explode( '/', $relPath ), 0, -1 ); // last part is file name - if ( !empty( $params['topOnly'] ) ) { - $dirs[$parts[0]] = 1; // top directory - } else { - $current = ''; - foreach ( $parts as $part ) { // all directories - $dir = ( $current === '' ) ? $part : "$current/$part"; - $dirs[$dir] = 1; - $current = $dir; - } - } - } - } - - return array_keys( $dirs ); - } - - public function getFileListInternal( $container, $dir, array $params ) { - $files = []; - $prefix = rtrim( "$container/$dir", '/' ) . '/'; - $prefixLen = strlen( $prefix ); - foreach ( $this->files as $path => $data ) { - if ( strpos( $path, $prefix ) === 0 ) { - $relPath = substr( $path, $prefixLen ); - if ( $relPath === false ) { - continue; - } elseif ( !empty( $params['topOnly'] ) && strpos( $relPath, '/' ) !== false ) { - continue; - } - $files[] = $relPath; - } - } - - return $files; - } - - protected function directoriesAreVirtual() { - return true; - } - - /** - * Get the absolute file system path for a storage path - * - * @param string $storagePath Storage path - * @return string|null - */ - protected function resolveHashKey( $storagePath ) { - list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath ); - if ( $relPath === null ) { - return null; // invalid - } - - return ( $relPath !== '' ) ? "$fullCont/$relPath" : $fullCont; - } -} diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php deleted file mode 100644 index 2adf934a55..0000000000 --- a/includes/filebackend/SwiftFileBackend.php +++ /dev/null @@ -1,1940 +0,0 @@ -swiftAuthUrl = $config['swiftAuthUrl']; - $this->swiftUser = $config['swiftUser']; - $this->swiftKey = $config['swiftKey']; - // Optional settings - $this->authTTL = isset( $config['swiftAuthTTL'] ) - ? $config['swiftAuthTTL'] - : 15 * 60; // some sane number - $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) - ? $config['swiftTempUrlKey'] - : ''; - $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) - ? $config['shardViaHashLevels'] - : ''; - $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) - ? $config['rgwS3AccessKey'] - : ''; - $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) - ? $config['rgwS3SecretKey'] - : ''; - // HTTP helper client - $this->http = new MultiHttpClient( [] ); - // Cache container information to mask latency - if ( isset( $config['wanCache'] ) && $config['wanCache'] instanceof WANObjectCache ) { - $this->memCache = $config['wanCache']; - } - // Process cache for container info - $this->containerStatCache = new ProcessCacheLRU( 300 ); - // Cache auth token information to avoid RTTs - if ( !empty( $config['cacheAuthInfo'] ) ) { - if ( PHP_SAPI === 'cli' ) { - // Preferrably memcached - $this->srvCache = ObjectCache::getLocalClusterInstance(); - } else { - // Look for APC, XCache, WinCache, ect... - $this->srvCache = ObjectCache::getLocalServerInstance( CACHE_NONE ); - } - } else { - $this->srvCache = new EmptyBagOStuff(); - } - } - - public function getFeatures() { - return ( FileBackend::ATTR_UNICODE_PATHS | - FileBackend::ATTR_HEADERS | FileBackend::ATTR_METADATA ); - } - - protected function resolveContainerPath( $container, $relStoragePath ) { - if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { - return null; // not UTF-8, makes it hard to use CF and the swift HTTP API - } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { - return null; // too long for Swift - } - - return $relStoragePath; - } - - public function isPathUsableInternal( $storagePath ) { - list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); - if ( $rel === null ) { - return false; // invalid - } - - return is_array( $this->getContainerStat( $container ) ); - } - - /** - * Sanitize and filter the custom headers from a $params array. - * Only allows certain "standard" Content- and X-Content- headers. - * - * @param array $params - * @return array Sanitized value of 'headers' field in $params - */ - protected function sanitizeHdrs( array $params ) { - return isset( $params['headers'] ) - ? $this->getCustomHeaders( $params['headers'] ) - : []; - - } - - /** - * @param array $rawHeaders - * @return array Custom non-metadata HTTP headers - */ - protected function getCustomHeaders( array $rawHeaders ) { - $headers = []; - - // Normalize casing, and strip out illegal headers - foreach ( $rawHeaders as $name => $value ) { - $name = strtolower( $name ); - if ( preg_match( '/^content-(type|length)$/', $name ) ) { - continue; // blacklisted - } elseif ( preg_match( '/^(x-)?content-/', $name ) ) { - $headers[$name] = $value; // allowed - } elseif ( preg_match( '/^content-(disposition)/', $name ) ) { - $headers[$name] = $value; // allowed - } - } - // By default, Swift has annoyingly low maximum header value limits - if ( isset( $headers['content-disposition'] ) ) { - $disposition = ''; - // @note: assume FileBackend::makeContentDisposition() already used - foreach ( explode( ';', $headers['content-disposition'] ) as $part ) { - $part = trim( $part ); - $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}"; - if ( strlen( $new ) <= 255 ) { - $disposition = $new; - } else { - break; // too long; sigh - } - } - $headers['content-disposition'] = $disposition; - } - - return $headers; - } - - /** - * @param array $rawHeaders - * @return array Custom metadata headers - */ - protected function getMetadataHeaders( array $rawHeaders ) { - $headers = []; - foreach ( $rawHeaders as $name => $value ) { - $name = strtolower( $name ); - if ( strpos( $name, 'x-object-meta-' ) === 0 ) { - $headers[$name] = $value; - } - } - - return $headers; - } - - /** - * @param array $rawHeaders - * @return array Custom metadata headers with prefix removed - */ - protected function getMetadata( array $rawHeaders ) { - $metadata = []; - foreach ( $this->getMetadataHeaders( $rawHeaders ) as $name => $value ) { - $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value; - } - - return $metadata; - } - - protected function doCreateInternal( array $params ) { - $status = Status::newGood(); - - list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - $sha1Hash = Wikimedia\base_convert( sha1( $params['content'] ), 16, 36, 31 ); - $contentType = isset( $params['headers']['content-type'] ) - ? $params['headers']['content-type'] - : $this->getContentType( $params['dst'], $params['content'], null ); - - $reqs = [ [ - 'method' => 'PUT', - 'url' => [ $dstCont, $dstRel ], - 'headers' => [ - 'content-length' => strlen( $params['content'] ), - 'etag' => md5( $params['content'] ), - 'content-type' => $contentType, - 'x-object-meta-sha1base36' => $sha1Hash - ] + $this->sanitizeHdrs( $params ), - 'body' => $params['content'] - ] ]; - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $rcode === 201 ) { - // good - } elseif ( $rcode === 412 ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually write the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doStoreInternal( array $params ) { - $status = Status::newGood(); - - list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - MediaWiki\suppressWarnings(); - $sha1Hash = sha1_file( $params['src'] ); - MediaWiki\restoreWarnings(); - if ( $sha1Hash === false ) { // source doesn't exist? - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - - return $status; - } - $sha1Hash = Wikimedia\base_convert( $sha1Hash, 16, 36, 31 ); - $contentType = isset( $params['headers']['content-type'] ) - ? $params['headers']['content-type'] - : $this->getContentType( $params['dst'], null, $params['src'] ); - - $handle = fopen( $params['src'], 'rb' ); - if ( $handle === false ) { // source doesn't exist? - $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); - - return $status; - } - - $reqs = [ [ - 'method' => 'PUT', - 'url' => [ $dstCont, $dstRel ], - 'headers' => [ - 'content-length' => filesize( $params['src'] ), - 'etag' => md5_file( $params['src'] ), - 'content-type' => $contentType, - 'x-object-meta-sha1base36' => $sha1Hash - ] + $this->sanitizeHdrs( $params ), - 'body' => $handle // resource - ] ]; - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $rcode === 201 ) { - // good - } elseif ( $rcode === 412 ) { - $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually write the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doCopyInternal( array $params ) { - $status = Status::newGood(); - - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - $reqs = [ [ - 'method' => 'PUT', - 'url' => [ $dstCont, $dstRel ], - 'headers' => [ - 'x-copy-from' => '/' . rawurlencode( $srcCont ) . - '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) - ] + $this->sanitizeHdrs( $params ), // extra headers merged into object - ] ]; - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $rcode === 201 ) { - // good - } elseif ( $rcode === 404 ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually write the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doMoveInternal( array $params ) { - $status = Status::newGood(); - - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - - return $status; - } - - $reqs = [ - [ - 'method' => 'PUT', - 'url' => [ $dstCont, $dstRel ], - 'headers' => [ - 'x-copy-from' => '/' . rawurlencode( $srcCont ) . - '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) ) - ] + $this->sanitizeHdrs( $params ) // extra headers merged into object - ] - ]; - if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) { - $reqs[] = [ - 'method' => 'DELETE', - 'url' => [ $srcCont, $srcRel ], - 'headers' => [] - ]; - } - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $request['method'] === 'PUT' && $rcode === 201 ) { - // good - } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) { - // good - } elseif ( $rcode === 404 ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually move the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doDeleteInternal( array $params ) { - $status = Status::newGood(); - - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - $reqs = [ [ - 'method' => 'DELETE', - 'url' => [ $srcCont, $srcRel ], - 'headers' => [] - ] ]; - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $rcode === 204 ) { - // good - } elseif ( $rcode === 404 ) { - if ( empty( $params['ignoreMissingSource'] ) ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); - } - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually delete the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doDescribeInternal( array $params ) { - $status = Status::newGood(); - - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - // Fetch the old object headers/metadata...this should be in stat cache by now - $stat = $this->getFileStat( [ 'src' => $params['src'], 'latest' => 1 ] ); - if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry - $stat = $this->doGetFileStat( [ 'src' => $params['src'], 'latest' => 1 ] ); - } - if ( !$stat ) { - $status->fatal( 'backend-fail-describe', $params['src'] ); - - return $status; - } - - // POST clears prior headers, so we need to merge the changes in to the old ones - $metaHdrs = []; - foreach ( $stat['xattr']['metadata'] as $name => $value ) { - $metaHdrs["x-object-meta-$name"] = $value; - } - $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers']; - - $reqs = [ [ - 'method' => 'POST', - 'url' => [ $srcCont, $srcRel ], - 'headers' => $metaHdrs + $customHdrs - ] ]; - - $method = __METHOD__; - $handler = function ( array $request, Status $status ) use ( $method, $params ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response']; - if ( $rcode === 202 ) { - // good - } elseif ( $rcode === 404 ) { - $status->fatal( 'backend-fail-describe', $params['src'] ); - } else { - $this->onError( $status, $method, $params, $rerr, $rcode, $rdesc ); - } - }; - - $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs ); - if ( !empty( $params['async'] ) ) { // deferred - $status->value = $opHandle; - } else { // actually change the object in Swift - $status->merge( current( $this->doExecuteOpHandlesInternal( [ $opHandle ] ) ) ); - } - - return $status; - } - - protected function doPrepareInternal( $fullCont, $dir, array $params ) { - $status = Status::newGood(); - - // (a) Check if container already exists - $stat = $this->getContainerStat( $fullCont ); - if ( is_array( $stat ) ) { - return $status; // already there - } elseif ( $stat === null ) { - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); - - return $status; - } - - // (b) Create container as needed with proper ACLs - if ( $stat === false ) { - $params['op'] = 'prepare'; - $status->merge( $this->createContainer( $fullCont, $params ) ); - } - - return $status; - } - - protected function doSecureInternal( $fullCont, $dir, array $params ) { - $status = Status::newGood(); - if ( empty( $params['noAccess'] ) ) { - return $status; // nothing to do - } - - $stat = $this->getContainerStat( $fullCont ); - if ( is_array( $stat ) ) { - // Make container private to end-users... - $status->merge( $this->setContainerAccess( - $fullCont, - [ $this->swiftUser ], // read - [ $this->swiftUser ] // write - ) ); - } elseif ( $stat === false ) { - $status->fatal( 'backend-fail-usable', $params['dir'] ); - } else { - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); - } - - return $status; - } - - protected function doPublishInternal( $fullCont, $dir, array $params ) { - $status = Status::newGood(); - - $stat = $this->getContainerStat( $fullCont ); - if ( is_array( $stat ) ) { - // Make container public to end-users... - $status->merge( $this->setContainerAccess( - $fullCont, - [ $this->swiftUser, '.r:*' ], // read - [ $this->swiftUser ] // write - ) ); - } elseif ( $stat === false ) { - $status->fatal( 'backend-fail-usable', $params['dir'] ); - } else { - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); - } - - return $status; - } - - protected function doCleanInternal( $fullCont, $dir, array $params ) { - $status = Status::newGood(); - - // Only containers themselves can be removed, all else is virtual - if ( $dir != '' ) { - return $status; // nothing to do - } - - // (a) Check the container - $stat = $this->getContainerStat( $fullCont, true ); - if ( $stat === false ) { - return $status; // ok, nothing to do - } elseif ( !is_array( $stat ) ) { - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); - - return $status; - } - - // (b) Delete the container if empty - if ( $stat['count'] == 0 ) { - $params['op'] = 'clean'; - $status->merge( $this->deleteContainer( $fullCont, $params ) ); - } - - return $status; - } - - protected function doGetFileStat( array $params ) { - $params = [ 'srcs' => [ $params['src'] ], 'concurrency' => 1 ] + $params; - unset( $params['src'] ); - $stats = $this->doGetFileStatMulti( $params ); - - return reset( $stats ); - } - - /** - * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z". - * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings, - * missing the timezone suffix (though Ceph RGW does not appear to have this bug). - * - * @param string $ts - * @param int $format Output format (TS_* constant) - * @return string - * @throws FileBackendError - */ - protected function convertSwiftDate( $ts, $format = TS_MW ) { - try { - $timestamp = new MWTimestamp( $ts ); - - return $timestamp->getTimestamp( $format ); - } catch ( Exception $e ) { - throw new FileBackendError( $e->getMessage() ); - } - } - - /** - * Fill in any missing object metadata and save it to Swift - * - * @param array $objHdrs Object response headers - * @param string $path Storage path to object - * @return array New headers - */ - protected function addMissingMetadata( array $objHdrs, $path ) { - if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) { - return $objHdrs; // nothing to do - } - - /** @noinspection PhpUnusedLocalVariableInspection */ - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ": $path was not stored with SHA-1 metadata." ); - - $objHdrs['x-object-meta-sha1base36'] = false; - - $auth = $this->getAuthentication(); - if ( !$auth ) { - return $objHdrs; // failed - } - - // Find prior custom HTTP headers - $postHeaders = $this->getCustomHeaders( $objHdrs ); - // Find prior metadata headers - $postHeaders += $this->getMetadataHeaders( $objHdrs ); - - $status = Status::newGood(); - /** @noinspection PhpUnusedLocalVariableInspection */ - $scopeLockS = $this->getScopedFileLocks( [ $path ], LockManager::LOCK_UW, $status ); - if ( $status->isOK() ) { - $tmpFile = $this->getLocalCopy( [ 'src' => $path, 'latest' => 1 ] ); - if ( $tmpFile ) { - $hash = $tmpFile->getSha1Base36(); - if ( $hash !== false ) { - $objHdrs['x-object-meta-sha1base36'] = $hash; - // Merge new SHA1 header into the old ones - $postHeaders['x-object-meta-sha1base36'] = $hash; - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - list( $rcode ) = $this->http->run( [ - 'method' => 'POST', - 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), - 'headers' => $this->authTokenHeaders( $auth ) + $postHeaders - ] ); - if ( $rcode >= 200 && $rcode <= 299 ) { - $this->deleteFileCache( $path ); - - return $objHdrs; // success - } - } - } - } - - wfDebugLog( 'SwiftBackend', __METHOD__ . ": unable to set SHA-1 metadata for $path" ); - - return $objHdrs; // failed - } - - protected function doGetFileContentsMulti( array $params ) { - $contents = []; - - $auth = $this->getAuthentication(); - - $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging - // Blindly create tmp files and stream to them, catching any exception if the file does - // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). - $reqs = []; // (path => op) - - foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - if ( $srcRel === null || !$auth ) { - $contents[$path] = false; - continue; - } - // Create a new temporary memory file... - $handle = fopen( 'php://temp', 'wb' ); - if ( $handle ) { - $reqs[$path] = [ - 'method' => 'GET', - 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), - 'headers' => $this->authTokenHeaders( $auth ) - + $this->headersFromParams( $params ), - 'stream' => $handle, - ]; - } - $contents[$path] = false; - } - - $opts = [ 'maxConnsPerHost' => $params['concurrency'] ]; - $reqs = $this->http->runMulti( $reqs, $opts ); - foreach ( $reqs as $path => $op ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; - if ( $rcode >= 200 && $rcode <= 299 ) { - rewind( $op['stream'] ); // start from the beginning - $contents[$path] = stream_get_contents( $op['stream'] ); - } elseif ( $rcode === 404 ) { - $contents[$path] = false; - } else { - $this->onError( null, __METHOD__, - [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc ); - } - fclose( $op['stream'] ); // close open handle - } - - return $contents; - } - - protected function doDirectoryExists( $fullCont, $dir, array $params ) { - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix ); - if ( $status->isOK() ) { - return ( count( $status->value ) ) > 0; - } - - return null; // error - } - - /** - * @see FileBackendStore::getDirectoryListInternal() - * @param string $fullCont - * @param string $dir - * @param array $params - * @return SwiftFileBackendDirList - */ - public function getDirectoryListInternal( $fullCont, $dir, array $params ) { - return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params ); - } - - /** - * @see FileBackendStore::getFileListInternal() - * @param string $fullCont - * @param string $dir - * @param array $params - * @return SwiftFileBackendFileList - */ - public function getFileListInternal( $fullCont, $dir, array $params ) { - return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params ); - } - - /** - * Do not call this function outside of SwiftFileBackendFileList - * - * @param string $fullCont Resolved container name - * @param string $dir Resolved storage directory with no trailing slash - * @param string|null $after Resolved container relative path to list items after - * @param int $limit Max number of items to list - * @param array $params Parameters for getDirectoryList() - * @return array List of container relative resolved paths of directories directly under $dir - * @throws FileBackendError - */ - public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { - $dirs = []; - if ( $after === INF ) { - return $dirs; // nothing more - } - - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - // Non-recursive: only list dirs right under $dir - if ( !empty( $params['topOnly'] ) ) { - $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); - if ( !$status->isOK() ) { - throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); - } - $objects = $status->value; - foreach ( $objects as $object ) { // files and directories - if ( substr( $object, -1 ) === '/' ) { - $dirs[] = $object; // directories end in '/' - } - } - } else { - // Recursive: list all dirs under $dir and its subdirs - $getParentDir = function ( $path ) { - return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; - }; - - // Get directory from last item of prior page - $lastDir = $getParentDir( $after ); // must be first page - $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); - - if ( !$status->isOK() ) { - throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); - } - - $objects = $status->value; - - foreach ( $objects as $object ) { // files - $objectDir = $getParentDir( $object ); // directory of object - - if ( $objectDir !== false && $objectDir !== $dir ) { - // Swift stores paths in UTF-8, using binary sorting. - // See function "create_container_table" in common/db.py. - // If a directory is not "greater" than the last one, - // then it was already listed by the calling iterator. - if ( strcmp( $objectDir, $lastDir ) > 0 ) { - $pDir = $objectDir; - do { // add dir and all its parent dirs - $dirs[] = "{$pDir}/"; - $pDir = $getParentDir( $pDir ); - } while ( $pDir !== false // sanity - && strcmp( $pDir, $lastDir ) > 0 // not done already - && strlen( $pDir ) > strlen( $dir ) // within $dir - ); - } - $lastDir = $objectDir; - } - } - } - // Page on the unfiltered directory listing (what is returned may be filtered) - if ( count( $objects ) < $limit ) { - $after = INF; // avoid a second RTT - } else { - $after = end( $objects ); // update last item - } - - return $dirs; - } - - /** - * Do not call this function outside of SwiftFileBackendFileList - * - * @param string $fullCont Resolved container name - * @param string $dir Resolved storage directory with no trailing slash - * @param string|null $after Resolved container relative path of file to list items after - * @param int $limit Max number of items to list - * @param array $params Parameters for getDirectoryList() - * @return array List of resolved container relative paths of files under $dir - * @throws FileBackendError - */ - public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { - $files = []; // list of (path, stat array or null) entries - if ( $after === INF ) { - return $files; // nothing more - } - - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - $prefix = ( $dir == '' ) ? null : "{$dir}/"; - // $objects will contain a list of unfiltered names or CF_Object items - // Non-recursive: only list files right under $dir - if ( !empty( $params['topOnly'] ) ) { - if ( !empty( $params['adviseStat'] ) ) { - $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' ); - } else { - $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' ); - } - } else { - // Recursive: list all files under $dir and its subdirs - if ( !empty( $params['adviseStat'] ) ) { - $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix ); - } else { - $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix ); - } - } - - // Reformat this list into a list of (name, stat array or null) entries - if ( !$status->isOK() ) { - throw new FileBackendError( "Iterator page I/O error: {$status->getMessage()}" ); - } - - $objects = $status->value; - $files = $this->buildFileObjectListing( $params, $dir, $objects ); - - // Page on the unfiltered object listing (what is returned may be filtered) - if ( count( $objects ) < $limit ) { - $after = INF; // avoid a second RTT - } else { - $after = end( $objects ); // update last item - $after = is_object( $after ) ? $after->name : $after; - } - - return $files; - } - - /** - * Build a list of file objects, filtering out any directories - * and extracting any stat info if provided in $objects (for CF_Objects) - * - * @param array $params Parameters for getDirectoryList() - * @param string $dir Resolved container directory path - * @param array $objects List of CF_Object items or object names - * @return array List of (names,stat array or null) entries - */ - private function buildFileObjectListing( array $params, $dir, array $objects ) { - $names = []; - foreach ( $objects as $object ) { - if ( is_object( $object ) ) { - if ( isset( $object->subdir ) || !isset( $object->name ) ) { - continue; // virtual directory entry; ignore - } - $stat = [ - // Convert various random Swift dates to TS_MW - 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ), - 'size' => (int)$object->bytes, - 'sha1' => null, - // Note: manifiest ETags are not an MD5 of the file - 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null, - 'latest' => false // eventually consistent - ]; - $names[] = [ $object->name, $stat ]; - } elseif ( substr( $object, -1 ) !== '/' ) { - // Omit directories, which end in '/' in listings - $names[] = [ $object, null ]; - } - } - - return $names; - } - - /** - * Do not call this function outside of SwiftFileBackendFileList - * - * @param string $path Storage path - * @param array $val Stat value - */ - public function loadListingStatInternal( $path, array $val ) { - $this->cheapCache->set( $path, 'stat', $val ); - } - - protected function doGetFileXAttributes( array $params ) { - $stat = $this->getFileStat( $params ); - if ( $stat ) { - if ( !isset( $stat['xattr'] ) ) { - // Stat entries filled by file listings don't include metadata/headers - $this->clearCache( [ $params['src'] ] ); - $stat = $this->getFileStat( $params ); - } - - return $stat['xattr']; - } else { - return false; - } - } - - protected function doGetFileSha1base36( array $params ) { - $stat = $this->getFileStat( $params ); - if ( $stat ) { - if ( !isset( $stat['sha1'] ) ) { - // Stat entries filled by file listings don't include SHA1 - $this->clearCache( [ $params['src'] ] ); - $stat = $this->getFileStat( $params ); - } - - return $stat['sha1']; - } else { - return false; - } - } - - protected function doStreamFile( array $params ) { - $status = Status::newGood(); - - $flags = !empty( $params['headless'] ) ? StreamFile::STREAM_HEADLESS : 0; - - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - StreamFile::send404Message( $params['src'], $flags ); - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - - return $status; - } - - $auth = $this->getAuthentication(); - if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) { - StreamFile::send404Message( $params['src'], $flags ); - $status->fatal( 'backend-fail-stream', $params['src'] ); - - return $status; - } - - // If "headers" is set, we only want to send them if the file is there. - // Do not bother checking if the file exists if headers are not set though. - if ( $params['headers'] && !$this->fileExists( $params ) ) { - StreamFile::send404Message( $params['src'], $flags ); - $status->fatal( 'backend-fail-stream', $params['src'] ); - - return $status; - } - - // Send the requested additional headers - foreach ( $params['headers'] as $header ) { - header( $header ); // aways send - } - - if ( empty( $params['allowOB'] ) ) { - // Cancel output buffering and gzipping if set - wfResetOutputBuffers(); - } - - $handle = fopen( 'php://output', 'wb' ); - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'GET', - 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), - 'headers' => $this->authTokenHeaders( $auth ) - + $this->headersFromParams( $params ) + $params['options'], - 'stream' => $handle, - 'flags' => [ 'relayResponseHeaders' => empty( $params['headless'] ) ] - ] ); - - if ( $rcode >= 200 && $rcode <= 299 ) { - // good - } elseif ( $rcode === 404 ) { - $status->fatal( 'backend-fail-stream', $params['src'] ); - // Per bug 41113, nasty things can happen if bad cache entries get - // stuck in cache. It's also possible that this error can come up - // with simple race conditions. Clear out the stat cache to be safe. - $this->clearCache( [ $params['src'] ] ); - $this->deleteFileCache( $params['src'] ); - } else { - $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); - } - - return $status; - } - - protected function doGetLocalCopyMulti( array $params ) { - $tmpFiles = []; - - $auth = $this->getAuthentication(); - - $ep = array_diff_key( $params, [ 'srcs' => 1 ] ); // for error logging - // Blindly create tmp files and stream to them, catching any exception if the file does - // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). - $reqs = []; // (path => op) - - foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - if ( $srcRel === null || !$auth ) { - $tmpFiles[$path] = null; - continue; - } - // Get source file extension - $ext = FileBackend::extensionFromPath( $path ); - // Create a new temporary file... - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( $tmpFile ) { - $handle = fopen( $tmpFile->getPath(), 'wb' ); - if ( $handle ) { - $reqs[$path] = [ - 'method' => 'GET', - 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), - 'headers' => $this->authTokenHeaders( $auth ) - + $this->headersFromParams( $params ), - 'stream' => $handle, - ]; - } else { - $tmpFile = null; - } - } - $tmpFiles[$path] = $tmpFile; - } - - $isLatest = ( $this->isRGW || !empty( $params['latest'] ) ); - $opts = [ 'maxConnsPerHost' => $params['concurrency'] ]; - $reqs = $this->http->runMulti( $reqs, $opts ); - foreach ( $reqs as $path => $op ) { - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response']; - fclose( $op['stream'] ); // close open handle - if ( $rcode >= 200 && $rcode <= 299 ) { - $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0; - // Double check that the disk is not full/broken - if ( $size != $rhdrs['content-length'] ) { - $tmpFiles[$path] = null; - $rerr = "Got {$size}/{$rhdrs['content-length']} bytes"; - $this->onError( null, __METHOD__, - [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc ); - } - // Set the file stat process cache in passing - $stat = $this->getStatFromHeaders( $rhdrs ); - $stat['latest'] = $isLatest; - $this->cheapCache->set( $path, 'stat', $stat ); - } elseif ( $rcode === 404 ) { - $tmpFiles[$path] = false; - } else { - $tmpFiles[$path] = null; - $this->onError( null, __METHOD__, - [ 'src' => $path ] + $ep, $rerr, $rcode, $rdesc ); - } - } - - return $tmpFiles; - } - - public function getFileHttpUrl( array $params ) { - if ( $this->swiftTempUrlKey != '' || - ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) - ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return null; // invalid path - } - - $auth = $this->getAuthentication(); - if ( !$auth ) { - return null; - } - - $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; - $expires = time() + $ttl; - - if ( $this->swiftTempUrlKey != '' ) { - $url = $this->storageUrl( $auth, $srcCont, $srcRel ); - // Swift wants the signature based on the unencoded object name - $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH ); - $signature = hash_hmac( 'sha1', - "GET\n{$expires}\n{$contPath}/{$srcRel}", - $this->swiftTempUrlKey - ); - - return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}"; - } else { // give S3 API URL for rgw - // Path for signature starts with the bucket - $spath = '/' . rawurlencode( $srcCont ) . '/' . - str_replace( '%2F', '/', rawurlencode( $srcRel ) ); - // Calculate the hash - $signature = base64_encode( hash_hmac( - 'sha1', - "GET\n\n\n{$expires}\n{$spath}", - $this->rgwS3SecretKey, - true // raw - ) ); - // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. - // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. - return wfAppendQuery( - str_replace( '/swift/v1', '', // S3 API is the rgw default - $this->storageUrl( $auth ) . $spath ), - [ - 'Signature' => $signature, - 'Expires' => $expires, - 'AWSAccessKeyId' => $this->rgwS3AccessKey ] - ); - } - } - - return null; - } - - protected function directoriesAreVirtual() { - return true; - } - - /** - * Get headers to send to Swift when reading a file based - * on a FileBackend params array, e.g. that of getLocalCopy(). - * $params is currently only checked for a 'latest' flag. - * - * @param array $params - * @return array - */ - protected function headersFromParams( array $params ) { - $hdrs = []; - if ( !empty( $params['latest'] ) ) { - $hdrs['x-newest'] = 'true'; - } - - return $hdrs; - } - - /** - * @param FileBackendStoreOpHandle[] $fileOpHandles - * - * @return Status[] - */ - protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { - $statuses = []; - - $auth = $this->getAuthentication(); - if ( !$auth ) { - foreach ( $fileOpHandles as $index => $fileOpHandle ) { - $statuses[$index] = Status::newFatal( 'backend-fail-connect', $this->name ); - } - - return $statuses; - } - - // Split the HTTP requests into stages that can be done concurrently - $httpReqsByStage = []; // map of (stage => index => HTTP request) - foreach ( $fileOpHandles as $index => $fileOpHandle ) { - $reqs = $fileOpHandle->httpOp; - // Convert the 'url' parameter to an actual URL using $auth - foreach ( $reqs as $stage => &$req ) { - list( $container, $relPath ) = $req['url']; - $req['url'] = $this->storageUrl( $auth, $container, $relPath ); - $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : []; - $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers']; - $httpReqsByStage[$stage][$index] = $req; - } - $statuses[$index] = Status::newGood(); - } - - // Run all requests for the first stage, then the next, and so on - $reqCount = count( $httpReqsByStage ); - for ( $stage = 0; $stage < $reqCount; ++$stage ) { - $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] ); - foreach ( $httpReqs as $index => $httpReq ) { - // Run the callback for each request of this operation - $callback = $fileOpHandles[$index]->callback; - call_user_func_array( $callback, [ $httpReq, $statuses[$index] ] ); - // On failure, abort all remaining requests for this operation - // (e.g. abort the DELETE request if the COPY request fails for a move) - if ( !$statuses[$index]->isOK() ) { - $stages = count( $fileOpHandles[$index]->httpOp ); - for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) { - unset( $httpReqsByStage[$s][$index] ); - } - } - } - } - - return $statuses; - } - - /** - * Set read/write permissions for a Swift container. - * - * @see http://swift.openstack.org/misc.html#acls - * - * In general, we don't allow listings to end-users. It's not useful, isn't well-defined - * (lists are truncated to 10000 item with no way to page), and is just a performance risk. - * - * @param string $container Resolved Swift container - * @param array $readGrps List of the possible criteria for a request to have - * access to read a container. Each item is one of the following formats: - * - account:user : Grants access if the request is by the given user - * - ".r:" : Grants access if the request is from a referrer host that - * matches the expression and the request is not for a listing. - * Setting this to '*' effectively makes a container public. - * -".rlistings:" : Grants access if the request is from a referrer host that - * matches the expression and the request is for a listing. - * @param array $writeGrps A list of the possible criteria for a request to have - * access to write to a container. Each item is of the following format: - * - account:user : Grants access if the request is by the given user - * @return Status - */ - protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) { - $status = Status::newGood(); - $auth = $this->getAuthentication(); - - if ( !$auth ) { - $status->fatal( 'backend-fail-connect', $this->name ); - - return $status; - } - - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'POST', - 'url' => $this->storageUrl( $auth, $container ), - 'headers' => $this->authTokenHeaders( $auth ) + [ - 'x-container-read' => implode( ',', $readGrps ), - 'x-container-write' => implode( ',', $writeGrps ) - ] - ] ); - - if ( $rcode != 204 && $rcode !== 202 ) { - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'SwiftBackend', __METHOD__ . ': unexpected rcode value (' . $rcode . ')' ); - } - - return $status; - } - - /** - * Get a Swift container stat array, possibly from process cache. - * Use $reCache if the file count or byte count is needed. - * - * @param string $container Container name - * @param bool $bypassCache Bypass all caches and load from Swift - * @return array|bool|null False on 404, null on failure - */ - protected function getContainerStat( $container, $bypassCache = false ) { - $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); - - if ( $bypassCache ) { // purge cache - $this->containerStatCache->clear( $container ); - } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) { - $this->primeContainerCache( [ $container ] ); // check persistent cache - } - if ( !$this->containerStatCache->has( $container, 'stat' ) ) { - $auth = $this->getAuthentication(); - if ( !$auth ) { - return null; - } - - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'HEAD', - 'url' => $this->storageUrl( $auth, $container ), - 'headers' => $this->authTokenHeaders( $auth ) - ] ); - - if ( $rcode === 204 ) { - $stat = [ - 'count' => $rhdrs['x-container-object-count'], - 'bytes' => $rhdrs['x-container-bytes-used'] - ]; - if ( $bypassCache ) { - return $stat; - } else { - $this->containerStatCache->set( $container, 'stat', $stat ); // cache it - $this->setContainerCache( $container, $stat ); // update persistent cache - } - } elseif ( $rcode === 404 ) { - return false; - } else { - $this->onError( null, __METHOD__, - [ 'cont' => $container ], $rerr, $rcode, $rdesc ); - - return null; - } - } - - return $this->containerStatCache->get( $container, 'stat' ); - } - - /** - * Create a Swift container - * - * @param string $container Container name - * @param array $params - * @return Status - */ - protected function createContainer( $container, array $params ) { - $status = Status::newGood(); - - $auth = $this->getAuthentication(); - if ( !$auth ) { - $status->fatal( 'backend-fail-connect', $this->name ); - - return $status; - } - - // @see SwiftFileBackend::setContainerAccess() - if ( empty( $params['noAccess'] ) ) { - $readGrps = [ '.r:*', $this->swiftUser ]; // public - } else { - $readGrps = [ $this->swiftUser ]; // private - } - $writeGrps = [ $this->swiftUser ]; // sanity - - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'PUT', - 'url' => $this->storageUrl( $auth, $container ), - 'headers' => $this->authTokenHeaders( $auth ) + [ - 'x-container-read' => implode( ',', $readGrps ), - 'x-container-write' => implode( ',', $writeGrps ) - ] - ] ); - - if ( $rcode === 201 ) { // new - // good - } elseif ( $rcode === 202 ) { // already there - // this shouldn't really happen, but is OK - } else { - $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); - } - - return $status; - } - - /** - * Delete a Swift container - * - * @param string $container Container name - * @param array $params - * @return Status - */ - protected function deleteContainer( $container, array $params ) { - $status = Status::newGood(); - - $auth = $this->getAuthentication(); - if ( !$auth ) { - $status->fatal( 'backend-fail-connect', $this->name ); - - return $status; - } - - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'DELETE', - 'url' => $this->storageUrl( $auth, $container ), - 'headers' => $this->authTokenHeaders( $auth ) - ] ); - - if ( $rcode >= 200 && $rcode <= 299 ) { // deleted - $this->containerStatCache->clear( $container ); // purge - } elseif ( $rcode === 404 ) { // not there - // this shouldn't really happen, but is OK - } elseif ( $rcode === 409 ) { // not empty - $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race? - } else { - $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); - } - - return $status; - } - - /** - * Get a list of objects under a container. - * Either just the names or a list of stdClass objects with details can be returned. - * - * @param string $fullCont - * @param string $type ('info' for a list of object detail maps, 'names' for names only) - * @param int $limit - * @param string|null $after - * @param string|null $prefix - * @param string|null $delim - * @return Status With the list as value - */ - private function objectListing( - $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null - ) { - $status = Status::newGood(); - - $auth = $this->getAuthentication(); - if ( !$auth ) { - $status->fatal( 'backend-fail-connect', $this->name ); - - return $status; - } - - $query = [ 'limit' => $limit ]; - if ( $type === 'info' ) { - $query['format'] = 'json'; - } - if ( $after !== null ) { - $query['marker'] = $after; - } - if ( $prefix !== null ) { - $query['prefix'] = $prefix; - } - if ( $delim !== null ) { - $query['delimiter'] = $delim; - } - - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'GET', - 'url' => $this->storageUrl( $auth, $fullCont ), - 'query' => $query, - 'headers' => $this->authTokenHeaders( $auth ) - ] ); - - $params = [ 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim ]; - if ( $rcode === 200 ) { // good - if ( $type === 'info' ) { - $status->value = FormatJson::decode( trim( $rbody ) ); - } else { - $status->value = explode( "\n", trim( $rbody ) ); - } - } elseif ( $rcode === 204 ) { - $status->value = []; // empty container - } elseif ( $rcode === 404 ) { - $status->value = []; // no container - } else { - $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); - } - - return $status; - } - - protected function doPrimeContainerCache( array $containerInfo ) { - foreach ( $containerInfo as $container => $info ) { - $this->containerStatCache->set( $container, 'stat', $info ); - } - } - - protected function doGetFileStatMulti( array $params ) { - $stats = []; - - $auth = $this->getAuthentication(); - - $reqs = []; - foreach ( $params['srcs'] as $path ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); - if ( $srcRel === null ) { - $stats[$path] = false; - continue; // invalid storage path - } elseif ( !$auth ) { - $stats[$path] = null; - continue; - } - - // (a) Check the container - $cstat = $this->getContainerStat( $srcCont ); - if ( $cstat === false ) { - $stats[$path] = false; - continue; // ok, nothing to do - } elseif ( !is_array( $cstat ) ) { - $stats[$path] = null; - continue; - } - - $reqs[$path] = [ - 'method' => 'HEAD', - 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ), - 'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params ) - ]; - } - - $opts = [ 'maxConnsPerHost' => $params['concurrency'] ]; - $reqs = $this->http->runMulti( $reqs, $opts ); - - foreach ( $params['srcs'] as $path ) { - if ( array_key_exists( $path, $stats ) ) { - continue; // some sort of failure above - } - // (b) Check the file - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response']; - if ( $rcode === 200 || $rcode === 204 ) { - // Update the object if it is missing some headers - $rhdrs = $this->addMissingMetadata( $rhdrs, $path ); - // Load the stat array from the headers - $stat = $this->getStatFromHeaders( $rhdrs ); - if ( $this->isRGW ) { - $stat['latest'] = true; // strong consistency - } - } elseif ( $rcode === 404 ) { - $stat = false; - } else { - $stat = null; - $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc ); - } - $stats[$path] = $stat; - } - - return $stats; - } - - /** - * @param array $rhdrs - * @return array - */ - protected function getStatFromHeaders( array $rhdrs ) { - // Fetch all of the custom metadata headers - $metadata = $this->getMetadata( $rhdrs ); - // Fetch all of the custom raw HTTP headers - $headers = $this->sanitizeHdrs( [ 'headers' => $rhdrs ] ); - - return [ - // Convert various random Swift dates to TS_MW - 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), - // Empty objects actually return no content-length header in Ceph - 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, - 'sha1' => isset( $metadata['sha1base36'] ) ? $metadata['sha1base36'] : null, - // Note: manifiest ETags are not an MD5 of the file - 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, - 'xattr' => [ 'metadata' => $metadata, 'headers' => $headers ] - ]; - } - - /** - * @return array|null Credential map - */ - protected function getAuthentication() { - if ( $this->authErrorTimestamp !== null ) { - if ( ( time() - $this->authErrorTimestamp ) < 60 ) { - return null; // failed last attempt; don't bother - } else { // actually retry this time - $this->authErrorTimestamp = null; - } - } - // Session keys expire after a while, so we renew them periodically - $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL ); - // Authenticate with proxy and get a session key... - if ( !$this->authCreds || $reAuth ) { - $this->authSessionTimestamp = 0; - $cacheKey = $this->getCredsCacheKey( $this->swiftUser ); - $creds = $this->srvCache->get( $cacheKey ); // credentials - // Try to use the credential cache - if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) { - $this->authCreds = $creds; - // Skew the timestamp for worst case to avoid using stale credentials - $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 ); - } else { // cache miss - list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( [ - 'method' => 'GET', - 'url' => "{$this->swiftAuthUrl}/v1.0", - 'headers' => [ - 'x-auth-user' => $this->swiftUser, - 'x-auth-key' => $this->swiftKey - ] - ] ); - - if ( $rcode >= 200 && $rcode <= 299 ) { // OK - $this->authCreds = [ - 'auth_token' => $rhdrs['x-auth-token'], - 'storage_url' => $rhdrs['x-storage-url'] - ]; - $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) ); - $this->authSessionTimestamp = time(); - } elseif ( $rcode === 401 ) { - $this->onError( null, __METHOD__, [], "Authentication failed.", $rcode ); - $this->authErrorTimestamp = time(); - - return null; - } else { - $this->onError( null, __METHOD__, [], "HTTP return code: $rcode", $rcode ); - $this->authErrorTimestamp = time(); - - return null; - } - } - // Ceph RGW does not use in URLs (OpenStack Swift uses "/v1/") - if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) { - $this->isRGW = true; // take advantage of strong consistency in Ceph - } - } - - return $this->authCreds; - } - - /** - * @param array $creds From getAuthentication() - * @param string $container - * @param string $object - * @return array - */ - protected function storageUrl( array $creds, $container = null, $object = null ) { - $parts = [ $creds['storage_url'] ]; - if ( strlen( $container ) ) { - $parts[] = rawurlencode( $container ); - } - if ( strlen( $object ) ) { - $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) ); - } - - return implode( '/', $parts ); - } - - /** - * @param array $creds From getAuthentication() - * @return array - */ - protected function authTokenHeaders( array $creds ) { - return [ 'x-auth-token' => $creds['auth_token'] ]; - } - - /** - * Get the cache key for a container - * - * @param string $username - * @return string - */ - private function getCredsCacheKey( $username ) { - return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl ); - } - - /** - * Log an unexpected exception for this backend. - * This also sets the Status object to have a fatal error. - * - * @param Status|null $status - * @param string $func - * @param array $params - * @param string $err Error string - * @param int $code HTTP status - * @param string $desc HTTP status description - */ - public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) { - if ( $status instanceof Status ) { - $status->fatal( 'backend-fail-internal', $this->name ); - } - if ( $code == 401 ) { // possibly a stale token - $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) ); - } - wfDebugLog( 'SwiftBackend', - "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . - ( $err ? ": $err" : "" ) - ); - } -} - -/** - * @see FileBackendStoreOpHandle - */ -class SwiftFileOpHandle extends FileBackendStoreOpHandle { - /** @var array List of Requests for MultiHttpClient */ - public $httpOp; - /** @var Closure */ - public $callback; - - /** - * @param SwiftFileBackend $backend - * @param Closure $callback Function that takes (HTTP request array, status) - * @param array $httpOp MultiHttpClient op - */ - public function __construct( SwiftFileBackend $backend, Closure $callback, array $httpOp ) { - $this->backend = $backend; - $this->callback = $callback; - $this->httpOp = $httpOp; - } -} - -/** - * SwiftFileBackend helper class to page through listings. - * Swift also has a listing limit of 10,000 objects for sanity. - * Do not use this class from places outside SwiftFileBackend. - * - * @ingroup FileBackend - */ -abstract class SwiftFileBackendList implements Iterator { - /** @var array List of path or (path,stat array) entries */ - protected $bufferIter = []; - - /** @var string List items *after* this path */ - protected $bufferAfter = null; - - /** @var int */ - protected $pos = 0; - - /** @var array */ - protected $params = []; - - /** @var SwiftFileBackend */ - protected $backend; - - /** @var string Container name */ - protected $container; - - /** @var string Storage directory */ - protected $dir; - - /** @var int */ - protected $suffixStart; - - const PAGE_SIZE = 9000; // file listing buffer size - - /** - * @param SwiftFileBackend $backend - * @param string $fullCont Resolved container name - * @param string $dir Resolved directory relative to container - * @param array $params - */ - public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { - $this->backend = $backend; - $this->container = $fullCont; - $this->dir = $dir; - if ( substr( $this->dir, -1 ) === '/' ) { - $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash - } - if ( $this->dir == '' ) { // whole container - $this->suffixStart = 0; - } else { // dir within container - $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/" - } - $this->params = $params; - } - - /** - * @see Iterator::key() - * @return int - */ - public function key() { - return $this->pos; - } - - /** - * @see Iterator::next() - */ - public function next() { - // Advance to the next file in the page - next( $this->bufferIter ); - ++$this->pos; - // Check if there are no files left in this page and - // advance to the next page if this page was not empty. - if ( !$this->valid() && count( $this->bufferIter ) ) { - $this->bufferIter = $this->pageFromList( - $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params - ); // updates $this->bufferAfter - } - } - - /** - * @see Iterator::rewind() - */ - public function rewind() { - $this->pos = 0; - $this->bufferAfter = null; - $this->bufferIter = $this->pageFromList( - $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params - ); // updates $this->bufferAfter - } - - /** - * @see Iterator::valid() - * @return bool - */ - public function valid() { - if ( $this->bufferIter === null ) { - return false; // some failure? - } else { - return ( current( $this->bufferIter ) !== false ); // no paths can have this value - } - } - - /** - * Get the given list portion (page) - * - * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param string $after - * @param int $limit - * @param array $params - * @return Traversable|array - */ - abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); -} - -/** - * Iterator for listing directories - */ -class SwiftFileBackendDirList extends SwiftFileBackendList { - /** - * @see Iterator::current() - * @return string|bool String (relative path) or false - */ - public function current() { - return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); - } - - protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { - return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); - } -} - -/** - * Iterator for listing regular files - */ -class SwiftFileBackendFileList extends SwiftFileBackendList { - /** - * @see Iterator::current() - * @return string|bool String (relative path) or false - */ - public function current() { - list( $path, $stat ) = current( $this->bufferIter ); - $relPath = substr( $path, $this->suffixStart ); - if ( is_array( $stat ) ) { - $storageDir = rtrim( $this->params['dir'], '/' ); - $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat ); - } - - return $relPath; - } - - protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { - return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); - } -} diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php deleted file mode 100644 index f57284080d..0000000000 --- a/includes/filebackend/TempFSFile.php +++ /dev/null @@ -1,157 +0,0 @@ - 1) for paths to delete on shutdown */ - protected static $pathsCollect = null; - - public function __construct( $path ) { - parent::__construct( $path ); - - if ( self::$pathsCollect === null ) { - self::$pathsCollect = []; - register_shutdown_function( [ __CLASS__, 'purgeAllOnShutdown' ] ); - } - } - - /** - * Make a new temporary file on the file system. - * Temporary files may be purged when the file object falls out of scope. - * - * @param string $prefix - * @param string $extension - * @return TempFSFile|null - */ - public static function factory( $prefix, $extension = '' ) { - $ext = ( $extension != '' ) ? ".{$extension}" : ''; - - $attempts = 5; - while ( $attempts-- ) { - $path = wfTempDir() . '/' . $prefix . wfRandomString( 12 ) . $ext; - MediaWiki\suppressWarnings(); - $newFileHandle = fopen( $path, 'x' ); - MediaWiki\restoreWarnings(); - if ( $newFileHandle ) { - fclose( $newFileHandle ); - $tmpFile = new self( $path ); - $tmpFile->autocollect(); - // Safely instantiated, end loop. - return $tmpFile; - } - } - - // Give up - return null; - } - - /** - * Purge this file off the file system - * - * @return bool Success - */ - public function purge() { - $this->canDelete = false; // done - MediaWiki\suppressWarnings(); - $ok = unlink( $this->path ); - MediaWiki\restoreWarnings(); - - unset( self::$pathsCollect[$this->path] ); - - return $ok; - } - - /** - * Clean up the temporary file only after an object goes out of scope - * - * @param object $object - * @return TempFSFile This object - */ - public function bind( $object ) { - if ( is_object( $object ) ) { - if ( !isset( $object->tempFSFileReferences ) ) { - // Init first since $object might use __get() and return only a copy variable - $object->tempFSFileReferences = []; - } - $object->tempFSFileReferences[] = $this; - } - - return $this; - } - - /** - * Set flag to not clean up after the temporary file - * - * @return TempFSFile This object - */ - public function preserve() { - $this->canDelete = false; - - unset( self::$pathsCollect[$this->path] ); - - return $this; - } - - /** - * Set flag clean up after the temporary file - * - * @return TempFSFile This object - */ - public function autocollect() { - $this->canDelete = true; - - self::$pathsCollect[$this->path] = 1; - - return $this; - } - - /** - * Try to make sure that all files are purged on error - * - * This method should only be called internally - */ - public static function purgeAllOnShutdown() { - foreach ( self::$pathsCollect as $path ) { - MediaWiki\suppressWarnings(); - unlink( $path ); - MediaWiki\restoreWarnings(); - } - } - - /** - * Cleans up after the temporary file by deleting it - */ - function __destruct() { - if ( $this->canDelete ) { - $this->purge(); - } - } -} diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php index 7efb3a15c1..2e06c40f72 100644 --- a/includes/filebackend/filejournal/DBFileJournal.php +++ b/includes/filebackend/filejournal/DBFileJournal.php @@ -48,10 +48,10 @@ class DBFileJournal extends FileJournal { * @see FileJournal::logChangeBatch() * @param array $entries * @param string $batchId - * @return Status + * @return StatusValue */ protected function doLogChangeBatch( array $entries, $batchId ) { - $status = Status::newGood(); + $status = StatusValue::newGood(); try { $dbw = $this->getMasterDB(); @@ -151,11 +151,11 @@ class DBFileJournal extends FileJournal { /** * @see FileJournal::purgeOldLogs() - * @return Status + * @return StatusValue * @throws DBError */ protected function doPurgeOldLogs() { - $status = Status::newGood(); + $status = StatusValue::newGood(); if ( $this->ttlDays <= 0 ) { return $status; // nothing to do } diff --git a/includes/filebackend/filejournal/FileJournal.php b/includes/filebackend/filejournal/FileJournal.php deleted file mode 100644 index b84e195989..0000000000 --- a/includes/filebackend/filejournal/FileJournal.php +++ /dev/null @@ -1,251 +0,0 @@ -ttlDays = isset( $config['ttlDays'] ) ? $config['ttlDays'] : false; - } - - /** - * Create an appropriate FileJournal object from config - * - * @param array $config - * @param string $backend A registered file backend name - * @throws Exception - * @return FileJournal - */ - final public static function factory( array $config, $backend ) { - $class = $config['class']; - $jrn = new $class( $config ); - if ( !$jrn instanceof self ) { - throw new Exception( "Class given is not an instance of FileJournal." ); - } - $jrn->backend = $backend; - - return $jrn; - } - - /** - * Get a statistically unique ID string - * - * @return string <9 char TS_MW timestamp in base 36><22 random base 36 chars> - */ - final public function getTimestampedUUID() { - $s = ''; - for ( $i = 0; $i < 5; $i++ ) { - $s .= mt_rand( 0, 2147483647 ); - } - $s = Wikimedia\base_convert( sha1( $s ), 16, 36, 31 ); - - return substr( Wikimedia\base_convert( wfTimestamp( TS_MW ), 10, 36, 9 ) . $s, 0, 31 ); - } - - /** - * Log changes made by a batch file operation. - * - * @param array $entries List of file operations (each an array of parameters) which contain: - * op : Basic operation name (create, update, delete) - * path : The storage path of the file - * newSha1 : The final base 36 SHA-1 of the file - * Note that 'false' should be used as the SHA-1 for non-existing files. - * @param string $batchId UUID string that identifies the operation batch - * @return Status - */ - final public function logChangeBatch( array $entries, $batchId ) { - if ( !count( $entries ) ) { - return Status::newGood(); - } - - return $this->doLogChangeBatch( $entries, $batchId ); - } - - /** - * @see FileJournal::logChangeBatch() - * - * @param array $entries List of file operations (each an array of parameters) - * @param string $batchId UUID string that identifies the operation batch - * @return Status - */ - abstract protected function doLogChangeBatch( array $entries, $batchId ); - - /** - * Get the position ID of the latest journal entry - * - * @return int|bool - */ - final public function getCurrentPosition() { - return $this->doGetCurrentPosition(); - } - - /** - * @see FileJournal::getCurrentPosition() - * @return int|bool - */ - abstract protected function doGetCurrentPosition(); - - /** - * Get the position ID of the latest journal entry at some point in time - * - * @param int|string $time Timestamp - * @return int|bool - */ - final public function getPositionAtTime( $time ) { - return $this->doGetPositionAtTime( $time ); - } - - /** - * @see FileJournal::getPositionAtTime() - * @param int|string $time Timestamp - * @return int|bool - */ - abstract protected function doGetPositionAtTime( $time ); - - /** - * Get an array of file change log entries. - * A starting change ID and/or limit can be specified. - * - * @param int $start Starting change ID or null - * @param int $limit Maximum number of items to return - * @param string &$next Updated to the ID of the next entry. - * @return array List of associative arrays, each having: - * id : unique, monotonic, ID for this change - * batch_uuid : UUID for an operation batch - * backend : the backend name - * op : primitive operation (create,update,delete,null) - * path : affected storage path - * new_sha1 : base 36 sha1 of the new file had the operation succeeded - * timestamp : TS_MW timestamp of the batch change - * Also, $next is updated to the ID of the next entry. - */ - final public function getChangeEntries( $start = null, $limit = 0, &$next = null ) { - $entries = $this->doGetChangeEntries( $start, $limit ? $limit + 1 : 0 ); - if ( $limit && count( $entries ) > $limit ) { - $last = array_pop( $entries ); // remove the extra entry - $next = $last['id']; // update for next call - } else { - $next = null; // end of list - } - - return $entries; - } - - /** - * @see FileJournal::getChangeEntries() - * @param int $start - * @param int $limit - * @return array - */ - abstract protected function doGetChangeEntries( $start, $limit ); - - /** - * Purge any old log entries - * - * @return Status - */ - final public function purgeOldLogs() { - return $this->doPurgeOldLogs(); - } - - /** - * @see FileJournal::purgeOldLogs() - * @return Status - */ - abstract protected function doPurgeOldLogs(); -} - -/** - * Simple version of FileJournal that does nothing - * @since 1.20 - */ -class NullFileJournal extends FileJournal { - /** - * @see FileJournal::doLogChangeBatch() - * @param array $entries - * @param string $batchId - * @return Status - */ - protected function doLogChangeBatch( array $entries, $batchId ) { - return Status::newGood(); - } - - /** - * @see FileJournal::doGetCurrentPosition() - * @return int|bool - */ - protected function doGetCurrentPosition() { - return false; - } - - /** - * @see FileJournal::doGetPositionAtTime() - * @param int|string $time Timestamp - * @return int|bool - */ - protected function doGetPositionAtTime( $time ) { - return false; - } - - /** - * @see FileJournal::doGetChangeEntries() - * @param int $start - * @param int $limit - * @return array - */ - protected function doGetChangeEntries( $start, $limit ) { - return []; - } - - /** - * @see FileJournal::doPurgeOldLogs() - * @return Status - */ - protected function doPurgeOldLogs() { - return Status::newGood(); - } -} diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php deleted file mode 100644 index cccf71a929..0000000000 --- a/includes/filebackend/lockmanager/DBLockManager.php +++ /dev/null @@ -1,246 +0,0 @@ - server config array) - /** @var BagOStuff */ - protected $statusCache; - - protected $lockExpiry; // integer number of seconds - protected $safeDelay; // integer number of seconds - - protected $session = 0; // random integer - /** @var IDatabase[] Map Database connections (DB name => Database) */ - protected $conns = []; - - /** - * Construct a new instance from configuration. - * - * @param array $config Parameters include: - * - dbServers : Associative array of DB names to server configuration. - * Configuration is an associative array that includes: - * - host : DB server name - * - dbname : DB name - * - type : DB type (mysql,postgres,...) - * - user : DB user - * - password : DB user password - * - tablePrefix : DB table prefix - * - flags : DB flags (see DatabaseBase) - * - dbsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of DB names (peers) as values. - * Any DB named 'localDBMaster' will automatically use the DB master - * settings for this wiki (without the need for a dbServers entry). - * Only use 'localDBMaster' if the domain is a valid wiki ID. - * - lockExpiry : Lock timeout (seconds) for dropped connections. [optional] - * This tells the DB server how long to wait before assuming - * connection failure and releasing all the locks for a session. - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - $this->dbServers = isset( $config['dbServers'] ) - ? $config['dbServers'] - : []; // likely just using 'localDBMaster' - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['dbsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - if ( isset( $config['lockExpiry'] ) ) { - $this->lockExpiry = $config['lockExpiry']; - } else { - $met = ini_get( 'max_execution_time' ); - $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0 - } - $this->safeDelay = ( $this->lockExpiry <= 0 ) - ? 60 // pick a safe-ish number to match DB timeout default - : $this->lockExpiry; // cover worst case - - foreach ( $this->srvsByBucket as $bucket ) { - if ( count( $bucket ) > 1 ) { // multiple peers - // Tracks peers that couldn't be queried recently to avoid lengthy - // connection timeouts. This is useless if each bucket has one peer. - $this->statusCache = ObjectCache::getLocalServerInstance(); - break; - } - } - - $this->session = wfRandomString( 31 ); - } - - // @todo change this code to work in one batch - protected function getLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); - } - - return $status; - } - - abstract protected function doGetLocksOnServer( $lockSrv, array $paths, $type ); - - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - return Status::newGood(); - } - - /** - * @see QuorumLockManager::isServerUp() - * @param string $lockSrv - * @return bool - */ - protected function isServerUp( $lockSrv ) { - if ( !$this->cacheCheckFailures( $lockSrv ) ) { - return false; // recent failure to connect - } - try { - $this->getConnection( $lockSrv ); - } catch ( DBError $e ) { - $this->cacheRecordFailure( $lockSrv ); - - return false; // failed to connect - } - - return true; - } - - /** - * Get (or reuse) a connection to a lock DB - * - * @param string $lockDb - * @return IDatabase - * @throws DBError - * @throws UnexpectedValueException - */ - protected function getConnection( $lockDb ) { - if ( !isset( $this->conns[$lockDb] ) ) { - if ( $lockDb === 'localDBMaster' ) { - $lb = $this->getLocalLB(); - $db = $lb->getConnection( DB_MASTER, [], $this->domain ); - # Do not mess with settings if the LoadBalancer is the main singleton - # to avoid clobbering the settings of handles from wfGetDB( DB_MASTER ). - $init = ( wfGetLB() !== $lb ); - } elseif ( isset( $this->dbServers[$lockDb] ) ) { - $config = $this->dbServers[$lockDb]; - $db = DatabaseBase::factory( $config['type'], $config ); - $init = true; - } else { - throw new UnexpectedValueException( "No server called '$lockDb'." ); - } - - if ( $init ) { - $db->clearFlag( DBO_TRX ); - # If the connection drops, try to avoid letting the DB rollback - # and release the locks before the file operations are finished. - # This won't handle the case of DB server restarts however. - $options = []; - if ( $this->lockExpiry > 0 ) { - $options['connTimeout'] = $this->lockExpiry; - } - $db->setSessionOptions( $options ); - $this->initConnection( $lockDb, $db ); - } - - $this->conns[$lockDb] = $db; - } - - return $this->conns[$lockDb]; - } - - /** - * @return LoadBalancer - */ - protected function getLocalLB() { - return wfGetLBFactory()->getMainLB( $this->domain ); - } - - /** - * Do additional initialization for new lock DB connection - * - * @param string $lockDb - * @param IDatabase $db - * @throws DBError - */ - protected function initConnection( $lockDb, IDatabase $db ) { - } - - /** - * Checks if the DB has not recently had connection/query errors. - * This just avoids wasting time on doomed connection attempts. - * - * @param string $lockDb - * @return bool - */ - protected function cacheCheckFailures( $lockDb ) { - return ( $this->statusCache && $this->safeDelay > 0 ) - ? !$this->statusCache->get( $this->getMissKey( $lockDb ) ) - : true; - } - - /** - * Log a lock request failure to the cache - * - * @param string $lockDb - * @return bool Success - */ - protected function cacheRecordFailure( $lockDb ) { - return ( $this->statusCache && $this->safeDelay > 0 ) - ? $this->statusCache->set( $this->getMissKey( $lockDb ), 1, $this->safeDelay ) - : true; - } - - /** - * Get a cache key for recent query misses for a DB - * - * @param string $lockDb - * @return string - */ - protected function getMissKey( $lockDb ) { - $lockDb = ( $lockDb === 'localDBMaster' ) ? wfWikiID() : $lockDb; // non-relative - return 'dblockmanager:downservers:' . str_replace( ' ', '_', $lockDb ); - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - $this->releaseAllLocks(); - foreach ( $this->conns as $db ) { - $db->close(); - } - } -} diff --git a/includes/filebackend/lockmanager/FSLockManager.php b/includes/filebackend/lockmanager/FSLockManager.php deleted file mode 100644 index 2b660ec77e..0000000000 --- a/includes/filebackend/lockmanager/FSLockManager.php +++ /dev/null @@ -1,248 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - protected $lockDir; // global dir for all servers - - /** @var array Map of (locked key => lock file handle) */ - protected $handles = []; - - /** - * Construct a new instance from configuration. - * - * @param array $config Includes: - * - lockDirectory : Directory containing the lock files - */ - function __construct( array $config ) { - parent::__construct( $config ); - - $this->lockDir = $config['lockDirectory']; - } - - /** - * @see LockManager::doLock() - * @param array $paths - * @param int $type - * @return Status - */ - protected function doLock( array $paths, $type ) { - $status = Status::newGood(); - - $lockedPaths = []; // files locked in this attempt - foreach ( $paths as $path ) { - $status->merge( $this->doSingleLock( $path, $type ) ); - if ( $status->isOK() ) { - $lockedPaths[] = $path; - } else { - // Abort and unlock everything - $status->merge( $this->doUnlock( $lockedPaths, $type ) ); - - return $status; - } - } - - return $status; - } - - /** - * @see LockManager::doUnlock() - * @param array $paths - * @param int $type - * @return Status - */ - protected function doUnlock( array $paths, $type ) { - $status = Status::newGood(); - - foreach ( $paths as $path ) { - $status->merge( $this->doSingleUnlock( $path, $type ) ); - } - - return $status; - } - - /** - * Lock a single resource key - * - * @param string $path - * @param int $type - * @return Status - */ - protected function doSingleLock( $path, $type ) { - $status = Status::newGood(); - - if ( isset( $this->locksHeld[$path][$type] ) ) { - ++$this->locksHeld[$path][$type]; - } elseif ( isset( $this->locksHeld[$path][self::LOCK_EX] ) ) { - $this->locksHeld[$path][$type] = 1; - } else { - if ( isset( $this->handles[$path] ) ) { - $handle = $this->handles[$path]; - } else { - MediaWiki\suppressWarnings(); - $handle = fopen( $this->getLockPath( $path ), 'a+' ); - MediaWiki\restoreWarnings(); - if ( !$handle ) { // lock dir missing? - wfMkdirParents( $this->lockDir ); - $handle = fopen( $this->getLockPath( $path ), 'a+' ); // try again - } - } - if ( $handle ) { - // Either a shared or exclusive lock - $lock = ( $type == self::LOCK_SH ) ? LOCK_SH : LOCK_EX; - if ( flock( $handle, $lock | LOCK_NB ) ) { - // Record this lock as active - $this->locksHeld[$path][$type] = 1; - $this->handles[$path] = $handle; - } else { - fclose( $handle ); - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } else { - $status->fatal( 'lockmanager-fail-openlock', $path ); - } - } - - return $status; - } - - /** - * Unlock a single resource key - * - * @param string $path - * @param int $type - * @return Status - */ - protected function doSingleUnlock( $path, $type ) { - $status = Status::newGood(); - - if ( !isset( $this->locksHeld[$path] ) ) { - $status->warning( 'lockmanager-notlocked', $path ); - } elseif ( !isset( $this->locksHeld[$path][$type] ) ) { - $status->warning( 'lockmanager-notlocked', $path ); - } else { - $handlesToClose = []; - --$this->locksHeld[$path][$type]; - if ( $this->locksHeld[$path][$type] <= 0 ) { - unset( $this->locksHeld[$path][$type] ); - } - if ( !count( $this->locksHeld[$path] ) ) { - unset( $this->locksHeld[$path] ); // no locks on this path - if ( isset( $this->handles[$path] ) ) { - $handlesToClose[] = $this->handles[$path]; - unset( $this->handles[$path] ); - } - } - // Unlock handles to release locks and delete - // any lock files that end up with no locks on them... - if ( wfIsWindows() ) { - // Windows: for any process, including this one, - // calling unlink() on a locked file will fail - $status->merge( $this->closeLockHandles( $path, $handlesToClose ) ); - $status->merge( $this->pruneKeyLockFiles( $path ) ); - } else { - // Unix: unlink() can be used on files currently open by this - // process and we must do so in order to avoid race conditions - $status->merge( $this->pruneKeyLockFiles( $path ) ); - $status->merge( $this->closeLockHandles( $path, $handlesToClose ) ); - } - } - - return $status; - } - - /** - * @param string $path - * @param array $handlesToClose - * @return Status - */ - private function closeLockHandles( $path, array $handlesToClose ) { - $status = Status::newGood(); - foreach ( $handlesToClose as $handle ) { - if ( !flock( $handle, LOCK_UN ) ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - if ( !fclose( $handle ) ) { - $status->warning( 'lockmanager-fail-closelock', $path ); - } - } - - return $status; - } - - /** - * @param string $path - * @return Status - */ - private function pruneKeyLockFiles( $path ) { - $status = Status::newGood(); - if ( !isset( $this->locksHeld[$path] ) ) { - # No locks are held for the lock file anymore - if ( !unlink( $this->getLockPath( $path ) ) ) { - $status->warning( 'lockmanager-fail-deletelock', $path ); - } - unset( $this->handles[$path] ); - } - - return $status; - } - - /** - * Get the path to the lock file for a key - * @param string $path - * @return string - */ - protected function getLockPath( $path ) { - return "{$this->lockDir}/{$this->sha1Base36Absolute( $path )}.lock"; - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - while ( count( $this->locksHeld ) ) { - foreach ( $this->locksHeld as $path => $locks ) { - $this->doSingleUnlock( $path, self::LOCK_EX ); - $this->doSingleUnlock( $path, self::LOCK_SH ); - } - } - } -} diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php deleted file mode 100644 index 567a29892e..0000000000 --- a/includes/filebackend/lockmanager/LockManager.php +++ /dev/null @@ -1,258 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_EX, // subclasses may use self::LOCK_SH - self::LOCK_EX => self::LOCK_EX - ]; - - /** @var array Map of (resource path => lock type => count) */ - protected $locksHeld = []; - - protected $domain; // string; domain (usually wiki ID) - protected $lockTTL; // integer; maximum time locks can be held - - /** Lock types; stronger locks have higher values */ - const LOCK_SH = 1; // shared lock (for reads) - const LOCK_UW = 2; // shared lock (for reads used to write elsewhere) - const LOCK_EX = 3; // exclusive lock (for writes) - - /** - * Construct a new instance from configuration - * - * @param array $config Parameters include: - * - domain : Domain (usually wiki ID) that all resources are relative to [optional] - * - lockTTL : Age (in seconds) at which resource locks should expire. - * This only applies if locks are not tied to a connection/process. - */ - public function __construct( array $config ) { - $this->domain = isset( $config['domain'] ) ? $config['domain'] : wfWikiID(); - if ( isset( $config['lockTTL'] ) ) { - $this->lockTTL = max( 5, $config['lockTTL'] ); - } elseif ( PHP_SAPI === 'cli' ) { - $this->lockTTL = 3600; - } else { - $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode - $this->lockTTL = max( 5 * 60, 2 * (int)$met ); - } - } - - /** - * Lock the resources at the given abstract paths - * - * @param array $paths List of resource names - * @param int $type LockManager::LOCK_* constant - * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21) - * @return Status - */ - final public function lock( array $paths, $type = self::LOCK_EX, $timeout = 0 ) { - return $this->lockByType( [ $type => $paths ], $timeout ); - } - - /** - * Lock the resources at the given abstract paths - * - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21) - * @return Status - * @since 1.22 - */ - final public function lockByType( array $pathsByType, $timeout = 0 ) { - $pathsByType = $this->normalizePathsByType( $pathsByType ); - $msleep = [ 0, 50, 100, 300, 500 ]; // retry backoff times - $start = microtime( true ); - do { - $status = $this->doLockByType( $pathsByType ); - $elapsed = microtime( true ) - $start; - if ( $status->isOK() || $elapsed >= $timeout || $elapsed < 0 ) { - break; // success, timeout, or clock set back - } - usleep( 1e3 * ( next( $msleep ) ?: 1000 ) ); // use 1 sec after enough times - $elapsed = microtime( true ) - $start; - } while ( $elapsed < $timeout && $elapsed >= 0 ); - - return $status; - } - - /** - * Unlock the resources at the given abstract paths - * - * @param array $paths List of paths - * @param int $type LockManager::LOCK_* constant - * @return Status - */ - final public function unlock( array $paths, $type = self::LOCK_EX ) { - return $this->unlockByType( [ $type => $paths ] ); - } - - /** - * Unlock the resources at the given abstract paths - * - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - * @since 1.22 - */ - final public function unlockByType( array $pathsByType ) { - $pathsByType = $this->normalizePathsByType( $pathsByType ); - $status = $this->doUnlockByType( $pathsByType ); - - return $status; - } - - /** - * Get the base 36 SHA-1 of a string, padded to 31 digits. - * Before hashing, the path will be prefixed with the domain ID. - * This should be used interally for lock key or file names. - * - * @param string $path - * @return string - */ - final protected function sha1Base36Absolute( $path ) { - return Wikimedia\base_convert( sha1( "{$this->domain}:{$path}" ), 16, 36, 31 ); - } - - /** - * Get the base 16 SHA-1 of a string, padded to 31 digits. - * Before hashing, the path will be prefixed with the domain ID. - * This should be used interally for lock key or file names. - * - * @param string $path - * @return string - */ - final protected function sha1Base16Absolute( $path ) { - return sha1( "{$this->domain}:{$path}" ); - } - - /** - * Normalize the $paths array by converting LOCK_UW locks into the - * appropriate type and removing any duplicated paths for each lock type. - * - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return array - * @since 1.22 - */ - final protected function normalizePathsByType( array $pathsByType ) { - $res = []; - foreach ( $pathsByType as $type => $paths ) { - $res[$this->lockTypeMap[$type]] = array_unique( $paths ); - } - - return $res; - } - - /** - * @see LockManager::lockByType() - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - * @since 1.22 - */ - protected function doLockByType( array $pathsByType ) { - $status = Status::newGood(); - $lockedByType = []; // map of (type => paths) - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doLock( $paths, $type ) ); - if ( $status->isOK() ) { - $lockedByType[$type] = $paths; - } else { - // Release the subset of locks that were acquired - foreach ( $lockedByType as $lType => $lPaths ) { - $status->merge( $this->doUnlock( $lPaths, $lType ) ); - } - break; - } - } - - return $status; - } - - /** - * Lock resources with the given keys and lock type - * - * @param array $paths List of paths - * @param int $type LockManager::LOCK_* constant - * @return Status - */ - abstract protected function doLock( array $paths, $type ); - - /** - * @see LockManager::unlockByType() - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - * @since 1.22 - */ - protected function doUnlockByType( array $pathsByType ) { - $status = Status::newGood(); - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doUnlock( $paths, $type ) ); - } - - return $status; - } - - /** - * Unlock resources with the given keys and lock type - * - * @param array $paths List of paths - * @param int $type LockManager::LOCK_* constant - * @return Status - */ - abstract protected function doUnlock( array $paths, $type ); -} - -/** - * Simple version of LockManager that does nothing - * @since 1.19 - */ -class NullLockManager extends LockManager { - protected function doLock( array $paths, $type ) { - return Status::newGood(); - } - - protected function doUnlock( array $paths, $type ) { - return Status::newGood(); - } -} diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php index 602b876b8a..1e66e6e011 100644 --- a/includes/filebackend/lockmanager/LockManagerGroup.php +++ b/includes/filebackend/lockmanager/LockManagerGroup.php @@ -20,6 +20,8 @@ * @file * @ingroup LockManager */ +use MediaWiki\MediaWikiServices; +use MediaWiki\Logger\LoggerFactory; /** * Class to handle file lock manager registration @@ -29,7 +31,7 @@ * @since 1.19 */ class LockManagerGroup { - /** @var array (domain => LockManager) */ + /** @var LockManagerGroup[] (domain => LockManagerGroup) */ protected static $instances = []; protected $domain; // string; domain (usually wiki ID) @@ -115,6 +117,16 @@ class LockManagerGroup { if ( !isset( $this->managers[$name]['instance'] ) ) { $class = $this->managers[$name]['class']; $config = $this->managers[$name]['config']; + if ( $class === 'DBLockManager' ) { + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $lb = $lbFactory->newMainLB( $config['domain'] ); + $dbw = $lb->getLazyConnectionRef( DB_MASTER, [], $config['domain'] ); + + $config['dbServers']['localDBMaster'] = $dbw; + $config['srvCache'] = ObjectCache::getLocalServerInstance( 'hash' ); + } + $config['logger'] = LoggerFactory::getInstance( 'LockManager' ); + $this->managers[$name]['instance'] = new $class( $config ); } diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php deleted file mode 100644 index cb5266acd8..0000000000 --- a/includes/filebackend/lockmanager/MemcLockManager.php +++ /dev/null @@ -1,384 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - /** @var array Map server names to MemcachedBagOStuff objects */ - protected $bagOStuffs = []; - - /** @var array (server name => bool) */ - protected $serversUp = []; - - /** @var string Random UUID */ - protected $session = ''; - - /** - * Construct a new instance from configuration. - * - * @param array $config Parameters include: - * - lockServers : Associative array of server names to ":" strings. - * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of server names (peers) as values. - * - memcConfig : Configuration array for ObjectCache::newFromParams. [optional] - * If set, this must use one of the memcached classes. - * @throws Exception - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - $memcConfig = isset( $config['memcConfig'] ) - ? $config['memcConfig'] - : [ 'class' => 'MemcachedPhpBagOStuff' ]; - - foreach ( $config['lockServers'] as $name => $address ) { - $params = [ 'servers' => [ $address ] ] + $memcConfig; - $cache = ObjectCache::newFromParams( $params ); - if ( $cache instanceof MemcachedBagOStuff ) { - $this->bagOStuffs[$name] = $cache; - } else { - throw new Exception( - 'Only MemcachedBagOStuff classes are supported by MemcLockManager.' ); - } - } - - $this->session = wfRandomString( 32 ); - } - - // @todo Change this code to work in one batch - protected function getLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - - $lockedPaths = []; - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); - if ( $status->isOK() ) { - $lockedPaths[$type] = isset( $lockedPaths[$type] ) - ? array_merge( $lockedPaths[$type], $paths ) - : $paths; - } else { - foreach ( $lockedPaths as $lType => $lPaths ) { - $status->merge( $this->doFreeLocksOnServer( $lockSrv, $lPaths, $lType ) ); - } - break; - } - } - - return $status; - } - - // @todo Change this code to work in one batch - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); - } - - return $status; - } - - /** - * @see QuorumLockManager::getLocksOnServer() - * @param string $lockSrv - * @param array $paths - * @param string $type - * @return Status - */ - protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - - $memc = $this->getCache( $lockSrv ); - $keys = array_map( [ $this, 'recordKeyForPath' ], $paths ); // lock records - - // Lock all of the active lock record keys... - if ( !$this->acquireMutexes( $memc, $keys ) ) { - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - - return $status; - } - - // Fetch all the existing lock records... - $lockRecords = $memc->getMulti( $keys ); - - $now = time(); - // Check if the requested locks conflict with existing ones... - foreach ( $paths as $path ) { - $locksKey = $this->recordKeyForPath( $path ); - $locksHeld = isset( $lockRecords[$locksKey] ) - ? self::sanitizeLockArray( $lockRecords[$locksKey] ) - : self::newLockArray(); // init - foreach ( $locksHeld[self::LOCK_EX] as $session => $expiry ) { - if ( $expiry < $now ) { // stale? - unset( $locksHeld[self::LOCK_EX][$session] ); - } elseif ( $session !== $this->session ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - if ( $type === self::LOCK_EX ) { - foreach ( $locksHeld[self::LOCK_SH] as $session => $expiry ) { - if ( $expiry < $now ) { // stale? - unset( $locksHeld[self::LOCK_SH][$session] ); - } elseif ( $session !== $this->session ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - } - if ( $status->isOK() ) { - // Register the session in the lock record array - $locksHeld[$type][$this->session] = $now + $this->lockTTL; - // We will update this record if none of the other locks conflict - $lockRecords[$locksKey] = $locksHeld; - } - } - - // If there were no lock conflicts, update all the lock records... - if ( $status->isOK() ) { - foreach ( $paths as $path ) { - $locksKey = $this->recordKeyForPath( $path ); - $locksHeld = $lockRecords[$locksKey]; - $ok = $memc->set( $locksKey, $locksHeld, 7 * 86400 ); - if ( !$ok ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } else { - wfDebug( __METHOD__ . ": acquired lock on key $locksKey.\n" ); - } - } - } - - // Unlock all of the active lock record keys... - $this->releaseMutexes( $memc, $keys ); - - return $status; - } - - /** - * @see QuorumLockManager::freeLocksOnServer() - * @param string $lockSrv - * @param array $paths - * @param string $type - * @return Status - */ - protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - - $memc = $this->getCache( $lockSrv ); - $keys = array_map( [ $this, 'recordKeyForPath' ], $paths ); // lock records - - // Lock all of the active lock record keys... - if ( !$this->acquireMutexes( $memc, $keys ) ) { - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - - return $status; - } - - // Fetch all the existing lock records... - $lockRecords = $memc->getMulti( $keys ); - - // Remove the requested locks from all records... - foreach ( $paths as $path ) { - $locksKey = $this->recordKeyForPath( $path ); // lock record - if ( !isset( $lockRecords[$locksKey] ) ) { - $status->warning( 'lockmanager-fail-releaselock', $path ); - continue; // nothing to do - } - $locksHeld = self::sanitizeLockArray( $lockRecords[$locksKey] ); - if ( isset( $locksHeld[$type][$this->session] ) ) { - unset( $locksHeld[$type][$this->session] ); // unregister this session - if ( $locksHeld === self::newLockArray() ) { - $ok = $memc->delete( $locksKey ); - } else { - $ok = $memc->set( $locksKey, $locksHeld ); - } - if ( !$ok ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - } else { - $status->warning( 'lockmanager-fail-releaselock', $path ); - } - wfDebug( __METHOD__ . ": released lock on key $locksKey.\n" ); - } - - // Unlock all of the active lock record keys... - $this->releaseMutexes( $memc, $keys ); - - return $status; - } - - /** - * @see QuorumLockManager::releaseAllLocks() - * @return Status - */ - protected function releaseAllLocks() { - return Status::newGood(); // not supported - } - - /** - * @see QuorumLockManager::isServerUp() - * @param string $lockSrv - * @return bool - */ - protected function isServerUp( $lockSrv ) { - return (bool)$this->getCache( $lockSrv ); - } - - /** - * Get the MemcachedBagOStuff object for a $lockSrv - * - * @param string $lockSrv Server name - * @return MemcachedBagOStuff|null - */ - protected function getCache( $lockSrv ) { - $memc = null; - if ( isset( $this->bagOStuffs[$lockSrv] ) ) { - $memc = $this->bagOStuffs[$lockSrv]; - if ( !isset( $this->serversUp[$lockSrv] ) ) { - $this->serversUp[$lockSrv] = $memc->set( __CLASS__ . ':ping', 1, 1 ); - if ( !$this->serversUp[$lockSrv] ) { - trigger_error( __METHOD__ . ": Could not contact $lockSrv.", E_USER_WARNING ); - } - } - if ( !$this->serversUp[$lockSrv] ) { - return null; // server appears to be down - } - } - - return $memc; - } - - /** - * @param string $path - * @return string - */ - protected function recordKeyForPath( $path ) { - return implode( ':', [ __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ] ); - } - - /** - * @return array An empty lock structure for a key - */ - protected static function newLockArray() { - return [ self::LOCK_SH => [], self::LOCK_EX => [] ]; - } - - /** - * @param array $a - * @return array An empty lock structure for a key - */ - protected static function sanitizeLockArray( $a ) { - if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) { - return $a; - } else { - trigger_error( __METHOD__ . ": reset invalid lock array.", E_USER_WARNING ); - - return self::newLockArray(); - } - } - - /** - * @param MemcachedBagOStuff $memc - * @param array $keys List of keys to acquire - * @return bool - */ - protected function acquireMutexes( MemcachedBagOStuff $memc, array $keys ) { - $lockedKeys = []; - - // Acquire the keys in lexicographical order, to avoid deadlock problems. - // If P1 is waiting to acquire a key P2 has, P2 can't also be waiting for a key P1 has. - sort( $keys ); - - // Try to quickly loop to acquire the keys, but back off after a few rounds. - // This reduces memcached spam, especially in the rare case where a server acquires - // some lock keys and dies without releasing them. Lock keys expire after a few minutes. - $rounds = 0; - $start = microtime( true ); - do { - if ( ( ++$rounds % 4 ) == 0 ) { - usleep( 1000 * 50 ); // 50 ms - } - foreach ( array_diff( $keys, $lockedKeys ) as $key ) { - if ( $memc->add( "$key:mutex", 1, 180 ) ) { // lock record - $lockedKeys[] = $key; - } else { - continue; // acquire in order - } - } - } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 3 ); - - if ( count( $lockedKeys ) != count( $keys ) ) { - $this->releaseMutexes( $memc, $lockedKeys ); // failed; release what was locked - return false; - } - - return true; - } - - /** - * @param MemcachedBagOStuff $memc - * @param array $keys List of acquired keys - */ - protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) { - foreach ( $keys as $key ) { - $memc->delete( "$key:mutex" ); - } - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - while ( count( $this->locksHeld ) ) { - foreach ( $this->locksHeld as $path => $locks ) { - $this->doUnlock( [ $path ], self::LOCK_EX ); - $this->doUnlock( [ $path ], self::LOCK_SH ); - } - } - } -} diff --git a/includes/filebackend/lockmanager/MySqlLockManager.php b/includes/filebackend/lockmanager/MySqlLockManager.php index 0536091bc4..5936e7d1d2 100644 --- a/includes/filebackend/lockmanager/MySqlLockManager.php +++ b/includes/filebackend/lockmanager/MySqlLockManager.php @@ -2,7 +2,11 @@ /** * MySQL version of DBLockManager that supports shared locks. * - * All lock servers must have the innodb table defined in locking/filelocks.sql. + * Do NOT use this on connection handles that are also being used for anything + * else as the transaction isolation will be wrong and all the other changes will + * get rolled back when the locks release! + * + * All lock servers must have the innodb table defined in maintenance/locking/filelocks.sql. * All locks are non-blocking, which avoids deadlocks. * * @ingroup LockManager @@ -15,9 +19,10 @@ class MySqlLockManager extends DBLockManager { self::LOCK_EX => self::LOCK_EX ]; - protected function getLocalLB() { - // Use a separate connection so releaseAllLocks() doesn't rollback the main trx - return wfGetLBFactory()->newMainLB( $this->domain ); + public function __construct( array $config ) { + parent::__construct( $config ); + + $this->session = substr( $this->session, 0, 31 ); // fit to field } protected function initConnection( $lockDb, IDatabase $db ) { @@ -29,16 +34,16 @@ class MySqlLockManager extends DBLockManager { /** * Get a connection to a lock DB and acquire locks on $paths. - * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. + * This does not use GET_LOCK() per https://bugs.mysql.com/bug.php?id=1118. * * @see DBLockManager::getLocksOnServer() * @param string $lockSrv * @param array $paths * @param string $type - * @return Status + * @return StatusValue */ protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); + $status = StatusValue::newGood(); $db = $this->getConnection( $lockSrv ); // checked in isServerUp() @@ -51,7 +56,7 @@ class MySqlLockManager extends DBLockManager { $keys[] = $key; $data[] = [ 'fls_key' => $key, 'fls_session' => $this->session ]; if ( !isset( $this->locksHeld[$path][self::LOCK_EX] ) ) { - $checkEXKeys[] = $key; + $checkEXKeys[] = $key; // this has no EX lock on $key itself } } @@ -59,13 +64,16 @@ class MySqlLockManager extends DBLockManager { $db->insert( 'filelocks_shared', $data, __METHOD__, [ 'IGNORE' ] ); # Actually do the locking queries... if ( $type == self::LOCK_SH ) { // reader locks - $blocked = false; # Bail if there are any existing writers... if ( count( $checkEXKeys ) ) { - $blocked = $db->selectField( 'filelocks_exclusive', '1', + $blocked = $db->selectField( + 'filelocks_exclusive', + '1', [ 'fle_key' => $checkEXKeys ], __METHOD__ ); + } else { + $blocked = false; } # Other prospective writers that haven't yet updated filelocks_exclusive # will recheck filelocks_shared after doing so and bail due to this entry. @@ -74,7 +82,9 @@ class MySqlLockManager extends DBLockManager { # Bail if there are any existing writers... # This may detect readers, but the safe check for them is below. # Note: if two writers come at the same time, both bail :) - $blocked = $db->selectField( 'filelocks_shared', '1', + $blocked = $db->selectField( + 'filelocks_shared', + '1', [ 'fls_key' => $keys, "fls_session != $encSession" ], __METHOD__ ); @@ -87,7 +97,9 @@ class MySqlLockManager extends DBLockManager { # Block new readers/writers... $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); # Bail if there are any existing readers... - $blocked = $db->selectField( 'filelocks_shared', '1', + $blocked = $db->selectField( + 'filelocks_shared', + '1', [ 'fls_key' => $keys, "fls_session != $encSession" ], __METHOD__ ); @@ -105,10 +117,10 @@ class MySqlLockManager extends DBLockManager { /** * @see QuorumLockManager::releaseAllLocks() - * @return Status + * @return StatusValue */ protected function releaseAllLocks() { - $status = Status::newGood(); + $status = StatusValue::newGood(); foreach ( $this->conns as $lockDb => $db ) { if ( $db->trxLevel() ) { // in transaction diff --git a/includes/filebackend/lockmanager/PostgreSqlLockManager.php b/includes/filebackend/lockmanager/PostgreSqlLockManager.php deleted file mode 100644 index d55b5ae060..0000000000 --- a/includes/filebackend/lockmanager/PostgreSqlLockManager.php +++ /dev/null @@ -1,79 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - if ( !count( $paths ) ) { - return $status; // nothing to lock - } - - $db = $this->getConnection( $lockSrv ); // checked in isServerUp() - $bigints = array_unique( array_map( - function ( $key ) { - return Wikimedia\base_convert( substr( $key, 0, 15 ), 16, 10 ); - }, - array_map( [ $this, 'sha1Base16Absolute' ], $paths ) - ) ); - - // Try to acquire all the locks... - $fields = []; - foreach ( $bigints as $bigint ) { - $fields[] = ( $type == self::LOCK_SH ) - ? "pg_try_advisory_lock_shared({$db->addQuotes( $bigint )}) AS K$bigint" - : "pg_try_advisory_lock({$db->addQuotes( $bigint )}) AS K$bigint"; - } - $res = $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); - $row = $res->fetchRow(); - - if ( in_array( 'f', $row ) ) { - // Release any acquired locks if some could not be acquired... - $fields = []; - foreach ( $row as $kbigint => $ok ) { - if ( $ok === 't' ) { // locked - $bigint = substr( $kbigint, 1 ); // strip off the "K" - $fields[] = ( $type == self::LOCK_SH ) - ? "pg_advisory_unlock_shared({$db->addQuotes( $bigint )})" - : "pg_advisory_unlock({$db->addQuotes( $bigint )})"; - } - } - if ( count( $fields ) ) { - $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); - } - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - - return $status; - } - - /** - * @see QuorumLockManager::releaseAllLocks() - * @return Status - */ - protected function releaseAllLocks() { - $status = Status::newGood(); - - foreach ( $this->conns as $lockDb => $db ) { - try { - $db->query( "SELECT pg_advisory_unlock_all()", __METHOD__ ); - } catch ( DBError $e ) { - $status->fatal( 'lockmanager-fail-db-release', $lockDb ); - } - } - - return $status; - } -} diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php deleted file mode 100644 index 108b8465cf..0000000000 --- a/includes/filebackend/lockmanager/QuorumLockManager.php +++ /dev/null @@ -1,248 +0,0 @@ - (lsrv1, lsrv2, ...)) - - /** @var array Map of degraded buckets */ - protected $degradedBuckets = []; // (buckey index => UNIX timestamp) - - final protected function doLock( array $paths, $type ) { - return $this->doLockByType( [ $type => $paths ] ); - } - - final protected function doUnlock( array $paths, $type ) { - return $this->doUnlockByType( [ $type => $paths ] ); - } - - protected function doLockByType( array $pathsByType ) { - $status = Status::newGood(); - - $pathsToLock = []; // (bucket => type => paths) - // Get locks that need to be acquired (buckets => locks)... - foreach ( $pathsByType as $type => $paths ) { - foreach ( $paths as $path ) { - if ( isset( $this->locksHeld[$path][$type] ) ) { - ++$this->locksHeld[$path][$type]; - } else { - $bucket = $this->getBucketFromPath( $path ); - $pathsToLock[$bucket][$type][] = $path; - } - } - } - - $lockedPaths = []; // files locked in this attempt (type => paths) - // Attempt to acquire these locks... - foreach ( $pathsToLock as $bucket => $pathsToLockByType ) { - // Try to acquire the locks for this bucket - $status->merge( $this->doLockingRequestBucket( $bucket, $pathsToLockByType ) ); - if ( !$status->isOK() ) { - $status->merge( $this->doUnlockByType( $lockedPaths ) ); - - return $status; - } - // Record these locks as active - foreach ( $pathsToLockByType as $type => $paths ) { - foreach ( $paths as $path ) { - $this->locksHeld[$path][$type] = 1; // locked - // Keep track of what locks were made in this attempt - $lockedPaths[$type][] = $path; - } - } - } - - return $status; - } - - protected function doUnlockByType( array $pathsByType ) { - $status = Status::newGood(); - - $pathsToUnlock = []; // (bucket => type => paths) - foreach ( $pathsByType as $type => $paths ) { - foreach ( $paths as $path ) { - if ( !isset( $this->locksHeld[$path][$type] ) ) { - $status->warning( 'lockmanager-notlocked', $path ); - } else { - --$this->locksHeld[$path][$type]; - // Reference count the locks held and release locks when zero - if ( $this->locksHeld[$path][$type] <= 0 ) { - unset( $this->locksHeld[$path][$type] ); - $bucket = $this->getBucketFromPath( $path ); - $pathsToUnlock[$bucket][$type][] = $path; - } - if ( !count( $this->locksHeld[$path] ) ) { - unset( $this->locksHeld[$path] ); // no SH or EX locks left for key - } - } - } - } - - // Remove these specific locks if possible, or at least release - // all locks once this process is currently not holding any locks. - foreach ( $pathsToUnlock as $bucket => $pathsToUnlockByType ) { - $status->merge( $this->doUnlockingRequestBucket( $bucket, $pathsToUnlockByType ) ); - } - if ( !count( $this->locksHeld ) ) { - $status->merge( $this->releaseAllLocks() ); - $this->degradedBuckets = []; // safe to retry the normal quorum - } - - return $status; - } - - /** - * Attempt to acquire locks with the peers for a bucket. - * This is all or nothing; if any key is locked then this totally fails. - * - * @param int $bucket - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - */ - final protected function doLockingRequestBucket( $bucket, array $pathsByType ) { - $status = Status::newGood(); - - $yesVotes = 0; // locks made on trustable servers - $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers - $quorum = floor( $votesLeft / 2 + 1 ); // simple majority - // Get votes for each peer, in order, until we have enough... - foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { - if ( !$this->isServerUp( $lockSrv ) ) { - --$votesLeft; - $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv ); - $this->degradedBuckets[$bucket] = time(); - continue; // server down? - } - // Attempt to acquire the lock on this peer - $status->merge( $this->getLocksOnServer( $lockSrv, $pathsByType ) ); - if ( !$status->isOK() ) { - return $status; // vetoed; resource locked - } - ++$yesVotes; // success for this peer - if ( $yesVotes >= $quorum ) { - return $status; // lock obtained - } - --$votesLeft; - $votesNeeded = $quorum - $yesVotes; - if ( $votesNeeded > $votesLeft ) { - break; // short-circuit - } - } - // At this point, we must not have met the quorum - $status->setResult( false ); - - return $status; - } - - /** - * Attempt to release locks with the peers for a bucket - * - * @param int $bucket - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - */ - final protected function doUnlockingRequestBucket( $bucket, array $pathsByType ) { - $status = Status::newGood(); - - $yesVotes = 0; // locks freed on trustable servers - $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers - $quorum = floor( $votesLeft / 2 + 1 ); // simple majority - $isDegraded = isset( $this->degradedBuckets[$bucket] ); // not the normal quorum? - foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { - if ( !$this->isServerUp( $lockSrv ) ) { - $status->warning( 'lockmanager-fail-svr-release', $lockSrv ); - } else { - // Attempt to release the lock on this peer - $status->merge( $this->freeLocksOnServer( $lockSrv, $pathsByType ) ); - ++$yesVotes; // success for this peer - // Normally the first peers form the quorum, and the others are ignored. - // Ignore them in this case, but not when an alternative quorum was used. - if ( $yesVotes >= $quorum && !$isDegraded ) { - break; // lock released - } - } - } - // Set a bad status if the quorum was not met. - // Assumes the same "up" servers as during the acquire step. - $status->setResult( $yesVotes >= $quorum ); - - return $status; - } - - /** - * Get the bucket for resource path. - * This should avoid throwing any exceptions. - * - * @param string $path - * @return int - */ - protected function getBucketFromPath( $path ) { - $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits) - return (int)base_convert( $prefix, 16, 10 ) % count( $this->srvsByBucket ); - } - - /** - * Check if a lock server is up. - * This should process cache results to reduce RTT. - * - * @param string $lockSrv - * @return bool - */ - abstract protected function isServerUp( $lockSrv ); - - /** - * Get a connection to a lock server and acquire locks - * - * @param string $lockSrv - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - */ - abstract protected function getLocksOnServer( $lockSrv, array $pathsByType ); - - /** - * Get a connection to a lock server and release locks on $paths. - * - * Subclasses must effectively implement this or releaseAllLocks(). - * - * @param string $lockSrv - * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths - * @return Status - */ - abstract protected function freeLocksOnServer( $lockSrv, array $pathsByType ); - - /** - * Release all locks that this session is holding. - * - * Subclasses must effectively implement this or freeLocksOnServer(). - * - * @return Status - */ - abstract protected function releaseAllLocks(); -} diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php deleted file mode 100644 index 4121ecb29d..0000000000 --- a/includes/filebackend/lockmanager/RedisLockManager.php +++ /dev/null @@ -1,276 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - /** @var RedisConnectionPool */ - protected $redisPool; - - /** @var array Map server names to hostname/IP and port numbers */ - protected $lockServers = []; - - /** @var string Random UUID */ - protected $session = ''; - - /** - * Construct a new instance from configuration. - * - * @param array $config Parameters include: - * - lockServers : Associative array of server names to ":" strings. - * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of server names (peers) as values. - * - redisConfig : Configuration for RedisConnectionPool::__construct(). - * @throws Exception - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - $this->lockServers = $config['lockServers']; - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - $config['redisConfig']['serializer'] = 'none'; - $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] ); - - $this->session = wfRandomString( 32 ); - } - - protected function getLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - - $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); - - $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server ); - if ( !$conn ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - - return $status; - } - - $pathsByKey = []; // (type:hash => path) map - foreach ( $pathsByType as $type => $paths ) { - $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; - foreach ( $paths as $path ) { - $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; - } - } - - try { - static $script = -<<luaEval( $script, - array_merge( - array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] - [ - $this->session, // ARGV[1] - $this->lockTTL, // ARGV[2] - time() // ARGV[3] - ] - ), - count( $pathsByKey ) # number of first argument(s) that are keys - ); - } catch ( RedisException $e ) { - $res = false; - $this->redisPool->handleError( $conn, $e ); - } - - if ( $res === false ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } else { - foreach ( $res as $key ) { - $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] ); - } - } - - return $status; - } - - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - $status = Status::newGood(); - - $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); - - $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server ); - if ( !$conn ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - - return $status; - } - - $pathsByKey = []; // (type:hash => path) map - foreach ( $pathsByType as $type => $paths ) { - $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; - foreach ( $paths as $path ) { - $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; - } - } - - try { - static $script = -<< 0 then - -- Remove the whole structure if it is now empty - if redis.call('hLen',resourceKey) == 0 then - redis.call('del',resourceKey) - end - else - failed[#failed+1] = requestKey - end - end - return failed -LUA; - $res = $conn->luaEval( $script, - array_merge( - array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] - [ - $this->session, // ARGV[1] - ] - ), - count( $pathsByKey ) # number of first argument(s) that are keys - ); - } catch ( RedisException $e ) { - $res = false; - $this->redisPool->handleError( $conn, $e ); - } - - if ( $res === false ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - } else { - foreach ( $res as $key ) { - $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] ); - } - } - - return $status; - } - - protected function releaseAllLocks() { - return Status::newGood(); // not supported - } - - protected function isServerUp( $lockSrv ) { - return (bool)$this->redisPool->getConnection( $this->lockServers[$lockSrv] ); - } - - /** - * @param string $path - * @param string $type One of (EX,SH) - * @return string - */ - protected function recordKeyForPath( $path, $type ) { - return implode( ':', - [ __CLASS__, 'locks', "$type:" . $this->sha1Base36Absolute( $path ) ] ); - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - while ( count( $this->locksHeld ) ) { - $pathsByType = []; - foreach ( $this->locksHeld as $path => $locks ) { - foreach ( $locks as $type => $count ) { - $pathsByType[$type][] = $path; - } - } - $this->unlockByType( $pathsByType ); - } - } -} diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php deleted file mode 100644 index e1a600ce11..0000000000 --- a/includes/filebackend/lockmanager/ScopedLock.php +++ /dev/null @@ -1,105 +0,0 @@ -manager = $manager; - $this->pathsByType = $pathsByType; - $this->status = $status; - } - - /** - * Get a ScopedLock object representing a lock on resource paths. - * Any locks are released once this object goes out of scope. - * The status object is updated with any errors or warnings. - * - * @param LockManager $manager - * @param array $paths List of storage paths or map of lock types to path lists - * @param int|string $type LockManager::LOCK_* constant or "mixed" and $paths - * can be a map of types to paths (since 1.22). Otherwise $type should be an - * integer and $paths should be a list of paths. - * @param Status $status - * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.22) - * @return ScopedLock|null Returns null on failure - */ - public static function factory( - LockManager $manager, array $paths, $type, Status $status, $timeout = 0 - ) { - $pathsByType = is_integer( $type ) ? [ $type => $paths ] : $paths; - $lockStatus = $manager->lockByType( $pathsByType, $timeout ); - $status->merge( $lockStatus ); - if ( $lockStatus->isOK() ) { - return new self( $manager, $pathsByType, $status ); - } - - return null; - } - - /** - * Release a scoped lock and set any errors in the attatched Status object. - * This is useful for early release of locks before function scope is destroyed. - * This is the same as setting the lock object to null. - * - * @param ScopedLock $lock - * @since 1.21 - */ - public static function release( ScopedLock &$lock = null ) { - $lock = null; - } - - /** - * Release the locks when this goes out of scope - */ - function __destruct() { - $wasOk = $this->status->isOK(); - $this->status->merge( $this->manager->unlockByType( $this->pathsByType ) ); - if ( $wasOk ) { - // Make sure status is OK, despite any unlockFiles() fatals - $this->status->setResult( true, $this->status->value ); - } - } -} diff --git a/includes/filerepo/FSRepo.php b/includes/filerepo/FSRepo.php index b24354dc4d..d06acf27ba 100644 --- a/includes/filerepo/FSRepo.php +++ b/includes/filerepo/FSRepo.php @@ -66,6 +66,7 @@ class FSRepo extends FileRepo { "{$repoName}-deleted" => $deletedDir ], 'fileMode' => $fileMode, + 'tmpDirectory' => wfTempDir() ] ); // Update repo config to use this backend $info['backend'] = $backend; diff --git a/includes/filerepo/FileBackendDBRepoWrapper.php b/includes/filerepo/FileBackendDBRepoWrapper.php index aec337e404..5bc60a0e0a 100644 --- a/includes/filerepo/FileBackendDBRepoWrapper.php +++ b/includes/filerepo/FileBackendDBRepoWrapper.php @@ -27,7 +27,7 @@ * @brief Proxy backend that manages file layout rewriting for FileRepo. * * LocalRepo may be configured to store files under their title names or by SHA-1. - * This acts as a shim in the later case, providing backwards compatability for + * This acts as a shim in the latter case, providing backwards compatability for * most callers. All "public"/"deleted" zone files actually go in an "original" * container and are never changed. * @@ -50,8 +50,10 @@ class FileBackendDBRepoWrapper extends FileBackend { protected $dbs; public function __construct( array $config ) { - $config['name'] = $config['backend']->getName(); - $config['wikiId'] = $config['backend']->getWikiId(); + /** @var FileBackend $backend */ + $backend = $config['backend']; + $config['name'] = $backend->getName(); + $config['wikiId'] = $backend->getWikiId(); parent::__construct( $config ); $this->backend = $config['backend']; $this->repoName = $config['repoName']; @@ -94,7 +96,7 @@ class FileBackendDBRepoWrapper extends FileBackend { * @return array Translated paths in same order */ public function getBackendPaths( array $paths, $latest = true ) { - $db = $this->getDB( $latest ? DB_MASTER : DB_SLAVE ); + $db = $this->getDB( $latest ? DB_MASTER : DB_REPLICA ); // @TODO: batching $resolved = []; @@ -256,7 +258,7 @@ class FileBackendDBRepoWrapper extends FileBackend { return $this->translateSrcParams( __FUNCTION__, $params ); } - public function getScopedLocksForOps( array $ops, Status $status ) { + public function getScopedLocksForOps( array $ops, StatusValue $status ) { return $this->backend->getScopedLocksForOps( $ops, $status ); } diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index 4ab913da0c..66dab99217 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -393,7 +393,7 @@ class FileRepo { if ( $this->oldFileFactory ) { return call_user_func( $this->oldFileFactory, $title, $this, $time ); } else { - return false; + return null; } } else { return call_user_func( $this->fileFactory, $title, $this ); @@ -482,8 +482,8 @@ class FileRepo { * @param array $items An array of titles, or an array of findFile() options with * the "title" option giving the title. Example: * - * $findItem = array( 'title' => $title, 'private' => true ); - * $findBatch = array( $findItem ); + * $findItem = [ 'title' => $title, 'private' => true ]; + * $findBatch = [ $findItem ]; * $repo->findFiles( $findBatch ); * * No title should appear in $items twice, as the result use titles as keys @@ -818,14 +818,14 @@ class FileRepo { * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the * same contents as the source * self::SKIP_LOCKING Skip any file locking when doing the store - * @return FileRepoStatus + * @return Status */ public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { $this->assertWritableRepo(); // fail out if read-only $status = $this->storeBatch( [ [ $srcPath, $dstZone, $dstRel ] ], $flags ); if ( $status->successCount == 0 ) { - $status->ok = false; + $status->setOK( false ); } return $status; @@ -841,7 +841,7 @@ class FileRepo { * same contents as the source * self::SKIP_LOCKING Skip any file locking when doing the store * @throws MWException - * @return FileRepoStatus + * @return Status */ public function storeBatch( array $triplets, $flags = 0 ) { $this->assertWritableRepo(); // fail out if read-only @@ -912,7 +912,7 @@ class FileRepo { * @param array $files List of files to delete * @param int $flags Bitwise combination of the following flags: * self::SKIP_LOCKING Skip any file locking when doing the deletions - * @return FileRepoStatus + * @return Status */ public function cleanupBatch( array $files, $flags = 0 ) { $this->assertWritableRepo(); // fail out if read-only @@ -952,7 +952,7 @@ class FileRepo { * @param array|string|null $options An array consisting of a key named headers * listing extra headers. If a string, taken as content-disposition header. * (Support for array of options new in 1.23) - * @return FileRepoStatus + * @return Status */ final public function quickImport( $src, $dst, $options = null ) { return $this->quickImportBatch( [ [ $src, $dst, $options ] ] ); @@ -964,7 +964,7 @@ class FileRepo { * This is intended for purging thumbnails. * * @param string $path Virtual URL or storage path - * @return FileRepoStatus + * @return Status */ final public function quickPurge( $path ) { return $this->quickPurgeBatch( [ $path ] ); @@ -995,7 +995,7 @@ class FileRepo { * When "headers" are given they are used as HTTP headers if supported. * * @param array $triples List of (source path or FSFile, destination path, disposition) - * @return FileRepoStatus + * @return Status */ public function quickImportBatch( array $triples ) { $status = $this->newGood(); @@ -1040,7 +1040,7 @@ class FileRepo { * This does no locking nor journaling and is intended for purging thumbnails. * * @param array $paths List of virtual URLs or storage paths - * @return FileRepoStatus + * @return Status */ public function quickPurgeBatch( array $paths ) { $status = $this->newGood(); @@ -1065,7 +1065,7 @@ class FileRepo { * @param string $originalName The base name of the file as specified * by the user. The file extension will be maintained. * @param string $srcPath The current location of the file. - * @return FileRepoStatus Object with the URL in the value. + * @return Status Object with the URL in the value. */ public function storeTemp( $originalName, $srcPath ) { $this->assertWritableRepo(); // fail out if read-only @@ -1107,7 +1107,7 @@ class FileRepo { * @param string $dstPath Target file system path * @param int $flags Bitwise combination of the following flags: * self::DELETE_SOURCE Delete the source files on success - * @return FileRepoStatus + * @return Status */ public function concatenate( array $srcPaths, $dstPath, $flags = 0 ) { $this->assertWritableRepo(); // fail out if read-only @@ -1156,7 +1156,7 @@ class FileRepo { * @param int $flags Bitfield, may be FileRepo::DELETE_SOURCE to indicate * that the source file should be deleted if possible * @param array $options Optional additional parameters - * @return FileRepoStatus + * @return Status */ public function publish( $src, $dstRel, $archiveRel, $flags = 0, array $options = [] @@ -1166,7 +1166,7 @@ class FileRepo { $status = $this->publishBatch( [ [ $src, $dstRel, $archiveRel, $options ] ], $flags ); if ( $status->successCount == 0 ) { - $status->ok = false; + $status->setOK( false ); } if ( isset( $status->value[0] ) ) { $status->value = $status->value[0]; @@ -1185,7 +1185,7 @@ class FileRepo { * @param int $flags Bitfield, may be FileRepo::DELETE_SOURCE to indicate * that the source files should be deleted if possible * @throws MWException - * @return FileRepoStatus + * @return Status */ public function publishBatch( array $ntuples, $flags = 0 ) { $this->assertWritableRepo(); // fail out if read-only @@ -1322,7 +1322,10 @@ class FileRepo { $params = [ 'noAccess' => true, 'noListing' => true ] + $params; } - return $this->backend->prepare( $params ); + $status = $this->newGood(); + $status->merge( $this->backend->prepare( $params ) ); + + return $status; } /** @@ -1380,7 +1383,7 @@ class FileRepo { * @param mixed $srcRel Relative path for the file to be deleted * @param mixed $archiveRel Relative path for the archive location. * Relative to a private archive directory. - * @return FileRepoStatus + * @return Status */ public function delete( $srcRel, $archiveRel ) { $this->assertWritableRepo(); // fail out if read-only @@ -1403,7 +1406,7 @@ class FileRepo { * public root in the first element, and the archive file path relative * to the deleted zone root in the second element. * @throws MWException - * @return FileRepoStatus + * @return Status */ public function deleteBatch( array $sourceDestPairs ) { $this->assertWritableRepo(); // fail out if read-only @@ -1539,9 +1542,15 @@ class FileRepo { * @return array */ public function getFileProps( $virtualUrl ) { - $path = $this->resolveToStoragePath( $virtualUrl ); + $fsFile = $this->getLocalReference( $virtualUrl ); + $mwProps = new MWFileProps( MimeMagic::singleton() ); + if ( $fsFile ) { + $props = $mwProps->getPropsFromPath( $fsFile->getPath(), true ); + } else { + $props = $mwProps->newPlaceholderProps(); + } - return $this->backend->getFileProps( [ 'src' => $path ] ); + return $props; } /** @@ -1593,7 +1602,10 @@ class FileRepo { $path = $this->resolveToStoragePath( $virtualUrl ); $params = [ 'src' => $path, 'headers' => $headers, 'options' => $optHeaders ]; - return $this->backend->streamFile( $params ); + $status = $this->newGood(); + $status->merge( $this->backend->streamFile( $params ) ); + + return $status; } /** diff --git a/includes/filerepo/ForeignAPIRepo.php b/includes/filerepo/ForeignAPIRepo.php index b48191fd1c..4176c8240e 100644 --- a/includes/filerepo/ForeignAPIRepo.php +++ b/includes/filerepo/ForeignAPIRepo.php @@ -28,13 +28,13 @@ use MediaWiki\Logger\LoggerFactory; * * Example config: * - * $wgForeignFileRepos[] = array( + * $wgForeignFileRepos[] = [ * 'class' => 'ForeignAPIRepo', * 'name' => 'shared', * 'apibase' => 'https://en.wikipedia.org/w/api.php', * 'fetchDescription' => true, // Optional * 'descriptionCacheExpiry' => 3600, - * ); + * ]; * * @ingroup FileRepo */ @@ -63,8 +63,8 @@ class ForeignAPIRepo extends FileRepo { /** @var array */ protected $mFileExists = []; - /** @var array */ - private $mQueryCache = []; + /** @var string */ + private $mApiBase; /** * @param array|null $info @@ -397,7 +397,8 @@ class ForeignAPIRepo extends FileRepo { } /* There is a new Commons file, or existing thumbnail older than a month */ } - $thumb = self::httpGet( $foreignUrl ); + + $thumb = self::httpGet( $foreignUrl, 'default', [], $mtime ); if ( !$thumb ) { wfDebug( __METHOD__ . " Could not download thumb\n" ); @@ -413,7 +414,11 @@ class ForeignAPIRepo extends FileRepo { return $foreignUrl; } $knownThumbUrls[$sizekey] = $localUrl; - $cache->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry ); + + $ttl = $mtime + ? $cache->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry ) + : $this->apiThumbCacheExpiry; + $cache->set( $key, $knownThumbUrls, $ttl ); wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache \n" ); return $localUrl; @@ -506,9 +511,12 @@ class ForeignAPIRepo extends FileRepo { * @param string $url * @param string $timeout * @param array $options + * @param integer|bool &$mtime Resulting Last-Modified UNIX timestamp if received * @return bool|string */ - public static function httpGet( $url, $timeout = 'default', $options = [] ) { + public static function httpGet( + $url, $timeout = 'default', $options = [], &$mtime = false + ) { $options['timeout'] = $timeout; /* Http::get */ $url = wfExpandUrl( $url, PROTO_HTTP ); @@ -524,6 +532,9 @@ class ForeignAPIRepo extends FileRepo { $status = $req->execute(); if ( $status->isOK() ) { + $lmod = $req->getResponseHeader( 'Last-Modified' ); + $mtime = $lmod ? wfTimestamp( TS_UNIX, $lmod ) : false; + return $req->getContent(); } else { $logger = LoggerFactory::getInstance( 'http' ); @@ -531,6 +542,7 @@ class ForeignAPIRepo extends FileRepo { $status->getWikiText( false, false, 'en' ), [ 'caller' => 'ForeignAPIRepo::httpGet' ] ); + return false; } } @@ -548,7 +560,7 @@ class ForeignAPIRepo extends FileRepo { * @param string $target Used in cache key creation, mostly * @param array $query The query parameters for the API request * @param int $cacheTTL Time to live for the memcached caching - * @return null + * @return string|null */ public function httpGetCached( $target, $query, $cacheTTL = 3600 ) { if ( $this->mApiBase ) { @@ -557,28 +569,23 @@ class ForeignAPIRepo extends FileRepo { $url = $this->makeUrl( $query, 'api' ); } - if ( !isset( $this->mQueryCache[$url] ) ) { - $data = ObjectCache::getMainWANInstance()->getWithSetCallback( - $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) ), - $cacheTTL, - function () use ( $url ) { - return ForeignAPIRepo::httpGet( $url ); + $cache = ObjectCache::getMainWANInstance(); + return $cache->getWithSetCallback( + $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) ), + $cacheTTL, + function ( $curValue, &$ttl ) use ( $url, $cache ) { + $html = self::httpGet( $url, 'default', [], $mtime ); + if ( $html !== false ) { + $ttl = $mtime ? $cache->adaptiveTTL( $mtime, $ttl ) : $ttl; + } else { + $ttl = $cache->adaptiveTTL( $mtime, $ttl ); + $html = null; // caches negatives } - ); - if ( !$data ) { - return null; - } - - if ( count( $this->mQueryCache ) > 100 ) { - // Keep the cache from growing infinitely - $this->mQueryCache = []; - } - - $this->mQueryCache[$url] = $data; - } - - return $this->mQueryCache[$url]; + return $html; + }, + [ 'pcTTL' => $cache::TTL_PROC_LONG ] + ); } /** diff --git a/includes/filerepo/ForeignDBRepo.php b/includes/filerepo/ForeignDBRepo.php index 001800f3ba..f49ef88c5c 100644 --- a/includes/filerepo/ForeignDBRepo.php +++ b/includes/filerepo/ForeignDBRepo.php @@ -51,9 +51,12 @@ class ForeignDBRepo extends LocalRepo { /** @var bool */ protected $hasSharedCache; - # Other stuff + /** @var IDatabase */ protected $dbConn; + + /** @var callable */ protected $fileFactory = [ 'ForeignDBFile', 'newFromTitle' ]; + /** @var callable */ protected $fileFromRowFactory = [ 'ForeignDBFile', 'newFromRow' ]; /** @@ -86,7 +89,7 @@ class ForeignDBRepo extends LocalRepo { /** * @return IDatabase */ - function getSlaveDB() { + function getReplicaDB() { return $this->getMasterDB(); } @@ -106,7 +109,7 @@ class ForeignDBRepo extends LocalRepo { ]; return function ( $index ) use ( $type, $params ) { - return DatabaseBase::factory( $type, $params ); + return Database::factory( $type, $params ); }; } diff --git a/includes/filerepo/ForeignDBViaLBRepo.php b/includes/filerepo/ForeignDBViaLBRepo.php index a59ca34a04..a9cd030869 100644 --- a/includes/filerepo/ForeignDBViaLBRepo.php +++ b/includes/filerepo/ForeignDBViaLBRepo.php @@ -42,6 +42,9 @@ class ForeignDBViaLBRepo extends LocalRepo { /** @var array */ protected $fileFromRowFactory = [ 'ForeignDBFile', 'newFromRow' ]; + /** @var bool */ + protected $hasSharedCache; + /** * @param array|null $info */ @@ -56,23 +59,22 @@ class ForeignDBViaLBRepo extends LocalRepo { * @return IDatabase */ function getMasterDB() { - return wfGetDB( DB_MASTER, [], $this->wiki ); + return wfGetLB( $this->wiki )->getConnectionRef( DB_MASTER, [], $this->wiki ); } /** * @return IDatabase */ - function getSlaveDB() { - return wfGetDB( DB_SLAVE, [], $this->wiki ); + function getReplicaDB() { + return wfGetLB( $this->wiki )->getConnectionRef( DB_REPLICA, [], $this->wiki ); } /** * @return Closure */ protected function getDBFactory() { - $wiki = $this->wiki; - return function( $index ) use ( $wiki ) { - return wfGetDB( $index, [], $wiki ); + return function( $index ) { + return wfGetLB( $this->wiki )->getConnectionRef( $index, [], $this->wiki ); }; } diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php index eaec15129c..d49ae7bf4b 100644 --- a/includes/filerepo/LocalRepo.php +++ b/includes/filerepo/LocalRepo.php @@ -29,28 +29,24 @@ * @ingroup FileRepo */ class LocalRepo extends FileRepo { - /** @var array */ + /** @var callable */ protected $fileFactory = [ 'LocalFile', 'newFromTitle' ]; - - /** @var array */ + /** @var callable */ protected $fileFactoryKey = [ 'LocalFile', 'newFromKey' ]; - - /** @var array */ + /** @var callable */ protected $fileFromRowFactory = [ 'LocalFile', 'newFromRow' ]; - - /** @var array */ + /** @var callable */ protected $oldFileFromRowFactory = [ 'OldLocalFile', 'newFromRow' ]; - - /** @var array */ + /** @var callable */ protected $oldFileFactory = [ 'OldLocalFile', 'newFromTitle' ]; - - /** @var array */ + /** @var callable */ protected $oldFileFactoryKey = [ 'OldLocalFile', 'newFromKey' ]; function __construct( array $info = null ) { parent::__construct( $info ); - $this->hasSha1Storage = isset( $info['storageLayout'] ) && $info['storageLayout'] === 'sha1'; + $this->hasSha1Storage = isset( $info['storageLayout'] ) + && $info['storageLayout'] === 'sha1'; if ( $this->hasSha1Storage() ) { $this->backend = new FileBackendDBRepoWrapper( [ @@ -93,7 +89,7 @@ class LocalRepo extends FileRepo { * * @param array $storageKeys * - * @return FileRepoStatus + * @return Status */ function cleanupDeletedBatch( array $storageKeys ) { if ( $this->hasSha1Storage() ) { @@ -199,12 +195,12 @@ class LocalRepo extends FileRepo { $expiry = 86400; // has invalidation, 1 day } - $that = $this; + $method = __METHOD__; $redirDbKey = ObjectCache::getMainWANInstance()->getWithSetCallback( $memcKey, $expiry, - function ( $oldValue, &$ttl, array &$setOpts ) use ( $that, $title ) { - $dbr = $that->getSlaveDB(); // possibly remote DB + function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) { + $dbr = $this->getReplicaDB(); // possibly remote DB $setOpts += Database::getCacheSetOptions( $dbr ); @@ -217,7 +213,7 @@ class LocalRepo extends FileRepo { 'page_title' => $title->getDBkey(), 'rd_from = page_id' ], - __METHOD__ + $method ); } else { $row = false; @@ -302,7 +298,7 @@ class LocalRepo extends FileRepo { } }; - $dbr = $this->getSlaveDB(); + $dbr = $this->getReplicaDB(); // Query image table $imgNames = []; @@ -372,7 +368,7 @@ class LocalRepo extends FileRepo { * @return File[] */ function findBySha1( $hash ) { - $dbr = $this->getSlaveDB(); + $dbr = $this->getReplicaDB(); $res = $dbr->select( 'image', LocalFile::selectFields(), @@ -404,7 +400,7 @@ class LocalRepo extends FileRepo { return []; // empty parameter } - $dbr = $this->getSlaveDB(); + $dbr = $this->getReplicaDB(); $res = $dbr->select( 'image', LocalFile::selectFields(), @@ -434,7 +430,7 @@ class LocalRepo extends FileRepo { $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ]; // Query database - $dbr = $this->getSlaveDB(); + $dbr = $this->getReplicaDB(); $res = $dbr->select( 'image', LocalFile::selectFields(), @@ -453,23 +449,33 @@ class LocalRepo extends FileRepo { } /** - * Get a connection to the slave DB - * @return DatabaseBase + * Get a connection to the replica DB + * @return IDatabase + */ + function getReplicaDB() { + return wfGetDB( DB_REPLICA ); + } + + /** + * Alias for getReplicaDB() + * + * @return IDatabase + * @deprecated Since 1.29 */ function getSlaveDB() { - return wfGetDB( DB_SLAVE ); + return $this->getReplicaDB(); } /** * Get a connection to the master DB - * @return DatabaseBase + * @return IDatabase */ function getMasterDB() { return wfGetDB( DB_MASTER ); } /** - * Get a callback to get a DB handle given an index (DB_SLAVE/DB_MASTER) + * Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER) * @return Closure */ protected function getDBFactory() { @@ -500,9 +506,12 @@ class LocalRepo extends FileRepo { function invalidateImageRedirect( Title $title ) { $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) ); if ( $key ) { - $this->getMasterDB()->onTransactionPreCommitOrIdle( function() use ( $key ) { - ObjectCache::getMainWANInstance()->delete( $key ); - } ); + $this->getMasterDB()->onTransactionPreCommitOrIdle( + function () use ( $key ) { + ObjectCache::getMainWANInstance()->delete( $key ); + }, + __METHOD__ + ); } } @@ -559,7 +568,7 @@ class LocalRepo extends FileRepo { * * @param string $function * @param array $args - * @return FileRepoStatus + * @return Status */ protected function skipWriteOperationIfSha1( $function, array $args ) { $this->assertWritableRepo(); // fail out if read-only diff --git a/includes/filerepo/RepoGroup.php b/includes/filerepo/RepoGroup.php index d515b05088..d47624f8cd 100644 --- a/includes/filerepo/RepoGroup.php +++ b/includes/filerepo/RepoGroup.php @@ -135,17 +135,18 @@ class RepoGroup { } # Check the cache + $dbkey = $title->getDBkey(); if ( empty( $options['ignoreRedirect'] ) && empty( $options['private'] ) && empty( $options['bypassCache'] ) ) { $time = isset( $options['time'] ) ? $options['time'] : ''; - $dbkey = $title->getDBkey(); if ( $this->cache->has( $dbkey, $time, 60 ) ) { return $this->cache->get( $dbkey, $time ); } $useCache = true; } else { + $time = false; $useCache = false; } @@ -451,7 +452,9 @@ class RepoGroup { return $repo->getFileProps( $fileName ); } else { - return FSFile::getPropsFromPath( $fileName ); + $mwProps = new MWFileProps( MimeMagic::singleton() ); + + return $mwProps->getPropsFromPath( $fileName, true ); } } diff --git a/includes/filerepo/file/ArchivedFile.php b/includes/filerepo/file/ArchivedFile.php index ca1ea84827..921e129c36 100644 --- a/includes/filerepo/file/ArchivedFile.php +++ b/includes/filerepo/file/ArchivedFile.php @@ -177,7 +177,7 @@ class ArchivedFile { if ( !$this->title || $this->title->getNamespace() == NS_FILE ) { $this->dataLoaded = true; // set it here, to have also true on miss - $dbr = wfGetDB( DB_SLAVE ); + $dbr = wfGetDB( DB_REPLICA ); $row = $dbr->selectRow( 'filearchive', self::selectFields(), @@ -425,6 +425,7 @@ class ArchivedFile { */ function pageCount() { if ( !isset( $this->pageCount ) ) { + // @FIXME: callers expect File objects if ( $this->getHandler() && $this->handler->isMultiPage( $this ) ) { $this->pageCount = $this->handler->pageCount( $this ); } else { diff --git a/includes/filerepo/file/File.php b/includes/filerepo/file/File.php index 425a08cb04..c1d5573b68 100644 --- a/includes/filerepo/file/File.php +++ b/includes/filerepo/file/File.php @@ -1328,7 +1328,7 @@ abstract class File implements IDBAccessObject { */ protected function makeTransformTmpFile( $thumbPath ) { $thumbExt = FileBackend::extensionFromPath( $thumbPath ); - return TempFSFile::factory( 'transform_', $thumbExt ); + return TempFSFile::factory( 'transform_', $thumbExt, wfTempDir() ); } /** @@ -1805,7 +1805,7 @@ abstract class File implements IDBAccessObject { * @param int $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move rather than copy * @param array $options Optional additional parameters - * @return FileRepoStatus On success, the value member contains the + * @return Status On success, the value member contains the * archive name, or an empty string if it was a new file. * * STUB @@ -1905,7 +1905,7 @@ abstract class File implements IDBAccessObject { * and logging are caller's responsibility * * @param Title $target New file name - * @return FileRepoStatus + * @return Status */ function move( $target ) { $this->readOnlyError(); @@ -1922,7 +1922,7 @@ abstract class File implements IDBAccessObject { * @param string $reason * @param bool $suppress Hide content from sysops? * @param User|null $user - * @return FileRepoStatus + * @return Status * STUB * Overridden by LocalFile */ diff --git a/includes/filerepo/file/ForeignAPIFile.php b/includes/filerepo/file/ForeignAPIFile.php index f6752d8308..43b6855f82 100644 --- a/includes/filerepo/file/ForeignAPIFile.php +++ b/includes/filerepo/file/ForeignAPIFile.php @@ -28,7 +28,10 @@ * @ingroup FileAbstraction */ class ForeignAPIFile extends File { + /** @var bool */ private $mExists; + /** @var array */ + private $mInfo = []; protected $repoClass = 'ForeignApiRepo'; @@ -244,7 +247,7 @@ class ForeignAPIFile extends File { public function getUser( $type = 'text' ) { if ( $type == 'text' ) { return isset( $this->mInfo['user'] ) ? strval( $this->mInfo['user'] ) : null; - } elseif ( $type == 'id' ) { + } else { return 0; // What makes sense here, for a remote user? } } @@ -344,9 +347,6 @@ class ForeignAPIFile extends File { return $files; } - /** - * @see File::purgeCache() - */ function purgeCache( $options = [] ) { $this->purgeThumbnails( $options ); $this->purgeDescriptionPage(); diff --git a/includes/filerepo/file/ForeignDBFile.php b/includes/filerepo/file/ForeignDBFile.php index cf0045e43f..df50a670a2 100644 --- a/includes/filerepo/file/ForeignDBFile.php +++ b/includes/filerepo/file/ForeignDBFile.php @@ -57,7 +57,7 @@ class ForeignDBFile extends LocalFile { * @param string $srcPath * @param int $flags * @param array $options - * @return FileRepoStatus + * @return Status * @throws MWException */ function publish( $srcPath, $flags = 0, array $options = [] ) { @@ -84,7 +84,7 @@ class ForeignDBFile extends LocalFile { /** * @param array $versions * @param bool $unsuppress - * @return FileRepoStatus + * @return Status * @throws MWException */ function restore( $versions = [], $unsuppress = false ) { @@ -95,7 +95,7 @@ class ForeignDBFile extends LocalFile { * @param string $reason * @param bool $suppress * @param User|null $user - * @return FileRepoStatus + * @return Status * @throws MWException */ function delete( $reason, $suppress = false, $user = null ) { @@ -104,7 +104,7 @@ class ForeignDBFile extends LocalFile { /** * @param Title $target - * @return FileRepoStatus + * @return Status * @throws MWException */ function move( $target ) { @@ -136,7 +136,7 @@ class ForeignDBFile extends LocalFile { return false; } - $touched = $this->repo->getSlaveDB()->selectField( + $touched = $this->repo->getReplicaDB()->selectField( 'page', 'page_touched', [ @@ -179,7 +179,7 @@ class ForeignDBFile extends LocalFile { * @since 1.27 */ public function getDescriptionShortUrl() { - $dbr = $this->repo->getSlaveDB(); + $dbr = $this->repo->getReplicaDB(); $pageId = $dbr->selectField( 'page', 'page_id', diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index 40141c9cd0..011ba87ee7 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -23,11 +23,6 @@ use \MediaWiki\Logger\LoggerFactory; -/** - * Bump this number when serialized cache records may be incompatible. - */ -define( 'MW_FILE_VERSION', 9 ); - /** * Class to represent a local file in the wiki's own database * @@ -46,6 +41,8 @@ define( 'MW_FILE_VERSION', 9 ); * @ingroup FileAbstraction */ class LocalFile extends File { + const VERSION = 10; // cache version + const CACHE_FIELD_MAX_LEN = 1000; /** @var bool Does the file exist on disk? (loadFromXxx) */ @@ -177,7 +174,7 @@ class LocalFile extends File { * @return bool|LocalFile */ static function newFromKey( $sha1, $repo, $timestamp = false ) { - $dbr = $repo->getSlaveDB(); + $dbr = $repo->getReplicaDB(); $conds = [ 'img_sha1' => $sha1 ]; if ( $timestamp ) { @@ -240,77 +237,71 @@ class LocalFile extends File { * @return string|bool */ function getCacheKey() { - $hashedName = md5( $this->getName() ); - - return $this->repo->getSharedCacheKey( 'file', $hashedName ); + return $this->repo->getSharedCacheKey( 'file', sha1( $this->getName() ) ); } /** - * Try to load file metadata from memcached. Returns true on success. - * @return bool + * Try to load file metadata from memcached, falling back to the database */ private function loadFromCache() { $this->dataLoaded = false; $this->extraDataLoaded = false; - $key = $this->getCacheKey(); + $key = $this->getCacheKey(); if ( !$key ) { - return false; - } - - $cache = ObjectCache::getMainWANInstance(); - $cachedValues = $cache->get( $key ); + $this->loadFromDB( self::READ_NORMAL ); - // Check if the key existed and belongs to this version of MediaWiki - if ( is_array( $cachedValues ) && $cachedValues['version'] == MW_FILE_VERSION ) { - $this->fileExists = $cachedValues['fileExists']; - if ( $this->fileExists ) { - $this->setProps( $cachedValues ); - } - $this->dataLoaded = true; - $this->extraDataLoaded = true; - foreach ( $this->getLazyCacheFields( '' ) as $field ) { - $this->extraDataLoaded = $this->extraDataLoaded && isset( $cachedValues[$field] ); - } + return; } - return $this->dataLoaded; - } - - /** - * Save the file metadata to memcached - */ - private function saveToCache() { - $this->load(); + $cache = ObjectCache::getMainWANInstance(); + $cachedValues = $cache->getWithSetCallback( + $key, + $cache::TTL_WEEK, + function ( $oldValue, &$ttl, array &$setOpts ) use ( $cache ) { + $setOpts += Database::getCacheSetOptions( $this->repo->getReplicaDB() ); + + $this->loadFromDB( self::READ_NORMAL ); + + $fields = $this->getCacheFields( '' ); + $cacheVal['fileExists'] = $this->fileExists; + if ( $this->fileExists ) { + foreach ( $fields as $field ) { + $cacheVal[$field] = $this->$field; + } + } + // Strip off excessive entries from the subset of fields that can become large. + // If the cache value gets to large it will not fit in memcached and nothing will + // get cached at all, causing master queries for any file access. + foreach ( $this->getLazyCacheFields( '' ) as $field ) { + if ( isset( $cacheVal[$field] ) + && strlen( $cacheVal[$field] ) > 100 * 1024 + ) { + unset( $cacheVal[$field] ); // don't let the value get too big + } + } - $key = $this->getCacheKey(); - if ( !$key ) { - return; - } + if ( $this->fileExists ) { + $ttl = $cache->adaptiveTTL( wfTimestamp( TS_UNIX, $this->timestamp ), $ttl ); + } else { + $ttl = $cache::TTL_DAY; + } - $fields = $this->getCacheFields( '' ); - $cacheVal = [ 'version' => MW_FILE_VERSION ]; - $cacheVal['fileExists'] = $this->fileExists; + return $cacheVal; + }, + [ 'version' => self::VERSION ] + ); + $this->fileExists = $cachedValues['fileExists']; if ( $this->fileExists ) { - foreach ( $fields as $field ) { - $cacheVal[$field] = $this->$field; - } + $this->setProps( $cachedValues ); } - // Strip off excessive entries from the subset of fields that can become large. - // If the cache value gets to large it will not fit in memcached and nothing will - // get cached at all, causing master queries for any file access. + $this->dataLoaded = true; + $this->extraDataLoaded = true; foreach ( $this->getLazyCacheFields( '' ) as $field ) { - if ( isset( $cacheVal[$field] ) && strlen( $cacheVal[$field] ) > 100 * 1024 ) { - unset( $cacheVal[$field] ); // don't let the value get too big - } + $this->extraDataLoaded = $this->extraDataLoaded && isset( $cachedValues[$field] ); } - - // Cache presence for 1 week and negatives for 1 day - $ttl = $this->fileExists ? 86400 * 7 : 86400; - $opts = Database::getCacheSetOptions( $this->repo->getSlaveDB() ); - ObjectCache::getMainWANInstance()->set( $key, $cacheVal, $ttl, $opts ); } /** @@ -322,9 +313,12 @@ class LocalFile extends File { return; } - $this->repo->getMasterDB()->onTransactionPreCommitOrIdle( function() use ( $key ) { - ObjectCache::getMainWANInstance()->delete( $key ); - } ); + $this->repo->getMasterDB()->onTransactionPreCommitOrIdle( + function () use ( $key ) { + ObjectCache::getMainWANInstance()->delete( $key ); + }, + __METHOD__ + ); } /** @@ -396,7 +390,7 @@ class LocalFile extends File { $dbr = ( $flags & self::READ_LATEST ) ? $this->repo->getMasterDB() - : $this->repo->getSlaveDB(); + : $this->repo->getReplicaDB(); $row = $dbr->selectRow( 'image', $this->getCacheFields( 'img_' ), [ 'img_name' => $this->getName() ], $fname ); @@ -418,7 +412,7 @@ class LocalFile extends File { # Unconditionally set loaded=true, we don't want the accessors constantly rechecking $this->extraDataLoaded = true; - $fieldMap = $this->loadFieldsWithTimestamp( $this->repo->getSlaveDB(), $fname ); + $fieldMap = $this->loadFieldsWithTimestamp( $this->repo->getReplicaDB(), $fname ); if ( !$fieldMap ) { $fieldMap = $this->loadFieldsWithTimestamp( $this->repo->getMasterDB(), $fname ); } @@ -440,16 +434,18 @@ class LocalFile extends File { private function loadFieldsWithTimestamp( $dbr, $fname ) { $fieldMap = false; - $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ), - [ 'img_name' => $this->getName(), 'img_timestamp' => $this->getTimestamp() ], - $fname ); + $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ), [ + 'img_name' => $this->getName(), + 'img_timestamp' => $dbr->timestamp( $this->getTimestamp() ) + ], $fname ); if ( $row ) { $fieldMap = $this->unprefixRow( $row, 'img_' ); } else { # File may have been uploaded over in the meantime; check the old versions - $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ), - [ 'oi_name' => $this->getName(), 'oi_timestamp' => $this->getTimestamp() ], - $fname ); + $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ), [ + 'oi_name' => $this->getName(), + 'oi_timestamp' => $dbr->timestamp( $this->getTimestamp() ) + ], $fname ); if ( $row ) { $fieldMap = $this->unprefixRow( $row, 'oi_' ); } @@ -494,7 +490,7 @@ class LocalFile extends File { $decoded['timestamp'] = wfTimestamp( TS_MW, $decoded['timestamp'] ); - $decoded['metadata'] = $this->repo->getSlaveDB()->decodeBlob( $decoded['metadata'] ); + $decoded['metadata'] = $this->repo->getReplicaDB()->decodeBlob( $decoded['metadata'] ); if ( empty( $decoded['major_mime'] ) ) { $decoded['mime'] = 'unknown/unknown'; @@ -545,12 +541,13 @@ class LocalFile extends File { */ function load( $flags = 0 ) { if ( !$this->dataLoaded ) { - if ( ( $flags & self::READ_LATEST ) || !$this->loadFromCache() ) { + if ( $flags & self::READ_LATEST ) { $this->loadFromDB( $flags ); - $this->saveToCache(); + } else { + $this->loadFromCache(); } - $this->dataLoaded = true; } + if ( ( $flags & self::LOAD_ALL ) && !$this->extraDataLoaded ) { // @note: loads on name/timestamp to reduce race condition problems $this->loadExtraFromDB(); @@ -766,7 +763,7 @@ class LocalFile extends File { if ( $type == 'text' ) { return $this->user_text; - } elseif ( $type == 'id' ) { + } else { // id return (int)$this->user; } } @@ -1040,7 +1037,7 @@ class LocalFile extends File { * @return OldLocalFile[] */ function getHistory( $limit = null, $start = null, $end = null, $inc = true ) { - $dbr = $this->repo->getSlaveDB(); + $dbr = $this->repo->getReplicaDB(); $tables = [ 'oldimage' ]; $fields = OldLocalFile::selectFields(); $conds = $opts = $join_conds = []; @@ -1094,7 +1091,7 @@ class LocalFile extends File { # Polymorphic function name to distinguish foreign and local fetches $fname = get_class( $this ) . '::' . __FUNCTION__; - $dbr = $this->repo->getSlaveDB(); + $dbr = $this->repo->getReplicaDB(); if ( $this->historyLine == 0 ) { // called for the first time, return line from cur $this->historyRes = $dbr->select( 'image', @@ -1163,7 +1160,7 @@ class LocalFile extends File { * @param User|null $user User object or null to use $wgUser * @param string[] $tags Change tags to add to the log entry and page revision. * (This doesn't check $user's permissions.) - * @return FileRepoStatus On success, the value member contains the + * @return Status On success, the value member contains the * archive name, or an empty string if it was a new file. */ function upload( $src, $comment, $pageText, $flags = 0, $props = false, @@ -1182,7 +1179,8 @@ class LocalFile extends File { ) { $props = $this->repo->getFileProps( $srcPath ); } else { - $props = FSFile::getPropsFromPath( $srcPath ); + $mwProps = new MWFileProps( MimeMagic::singleton() ); + $props = $mwProps->getPropsFromPath( $srcPath, true ); } } @@ -1483,8 +1481,10 @@ class LocalFile extends File { ); if ( isset( $status->value['revision'] ) ) { + /** @var $rev Revision */ + $rev = $status->value['revision']; // Associate new page revision id - $logEntry->setAssociatedRevId( $status->value['revision']->getId() ); + $logEntry->setAssociatedRevId( $rev->getId() ); } // This relies on the resetArticleID() call in WikiPage::insertOn(), // which is triggered on $descTitle by doEditContent() above. @@ -1582,7 +1582,7 @@ class LocalFile extends File { * @param int $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move rather than copy * @param array $options Optional additional parameters - * @return FileRepoStatus On success, the value member contains the + * @return Status On success, the value member contains the * archive name, or an empty string if it was a new file. */ function publish( $src, $flags = 0, array $options = [] ) { @@ -1601,7 +1601,7 @@ class LocalFile extends File { * @param int $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move rather than copy * @param array $options Optional additional parameters - * @return FileRepoStatus On success, the value member contains the + * @return Status On success, the value member contains the * archive name, or an empty string if it was a new file. */ function publishTo( $src, $dstRel, $flags = 0, array $options = [] ) { @@ -1621,7 +1621,9 @@ class LocalFile extends File { $sha1 = $repo->isVirtualUrl( $srcPath ) ? $repo->getFileSha1( $srcPath ) : FSFile::getSha1Base36FromPath( $srcPath ); - $dst = $repo->getBackend()->getPathForSHA1( $sha1 ); + /** @var FileBackendDBRepoWrapper $wrapperBackend */ + $wrapperBackend = $repo->getBackend(); + $dst = $wrapperBackend->getPathForSHA1( $sha1 ); $status = $repo->quickImport( $src, $dst ); if ( $flags & File::DELETE_SOURCE ) { unlink( $srcPath ); @@ -1661,7 +1663,7 @@ class LocalFile extends File { * and logging are caller's responsibility * * @param Title $target New file name - * @return FileRepoStatus + * @return Status */ function move( $target ) { if ( $this->getRepo()->getReadOnlyReason() !== false ) { @@ -1720,7 +1722,7 @@ class LocalFile extends File { * @param string $reason * @param bool $suppress * @param User|null $user - * @return FileRepoStatus + * @return Status */ function delete( $reason, $suppress = false, $user = null ) { if ( $this->getRepo()->getReadOnlyReason() !== false ) { @@ -1778,7 +1780,7 @@ class LocalFile extends File { * @param bool $suppress * @param User|null $user * @throws MWException Exception on database or file store failure - * @return FileRepoStatus + * @return Status */ function deleteOld( $archiveName, $reason, $suppress = false, $user = null ) { if ( $this->getRepo()->getReadOnlyReason() !== false ) { @@ -1814,7 +1816,7 @@ class LocalFile extends File { * @param array $versions Set of record ids of deleted items to restore, * or empty to restore all revisions. * @param bool $unsuppress - * @return FileRepoStatus + * @return Status */ function restore( $versions = [], $unsuppress = false ) { if ( $this->getRepo()->getReadOnlyReason() !== false ) { @@ -1915,7 +1917,7 @@ class LocalFile extends File { 'page_namespace' => $this->title->getNamespace(), 'page_title' => $this->title->getDBkey() ]; - $touched = $this->repo->getSlaveDB()->selectField( 'page', 'page_touched', $cond, __METHOD__ ); + $touched = $this->repo->getReplicaDB()->selectField( 'page', 'page_touched', $cond, __METHOD__ ); $this->descriptionTouched = $touched ? wfTimestamp( TS_MW, $touched ) : false; } @@ -2006,12 +2008,15 @@ class LocalFile extends File { } // Release the lock *after* commit to avoid row-level contention. // Make sure it triggers on rollback() as well as commit() (T132921). - $dbw->onTransactionResolution( function () use ( $logger ) { - $status = $this->releaseFileLock(); - if ( !$status->isGood() ) { - $logger->error( "Failed to unlock '{file}'", [ 'file' => $this->name ] ); - } - } ); + $dbw->onTransactionResolution( + function () use ( $logger ) { + $status = $this->releaseFileLock(); + if ( !$status->isGood() ) { + $logger->error( "Failed to unlock '{file}'", [ 'file' => $this->name ] ); + } + }, + __METHOD__ + ); // Callers might care if the SELECT snapshot is safely fresh $this->lockedOwnTrx = $makesTransaction; } @@ -2229,12 +2234,7 @@ class LocalFileDeleteBatch { // Bitfields to further suppress the content if ( $this->suppress ) { - $bitfield = 0; - // This should be 15... - $bitfield |= Revision::DELETED_TEXT; - $bitfield |= Revision::DELETED_COMMENT; - $bitfield |= Revision::DELETED_USER; - $bitfield |= Revision::DELETED_RESTRICTED; + $bitfield = Revision::SUPPRESSED_ALL; } else { $bitfield = 'oi_deleted'; } @@ -2254,7 +2254,6 @@ class LocalFileDeleteBatch { 'fa_deleted_timestamp' => $encTimestamp, 'fa_deleted_reason' => $encReason, 'fa_deleted' => $this->suppress ? $bitfield : 0, - 'fa_name' => 'img_name', 'fa_archive_name' => 'NULL', 'fa_size' => 'img_size', @@ -2341,7 +2340,7 @@ class LocalFileDeleteBatch { /** * Run the transaction - * @return FileRepoStatus + * @return Status */ public function execute() { $repo = $this->file->getRepo(); @@ -2489,9 +2488,10 @@ class LocalFileRestoreBatch { * rows and there's no need to keep the image row locked while it's acquiring those locks * The caller may have its own transaction open. * So we save the batch and let the caller call cleanup() - * @return FileRepoStatus + * @return Status */ public function execute() { + /** @var Language */ global $wgLang; $repo = $this->file->getRepo(); @@ -2689,7 +2689,7 @@ class LocalFileRestoreBatch { // Even if some files could be copied, fail entirely as that is the // easiest thing to do without data loss $this->cleanupFailedBatch( $storeStatus, $storeBatch ); - $status->ok = false; + $status->setOK( false ); $this->file->unlock(); return $status; @@ -2789,7 +2789,7 @@ class LocalFileRestoreBatch { /** * Delete unused files in the deleted zone. * This should be called from outside the transaction in which execute() was called. - * @return FileRepoStatus + * @return Status */ public function cleanup() { if ( !$this->cleanupBatch ) { @@ -2846,7 +2846,7 @@ class LocalFileMoveBatch { protected $archive; - /** @var DatabaseBase */ + /** @var IDatabase */ protected $db; /** @@ -2924,7 +2924,7 @@ class LocalFileMoveBatch { /** * Perform the move. - * @return FileRepoStatus + * @return Status */ public function execute() { $repo = $this->file->repo; @@ -2949,7 +2949,7 @@ class LocalFileMoveBatch { if ( !$statusDb->isGood() ) { $destFile->unlock(); $this->file->unlock(); - $statusDb->ok = false; + $statusDb->setOK( false ); return $statusDb; } @@ -2968,7 +2968,7 @@ class LocalFileMoveBatch { $this->file->unlock(); wfDebugLog( 'imagemove', "Error in moving files: " . $statusMove->getWikiText( false, false, 'en' ) ); - $statusMove->ok = false; + $statusMove->setOK( false ); return $statusMove; } @@ -2996,7 +2996,7 @@ class LocalFileMoveBatch { * Verify the database updates and return a new FileRepoStatus indicating how * many rows would be updated. * - * @return FileRepoStatus + * @return Status */ protected function verifyDBUpdates() { $repo = $this->file->repo; diff --git a/includes/filerepo/file/OldLocalFile.php b/includes/filerepo/file/OldLocalFile.php index 31e62ecbe9..dfaae731c1 100644 --- a/includes/filerepo/file/OldLocalFile.php +++ b/includes/filerepo/file/OldLocalFile.php @@ -86,7 +86,7 @@ class OldLocalFile extends LocalFile { * @return bool|OldLocalFile */ static function newFromKey( $sha1, $repo, $timestamp = false ) { - $dbr = $repo->getSlaveDB(); + $dbr = $repo->getReplicaDB(); $conds = [ 'oi_sha1' => $sha1 ]; if ( $timestamp ) { @@ -179,7 +179,7 @@ class OldLocalFile extends LocalFile { $dbr = ( $flags & self::READ_LATEST ) ? $this->repo->getMasterDB() - : $this->repo->getSlaveDB(); + : $this->repo->getReplicaDB(); $conds = [ 'oi_name' => $this->getName() ]; if ( is_null( $this->requestedTime ) ) { @@ -194,16 +194,14 @@ class OldLocalFile extends LocalFile { } else { $this->fileExists = false; } - } /** * Load lazy file metadata from the DB */ protected function loadExtraFromDB() { - $this->extraDataLoaded = true; - $dbr = $this->repo->getSlaveDB(); + $dbr = $this->repo->getReplicaDB(); $conds = [ 'oi_name' => $this->getName() ]; if ( is_null( $this->requestedTime ) ) { $conds['oi_archive_name'] = $this->archive_name; @@ -227,7 +225,6 @@ class OldLocalFile extends LocalFile { } else { throw new MWException( "Could not find data for image '{$this->archive_name}'." ); } - } /** @@ -332,7 +329,7 @@ class OldLocalFile extends LocalFile { * @param string $timestamp * @param string $comment * @param User $user - * @return FileRepoStatus + * @return Status */ function uploadOld( $srcPath, $archiveName, $timestamp, $comment, $user ) { $this->lock(); diff --git a/includes/gallery/TraditionalImageGallery.php b/includes/gallery/TraditionalImageGallery.php index f6527b820e..0f889da683 100644 --- a/includes/gallery/TraditionalImageGallery.php +++ b/includes/gallery/TraditionalImageGallery.php @@ -238,8 +238,8 @@ class TraditionalImageGallery extends ImageGalleryBase { } /** - * How much padding such the thumb have between image and inner div that - * that contains the border. This is both for verical and horizontal + * How much padding the thumb has between the image and the inner div + * that contains the border. This is for both vertical and horizontal * padding. (However, it is cut in half in the vertical direction). * @return int */ diff --git a/includes/htmlform/HTMLForm.php b/includes/htmlform/HTMLForm.php index 3c88594eac..71ccaa36e5 100644 --- a/includes/htmlform/HTMLForm.php +++ b/includes/htmlform/HTMLForm.php @@ -147,12 +147,16 @@ class HTMLForm extends ContextSource { 'namespaceselect' => 'HTMLSelectNamespace', 'namespaceselectwithbutton' => 'HTMLSelectNamespaceWithButton', 'tagfilter' => 'HTMLTagFilter', + 'sizefilter' => 'HTMLSizeFilterField', 'submit' => 'HTMLSubmitField', 'hidden' => 'HTMLHiddenField', 'edittools' => 'HTMLEditTools', 'checkmatrix' => 'HTMLCheckMatrix', 'cloner' => 'HTMLFormFieldCloner', 'autocompleteselect' => 'HTMLAutoCompleteSelectField', + 'date' => 'HTMLDateTimeField', + 'time' => 'HTMLDateTimeField', + 'datetime' => 'HTMLDateTimeField', // HTMLTextField will output the correct type="" attribute automagically. // There are about four zillion other HTML5 input types, like range, but // we don't use those at the moment, so no point in adding all of them. @@ -173,7 +177,7 @@ class HTMLForm extends ContextSource { protected $mFieldTree; protected $mShowReset = false; protected $mShowSubmit = true; - protected $mSubmitFlags = [ 'constructive', 'primary' ]; + protected $mSubmitFlags = [ 'primary', 'progressive' ]; protected $mShowCancel = false; protected $mCancelTarget; @@ -600,10 +604,14 @@ class HTMLForm extends ContextSource { */ public function trySubmit() { $valid = true; - $hoistedErrors = []; - $hoistedErrors[] = isset( $this->mValidationErrorMessage ) - ? $this->mValidationErrorMessage - : [ 'htmlform-invalid-input' ]; + $hoistedErrors = Status::newGood(); + if ( $this->mValidationErrorMessage ) { + foreach ( (array)$this->mValidationErrorMessage as $error ) { + call_user_func_array( [ $hoistedErrors, 'fatal' ], $error ); + } + } else { + $hoistedErrors->fatal( 'htmlform-invalid-input' ); + } $this->mWasSubmitted = true; @@ -630,15 +638,16 @@ class HTMLForm extends ContextSource { if ( $res !== true ) { $valid = false; if ( $res !== false && !$field->canDisplayErrors() ) { - $hoistedErrors[] = [ 'rawmessage', $res ]; + if ( is_string( $res ) ) { + $hoistedErrors->fatal( 'rawmessage', $res ); + } else { + $hoistedErrors->fatal( $res ); + } } } } if ( !$valid ) { - if ( count( $hoistedErrors ) === 1 ) { - $hoistedErrors = $hoistedErrors[0]; - } return $hoistedErrors; } @@ -1014,7 +1023,8 @@ class HTMLForm extends ContextSource { $this->getOutput()->addModuleStyles( 'mediawiki.htmlform.styles' ); $html = '' - . $this->getErrors( $submitResult ) + . $this->getErrorsOrWarnings( $submitResult, 'error' ) + . $this->getErrorsOrWarnings( $submitResult, 'warning' ) . $this->getHeaderText() . $this->getBody() . $this->getHiddenFields() @@ -1230,23 +1240,46 @@ class HTMLForm extends ContextSource { * * @param string|array|Status $errors * + * @deprecated since 1.28, use getErrorsOrWarnings() instead + * * @return string */ public function getErrors( $errors ) { - if ( $errors instanceof Status ) { - if ( $errors->isOK() ) { - $errorstr = ''; + wfDeprecated( __METHOD__ ); + return $this->getErrorsOrWarnings( $errors, 'error' ); + } + + /** + * Returns a formatted list of errors or warnings from the given elements. + * + * @param string|array|Status $elements The set of errors/warnings to process. + * @param string $elementsType Should warnings or errors be returned. This is meant + * for Status objects, all other valid types are always considered as errors. + * @return string + */ + public function getErrorsOrWarnings( $elements, $elementsType ) { + if ( !in_array( $elementsType, [ 'error', 'warning' ], true ) ) { + throw new DomainException( $elementsType . ' is not a valid type.' ); + } + $elementstr = false; + if ( $elements instanceof Status ) { + list( $errorStatus, $warningStatus ) = $elements->splitByErrorType(); + $status = $elementsType === 'error' ? $errorStatus : $warningStatus; + if ( $status->isGood() ) { + $elementstr = ''; } else { - $errorstr = $this->getOutput()->parse( $errors->getWikiText() ); + $elementstr = $this->getOutput()->parse( + $status->getWikiText() + ); } - } elseif ( is_array( $errors ) ) { - $errorstr = $this->formatErrors( $errors ); - } else { - $errorstr = $errors; + } elseif ( is_array( $elements ) && $elementsType === 'error' ) { + $elementstr = $this->formatErrors( $elements ); + } elseif ( $elementsType === 'error' ) { + $elementstr = $elements; } - return $errorstr - ? Html::rawElement( 'div', [ 'class' => 'error' ], $errorstr ) + return $elementstr + ? Html::rawElement( 'div', [ 'class' => $elementsType ], $elementstr ) : ''; } diff --git a/includes/htmlform/HTMLFormField.php b/includes/htmlform/HTMLFormField.php index 25b4ccacaf..71aa275449 100644 --- a/includes/htmlform/HTMLFormField.php +++ b/includes/htmlform/HTMLFormField.php @@ -42,7 +42,7 @@ abstract class HTMLFormField { * * @return string Valid HTML. */ - abstract function getInputHTML( $value ); + abstract public function getInputHTML( $value ); /** * Same as getInputHTML, but returns an OOUI object. @@ -51,7 +51,7 @@ abstract class HTMLFormField { * @param string $value * @return OOUI\Widget|false */ - function getInputOOUI( $value ) { + public function getInputOOUI( $value ) { return false; } @@ -74,7 +74,7 @@ abstract class HTMLFormField { * * @return Message */ - function msg() { + public function msg() { $args = func_get_args(); if ( $this->mParent ) { @@ -266,7 +266,7 @@ abstract class HTMLFormField { * @param array $alldata The data collected from the form * @return bool */ - function isHidden( $alldata ) { + public function isHidden( $alldata ) { if ( !$this->mHideIf ) { return false; } @@ -284,7 +284,7 @@ abstract class HTMLFormField { * * @return bool True to cancel the submission */ - function cancelSubmit( $value, $alldata ) { + public function cancelSubmit( $value, $alldata ) { return false; } @@ -296,10 +296,10 @@ abstract class HTMLFormField { * @param string|array $value The value the field was submitted with * @param array $alldata The data collected from the form * - * @return bool|string True on success, or String error to display, or + * @return bool|string|Message True on success, or String/Message error to display, or * false to fail validation without displaying an error. */ - function validate( $value, $alldata ) { + public function validate( $value, $alldata ) { if ( $this->isHidden( $alldata ) ) { return true; } @@ -308,7 +308,7 @@ abstract class HTMLFormField { && $this->mParams['required'] !== false && $value === '' ) { - return $this->msg( 'htmlform-required' )->parse(); + return $this->msg( 'htmlform-required' ); } if ( isset( $this->mValidationCallback ) ) { @@ -318,7 +318,7 @@ abstract class HTMLFormField { return true; } - function filter( $value, $alldata ) { + public function filter( $value, $alldata ) { if ( isset( $this->mFilterCallback ) ) { $value = call_user_func( $this->mFilterCallback, $value, $alldata, $this->mParent ); } @@ -370,7 +370,7 @@ abstract class HTMLFormField { * @param WebRequest $request * @return string The value */ - function loadDataFromRequest( $request ) { + public function loadDataFromRequest( $request ) { if ( $request->getCheck( $this->mName ) ) { return $request->getText( $this->mName ); } else { @@ -386,7 +386,7 @@ abstract class HTMLFormField { * @since 1.22 The 'label' attribute no longer accepts raw HTML, use 'label-raw' instead * @throws MWException */ - function __construct( $params ) { + public function __construct( $params ) { $this->mParams = $params; if ( isset( $params['parent'] ) && $params['parent'] instanceof HTMLForm ) { @@ -472,7 +472,7 @@ abstract class HTMLFormField { * * @return string Complete HTML table row. */ - function getTableRow( $value ) { + public function getTableRow( $value ) { list( $errors, $errorClass ) = $this->getErrorsAndErrorClass( $value ); $inputHtml = $this->getInputHTML( $value ); $fieldType = get_class( $this ); @@ -602,7 +602,7 @@ abstract class HTMLFormField { } $fieldType = get_class( $this ); - $helpText = $this->getHelpText(); + $help = $this->getHelpText(); $errors = $this->getErrorsRaw( $value ); foreach ( $errors as &$error ) { $error = new OOUI\HtmlSnippet( $error ); @@ -616,7 +616,7 @@ abstract class HTMLFormField { $config = [ 'classes' => [ "mw-htmlform-field-$fieldType", $this->mClass ], 'align' => $this->getLabelAlignOOUI(), - 'help' => $helpText !== null ? new OOUI\HtmlSnippet( $helpText ) : null, + 'help' => ( $help !== null && $help !== '' ) ? new OOUI\HtmlSnippet( $help ) : null, 'errors' => $errors, 'notices' => $notices, 'infusable' => $infusable, @@ -821,7 +821,7 @@ abstract class HTMLFormField { /** * Determine the help text to display * @since 1.20 - * @return string HTML + * @return string|null HTML */ public function getHelpText() { $helptext = null; @@ -903,7 +903,7 @@ abstract class HTMLFormField { * @since 1.28 * @return string[] */ - function getNotices() { + public function getNotices() { $notices = []; if ( isset( $this->mParams['notice-message'] ) ) { @@ -924,11 +924,11 @@ abstract class HTMLFormField { /** * @return string HTML */ - function getLabel() { + public function getLabel() { return is_null( $this->mLabel ) ? '' : $this->mLabel; } - function getLabelHtml( $cellAttributes = [] ) { + public function getLabelHtml( $cellAttributes = [] ) { # Don't output a for= attribute for labels with no associated input. # Kind of hacky here, possibly we don't want these to be