maintenance/dumpCategoriesAsRdf.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  */
  19 use Wikimedia\Purtle\RdfWriter;
  20 use Wikimedia\Purtle\RdfWriterFactory;
  21 use Wikimedia\Rdbms\IDatabase;
  22
  23 require_once __DIR__ . '/Maintenance.php';
  24
  25 /**
  26  * Maintenance script to provide RDF representation of the category tree.
  27  *
  28  * @ingroup Maintenance
  29  * @since 1.30
  30  */
  31 class DumpCategoriesAsRdf extends Maintenance {
  32         /**
  33          * @var RdfWriter
  34          */
  35         private $rdfWriter;
  36         /**
  37          * Categories RDF helper.
  38          * @var CategoriesRdf
  39          */
  40         private $categoriesRdf;
  41
  42         public function __construct() {
  43                 parent::__construct();
  44
  45                 $this->addDescription( "Generate RDF dump of categories in a wiki." );
  46
  47                 $this->setBatchSize( 200 );
  48                 $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.",
  49                         false, true );
  50                 $this->addOption( 'format', "Set the dump format.", false, true );
  51         }
  52
  53         /**
  54          * Produce row iterator for categories.
  55          * @param IDatabase $dbr Database connection
  56          * @return RecursiveIterator
  57          */
  58         public function getCategoryIterator( IDatabase $dbr ) {
  59                 $it = new BatchRowIterator(
  60                         $dbr,
  61                         [ 'page', 'page_props', 'category' ],
  62                         [ 'page_title' ],
  63                         $this->getBatchSize()
  64                 );
  65                 $it->addConditions( [
  66                         'page_namespace' => NS_CATEGORY,
  67                 ] );
  68                 $it->setFetchColumns( [
  69                         'page_title',
  70                         'page_id',
  71                         'pp_propname',
  72                         'cat_pages',
  73                         'cat_subcats',
  74                         'cat_files'
  75                 ] );
  76                 $it->addJoinConditions(
  77                         [
  78                                 'page_props' => [
  79                                         'LEFT JOIN', [ 'pp_propname' => 'hiddencat', 'pp_page = page_id' ]
  80                                 ],
  81                                 'category' => [
  82                                         'LEFT JOIN', [ 'cat_title = page_title' ]
  83                                 ]
  84                         ]
  85
  86                 );
  87                 return $it;
  88         }
  89
  90         /**
  91          * Get iterator for links for categories.
  92          * @param IDatabase $dbr
  93          * @param array $ids List of page IDs
  94          * @return Traversable
  95          */
  96         public function getCategoryLinksIterator( IDatabase $dbr, array $ids ) {
  97                 $it = new BatchRowIterator(
  98                         $dbr,
  99                         'categorylinks',
 100                         [ 'cl_from', 'cl_to' ],
 101                         $this->getBatchSize()
 102                 );
 103                 $it->addConditions( [
 104                         'cl_type' => 'subcat',
 105                         'cl_from' => $ids
 106                 ] );
 107                 $it->setFetchColumns( [ 'cl_from', 'cl_to' ] );
 108                 return new RecursiveIteratorIterator( $it );
 109         }
 110
 111         /**
 112          * @param int $timestamp
 113          */
 114         public function addDumpHeader( $timestamp ) {
 115                 global $wgRightsUrl;
 116                 $licenseUrl = $wgRightsUrl;
 117                 if ( substr( $licenseUrl, 0, 2 ) == '//' ) {
 118                         $licenseUrl = 'https:' . $licenseUrl;
 119                 }
 120                 $this->rdfWriter->about( $this->categoriesRdf->getDumpURI() )
 121                         ->a( 'schema', 'Dataset' )
 122                         ->a( 'owl', 'Ontology' )
 123                         ->say( 'cc', 'license' )->is( $licenseUrl )
 124                         ->say( 'schema', 'softwareVersion' )->value( CategoriesRdf::FORMAT_VERSION )
 125                         ->say( 'schema', 'dateModified' )
 126                                 ->value( wfTimestamp( TS_ISO_8601, $timestamp ), 'xsd', 'dateTime' )
 127                         ->say( 'schema', 'isPartOf' )->is( wfExpandUrl( '/', PROTO_CANONICAL ) )
 128                         ->say( 'owl', 'imports' )->is( CategoriesRdf::OWL_URL );
 129         }
 130
 131         public function execute() {
 132                 $outFile = $this->getOption( 'output', 'php://stdout' );
 133
 134                 if ( $outFile === '-' ) {
 135                         $outFile = 'php://stdout';
 136                 }
 137
 138                 $output = fopen( $outFile, 'w' );
 139                 $this->rdfWriter = $this->createRdfWriter( $this->getOption( 'format', 'ttl' ) );
 140                 $this->categoriesRdf = new CategoriesRdf( $this->rdfWriter );
 141
 142                 $this->categoriesRdf->setupPrefixes();
 143                 $this->rdfWriter->start();
 144
 145                 $this->addDumpHeader( time() );
 146                 fwrite( $output, $this->rdfWriter->drain() );
 147
 148                 $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] );
 149
 150                 foreach ( $this->getCategoryIterator( $dbr ) as $batch ) {
 151                         $pages = [];
 152                         foreach ( $batch as $row ) {
 153                                 $this->categoriesRdf->writeCategoryData(
 154                                         $row->page_title,
 155                                         $row->pp_propname === 'hiddencat',
 156                                         (int)$row->cat_pages - (int)$row->cat_subcats - (int)$row->cat_files,
 157                                         (int)$row->cat_subcats
 158                                 );
 159                                 $pages[$row->page_id] = $row->page_title;
 160                         }
 161
 162                         foreach ( $this->getCategoryLinksIterator( $dbr, array_keys( $pages ) ) as $row ) {
 163                                 $this->categoriesRdf->writeCategoryLinkData( $pages[$row->cl_from], $row->cl_to );
 164                         }
 165                         fwrite( $output, $this->rdfWriter->drain() );
 166                 }
 167                 fflush( $output );
 168                 if ( $outFile !== '-' ) {
 169                         fclose( $output );
 170                 }
 171         }
 172
 173         /**
 174          * @param string $format Writer format
 175          * @return RdfWriter
 176          */
 177         private function createRdfWriter( $format ) {
 178                 $factory = new RdfWriterFactory();
 179                 return $factory->getWriter( $factory->getFormatName( $format ) );
 180         }
 181 }
 182
 183 $maintClass = DumpCategoriesAsRdf::class;
 184 require_once RUN_MAINTENANCE_IF_MAIN;