6e23e536391317f58d30e86f3a325541bf71f3d4
[lhc/web/wiklou.git] / tests / phpunit / includes / db / LBFactoryTest.php
1 <?php
2 /**
3 * Holds tests for LBFactory abstract MediaWiki class.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @author Antoine Musso
22 * @copyright © 2013 Antoine Musso
23 * @copyright © 2013 Wikimedia Foundation Inc.
24 */
25
26 use Wikimedia\Rdbms\LBFactorySimple;
27 use Wikimedia\Rdbms\LBFactoryMulti;
28 use Wikimedia\Rdbms\LoadBalancer;
29 use Wikimedia\Rdbms\ChronologyProtector;
30 use Wikimedia\Rdbms\DatabaseMysqli;
31 use Wikimedia\Rdbms\MySQLMasterPos;
32 use Wikimedia\Rdbms\DatabaseDomain;
33
34 /**
35 * @group Database
36 * @covers \Wikimedia\Rdbms\LBFactorySimple
37 * @covers \Wikimedia\Rdbms\LBFactoryMulti
38 */
39 class LBFactoryTest extends MediaWikiTestCase {
40
41 /**
42 * @covers MWLBFactory::getLBFactoryClass
43 * @dataProvider getLBFactoryClassProvider
44 */
45 public function testGetLBFactoryClass( $expected, $deprecated ) {
46 $mockDB = $this->getMockBuilder( DatabaseMysqli::class )
47 ->disableOriginalConstructor()
48 ->getMock();
49
50 $config = [
51 'class' => $deprecated,
52 'connection' => $mockDB,
53 # Various other parameters required:
54 'sectionsByDB' => [],
55 'sectionLoads' => [],
56 'serverTemplate' => [],
57 ];
58
59 $this->hideDeprecated( '$wgLBFactoryConf must be updated. See RELEASE-NOTES for details' );
60 $result = MWLBFactory::getLBFactoryClass( $config );
61
62 $this->assertEquals( $expected, $result );
63 }
64
65 public function getLBFactoryClassProvider() {
66 return [
67 # Format: new class, old class
68 [ Wikimedia\Rdbms\LBFactorySimple::class, 'LBFactory_Simple' ],
69 [ Wikimedia\Rdbms\LBFactorySingle::class, 'LBFactory_Single' ],
70 [ Wikimedia\Rdbms\LBFactoryMulti::class, 'LBFactory_Multi' ],
71 [ Wikimedia\Rdbms\LBFactorySimple::class, 'LBFactorySimple' ],
72 [ Wikimedia\Rdbms\LBFactorySingle::class, 'LBFactorySingle' ],
73 [ Wikimedia\Rdbms\LBFactoryMulti::class, 'LBFactoryMulti' ],
74 ];
75 }
76
77 public function testLBFactorySimpleServer() {
78 global $wgDBserver, $wgDBname, $wgDBuser, $wgDBpassword, $wgDBtype, $wgSQLiteDataDir;
79
80 $servers = [
81 [
82 'host' => $wgDBserver,
83 'dbname' => $wgDBname,
84 'user' => $wgDBuser,
85 'password' => $wgDBpassword,
86 'type' => $wgDBtype,
87 'dbDirectory' => $wgSQLiteDataDir,
88 'load' => 0,
89 'flags' => DBO_TRX // REPEATABLE-READ for consistency
90 ],
91 ];
92
93 $factory = new LBFactorySimple( [ 'servers' => $servers ] );
94 $lb = $factory->getMainLB();
95
96 $dbw = $lb->getConnection( DB_MASTER );
97 $this->assertTrue( $dbw->getLBInfo( 'master' ), 'master shows as master' );
98
99 $dbr = $lb->getConnection( DB_REPLICA );
100 $this->assertTrue( $dbr->getLBInfo( 'master' ), 'DB_REPLICA also gets the master' );
101
102 $factory->shutdown();
103 $lb->closeAll();
104 }
105
106 public function testLBFactorySimpleServers() {
107 global $wgDBserver, $wgDBname, $wgDBuser, $wgDBpassword, $wgDBtype, $wgSQLiteDataDir;
108
109 $servers = [
110 [ // master
111 'host' => $wgDBserver,
112 'dbname' => $wgDBname,
113 'user' => $wgDBuser,
114 'password' => $wgDBpassword,
115 'type' => $wgDBtype,
116 'dbDirectory' => $wgSQLiteDataDir,
117 'load' => 0,
118 'flags' => DBO_TRX // REPEATABLE-READ for consistency
119 ],
120 [ // emulated slave
121 'host' => $wgDBserver,
122 'dbname' => $wgDBname,
123 'user' => $wgDBuser,
124 'password' => $wgDBpassword,
125 'type' => $wgDBtype,
126 'dbDirectory' => $wgSQLiteDataDir,
127 'load' => 100,
128 'flags' => DBO_TRX // REPEATABLE-READ for consistency
129 ]
130 ];
131
132 $factory = new LBFactorySimple( [
133 'servers' => $servers,
134 'loadMonitorClass' => LoadMonitorNull::class
135 ] );
136 $lb = $factory->getMainLB();
137
138 $dbw = $lb->getConnection( DB_MASTER );
139 $this->assertTrue( $dbw->getLBInfo( 'master' ), 'master shows as master' );
140 $this->assertEquals(
141 ( $wgDBserver != '' ) ? $wgDBserver : 'localhost',
142 $dbw->getLBInfo( 'clusterMasterHost' ),
143 'cluster master set' );
144
145 $dbr = $lb->getConnection( DB_REPLICA );
146 $this->assertTrue( $dbr->getLBInfo( 'replica' ), 'slave shows as slave' );
147 $this->assertEquals(
148 ( $wgDBserver != '' ) ? $wgDBserver : 'localhost',
149 $dbr->getLBInfo( 'clusterMasterHost' ),
150 'cluster master set' );
151
152 $factory->shutdown();
153 $lb->closeAll();
154 }
155
156 public function testLBFactoryMultiConns() {
157 $factory = $this->newLBFactoryMultiLBs();
158
159 $dbw = $factory->getMainLB()->getConnection( DB_MASTER );
160 $this->assertTrue( $dbw->getLBInfo( 'master' ), 'master shows as master' );
161
162 $dbr = $factory->getMainLB()->getConnection( DB_REPLICA );
163 $this->assertTrue( $dbr->getLBInfo( 'replica' ), 'slave shows as slave' );
164
165 // Destructor should trigger without round stage errors
166 unset( $factory );
167 }
168
169 public function testLBFactoryMultiRoundCallbacks() {
170 $called = 0;
171 $countLBsFunc = function ( LBFactoryMulti $factory ) {
172 $count = 0;
173 $factory->forEachLB( function () use ( &$count ) {
174 ++$count;
175 } );
176
177 return $count;
178 };
179
180 $factory = $this->newLBFactoryMultiLBs();
181 $this->assertEquals( 0, $countLBsFunc( $factory ) );
182 $dbw = $factory->getMainLB()->getConnection( DB_MASTER );
183 $this->assertEquals( 1, $countLBsFunc( $factory ) );
184 // Test that LoadBalancer instances made during pre-commit callbacks in do not
185 // throw DBTransactionError due to transaction ROUND_* stages being mismatched.
186 $factory->beginMasterChanges( __METHOD__ );
187 $dbw->onTransactionPreCommitOrIdle( function () use ( $factory, &$called ) {
188 ++$called;
189 // Trigger s1 LoadBalancer instantiation during "finalize" stage.
190 // There is no s1wiki DB to select so it is not in getConnection(),
191 // but this fools getMainLB() at least.
192 $factory->getMainLB( 's1wiki' )->getConnection( DB_MASTER );
193 } );
194 $factory->commitMasterChanges( __METHOD__ );
195 $this->assertEquals( 1, $called );
196 $this->assertEquals( 2, $countLBsFunc( $factory ) );
197 $factory->shutdown();
198 $factory->closeAll();
199
200 $called = 0;
201 $factory = $this->newLBFactoryMultiLBs();
202 $this->assertEquals( 0, $countLBsFunc( $factory ) );
203 $dbw = $factory->getMainLB()->getConnection( DB_MASTER );
204 $this->assertEquals( 1, $countLBsFunc( $factory ) );
205 // Test that LoadBalancer instances made during pre-commit callbacks in do not
206 // throw DBTransactionError due to transaction ROUND_* stages being mismatched.hrow
207 // DBTransactionError due to transaction ROUND_* stages being mismatched.
208 $factory->beginMasterChanges( __METHOD__ );
209 $dbw->query( "SELECT 1 as t", __METHOD__ );
210 $dbw->onTransactionResolution( function () use ( $factory, &$called ) {
211 ++$called;
212 // Trigger s1 LoadBalancer instantiation during "finalize" stage.
213 // There is no s1wiki DB to select so it is not in getConnection(),
214 // but this fools getMainLB() at least.
215 $factory->getMainLB( 's1wiki' )->getConnection( DB_MASTER );
216 } );
217 $factory->commitMasterChanges( __METHOD__ );
218 $this->assertEquals( 1, $called );
219 $this->assertEquals( 2, $countLBsFunc( $factory ) );
220 $factory->shutdown();
221 $factory->closeAll();
222
223 $factory = $this->newLBFactoryMultiLBs();
224 $dbw = $factory->getMainLB()->getConnection( DB_MASTER );
225 // DBTransactionError should not be thrown
226 $ran = 0;
227 $dbw->onTransactionPreCommitOrIdle( function () use ( &$ran ) {
228 ++$ran;
229 } );
230 $factory->commitAll( __METHOD__ );
231 $this->assertEquals( 1, $ran );
232
233 $factory->shutdown();
234 $factory->closeAll();
235 }
236
237 private function newLBFactoryMultiLBs() {
238 global $wgDBserver, $wgDBname, $wgDBuser, $wgDBpassword, $wgDBtype, $wgSQLiteDataDir;
239
240 return new LBFactoryMulti( [
241 'sectionsByDB' => [
242 's1wiki' => 's1',
243 ],
244 'sectionLoads' => [
245 's1' => [
246 'test-db3' => 0,
247 'test-db4' => 100,
248 ],
249 'DEFAULT' => [
250 'test-db1' => 0,
251 'test-db2' => 100,
252 ]
253 ],
254 'serverTemplate' => [
255 'dbname' => $wgDBname,
256 'user' => $wgDBuser,
257 'password' => $wgDBpassword,
258 'type' => $wgDBtype,
259 'dbDirectory' => $wgSQLiteDataDir,
260 'flags' => DBO_DEFAULT
261 ],
262 'hostsByName' => [
263 'test-db1' => $wgDBserver,
264 'test-db2' => $wgDBserver,
265 'test-db3' => $wgDBserver,
266 'test-db4' => $wgDBserver
267 ],
268 'loadMonitorClass' => LoadMonitorNull::class
269 ] );
270 }
271
272 /**
273 * @covers \Wikimedia\Rdbms\ChronologyProtector
274 */
275 public function testChronologyProtector() {
276 $now = microtime( true );
277
278 // (a) First HTTP request
279 $m1Pos = new MySQLMasterPos( 'db1034-bin.000976/843431247', $now );
280 $m2Pos = new MySQLMasterPos( 'db1064-bin.002400/794074907', $now );
281
282 // Master DB 1
283 $mockDB1 = $this->getMockBuilder( DatabaseMysqli::class )
284 ->disableOriginalConstructor()
285 ->getMock();
286 $mockDB1->method( 'writesOrCallbacksPending' )->willReturn( true );
287 $mockDB1->method( 'lastDoneWrites' )->willReturn( $now );
288 $mockDB1->method( 'getMasterPos' )->willReturn( $m1Pos );
289 // Load balancer for master DB 1
290 $lb1 = $this->getMockBuilder( LoadBalancer::class )
291 ->disableOriginalConstructor()
292 ->getMock();
293 $lb1->method( 'getConnection' )->willReturn( $mockDB1 );
294 $lb1->method( 'getServerCount' )->willReturn( 2 );
295 $lb1->method( 'getAnyOpenConnection' )->willReturn( $mockDB1 );
296 $lb1->method( 'hasOrMadeRecentMasterChanges' )->will( $this->returnCallback(
297 function () use ( $mockDB1 ) {
298 $p = 0;
299 $p |= call_user_func( [ $mockDB1, 'writesOrCallbacksPending' ] );
300 $p |= call_user_func( [ $mockDB1, 'lastDoneWrites' ] );
301
302 return (bool)$p;
303 }
304 ) );
305 $lb1->method( 'getMasterPos' )->willReturn( $m1Pos );
306 $lb1->method( 'getServerName' )->with( 0 )->willReturn( 'master1' );
307 // Master DB 2
308 $mockDB2 = $this->getMockBuilder( DatabaseMysqli::class )
309 ->disableOriginalConstructor()
310 ->getMock();
311 $mockDB2->method( 'writesOrCallbacksPending' )->willReturn( true );
312 $mockDB2->method( 'lastDoneWrites' )->willReturn( $now );
313 $mockDB2->method( 'getMasterPos' )->willReturn( $m2Pos );
314 // Load balancer for master DB 2
315 $lb2 = $this->getMockBuilder( LoadBalancer::class )
316 ->disableOriginalConstructor()
317 ->getMock();
318 $lb2->method( 'getConnection' )->willReturn( $mockDB2 );
319 $lb2->method( 'getServerCount' )->willReturn( 2 );
320 $lb2->method( 'getAnyOpenConnection' )->willReturn( $mockDB2 );
321 $lb2->method( 'hasOrMadeRecentMasterChanges' )->will( $this->returnCallback(
322 function () use ( $mockDB2 ) {
323 $p = 0;
324 $p |= call_user_func( [ $mockDB2, 'writesOrCallbacksPending' ] );
325 $p |= call_user_func( [ $mockDB2, 'lastDoneWrites' ] );
326
327 return (bool)$p;
328 }
329 ) );
330 $lb2->method( 'getMasterPos' )->willReturn( $m2Pos );
331 $lb2->method( 'getServerName' )->with( 0 )->willReturn( 'master2' );
332
333 $bag = new HashBagOStuff();
334 $cp = new ChronologyProtector(
335 $bag,
336 [
337 'ip' => '127.0.0.1',
338 'agent' => "Totally-Not-FireFox"
339 ]
340 );
341
342 $mockDB1->expects( $this->exactly( 1 ) )->method( 'writesOrCallbacksPending' );
343 $mockDB1->expects( $this->exactly( 1 ) )->method( 'lastDoneWrites' );
344 $mockDB2->expects( $this->exactly( 1 ) )->method( 'writesOrCallbacksPending' );
345 $mockDB2->expects( $this->exactly( 1 ) )->method( 'lastDoneWrites' );
346
347 // Nothing to wait for on first HTTP request start
348 $cp->initLB( $lb1 );
349 $cp->initLB( $lb2 );
350 // Record positions in stash on first HTTP request end
351 $cp->shutdownLB( $lb1 );
352 $cp->shutdownLB( $lb2 );
353 $cpIndex = null;
354 $cp->shutdown( null, 'sync', $cpIndex );
355
356 $this->assertEquals( 1, $cpIndex, "CP write index set" );
357
358 // (b) Second HTTP request
359
360 // Load balancer for master DB 1
361 $lb1 = $this->getMockBuilder( LoadBalancer::class )
362 ->disableOriginalConstructor()
363 ->getMock();
364 $lb1->method( 'getServerCount' )->willReturn( 2 );
365 $lb1->method( 'getServerName' )->with( 0 )->willReturn( 'master1' );
366 $lb1->expects( $this->once() )
367 ->method( 'waitFor' )->with( $this->equalTo( $m1Pos ) );
368 // Load balancer for master DB 2
369 $lb2 = $this->getMockBuilder( LoadBalancer::class )
370 ->disableOriginalConstructor()
371 ->getMock();
372 $lb2->method( 'getServerCount' )->willReturn( 2 );
373 $lb2->method( 'getServerName' )->with( 0 )->willReturn( 'master2' );
374 $lb2->expects( $this->once() )
375 ->method( 'waitFor' )->with( $this->equalTo( $m2Pos ) );
376
377 $cp = new ChronologyProtector(
378 $bag,
379 [
380 'ip' => '127.0.0.1',
381 'agent' => "Totally-Not-FireFox"
382 ],
383 $cpIndex
384 );
385
386 // Wait for last positions to be reached on second HTTP request start
387 $cp->initLB( $lb1 );
388 $cp->initLB( $lb2 );
389 // Shutdown (nothing to record)
390 $cp->shutdownLB( $lb1 );
391 $cp->shutdownLB( $lb2 );
392 $cpIndex = null;
393 $cp->shutdown( null, 'sync', $cpIndex );
394
395 $this->assertEquals( null, $cpIndex, "CP write index retained" );
396 }
397
398 private function newLBFactoryMulti( array $baseOverride = [], array $serverOverride = [] ) {
399 global $wgDBserver, $wgDBuser, $wgDBpassword, $wgDBname, $wgDBprefix, $wgDBtype;
400 global $wgSQLiteDataDir;
401
402 return new LBFactoryMulti( $baseOverride + [
403 'sectionsByDB' => [],
404 'sectionLoads' => [
405 'DEFAULT' => [
406 'test-db1' => 1,
407 ],
408 ],
409 'serverTemplate' => $serverOverride + [
410 'dbname' => $wgDBname,
411 'tablePrefix' => $wgDBprefix,
412 'user' => $wgDBuser,
413 'password' => $wgDBpassword,
414 'type' => $wgDBtype,
415 'dbDirectory' => $wgSQLiteDataDir,
416 'flags' => DBO_DEFAULT
417 ],
418 'hostsByName' => [
419 'test-db1' => $wgDBserver,
420 ],
421 'loadMonitorClass' => LoadMonitorNull::class,
422 'localDomain' => new DatabaseDomain( $wgDBname, null, $wgDBprefix )
423 ] );
424 }
425
426 public function testNiceDomains() {
427 global $wgDBname;
428
429 if ( wfGetDB( DB_MASTER )->databasesAreIndependent() ) {
430 self::markTestSkipped( "Skipping tests about selecting DBs: not applicable" );
431 return;
432 }
433
434 $factory = $this->newLBFactoryMulti(
435 [],
436 []
437 );
438 $lb = $factory->getMainLB();
439
440 $db = $lb->getConnectionRef( DB_MASTER );
441 $this->assertEquals(
442 wfWikiID(),
443 $db->getDomainID()
444 );
445 unset( $db );
446
447 /** @var Database $db */
448 $db = $lb->getConnection( DB_MASTER, [], '' );
449
450 $this->assertEquals(
451 '',
452 $db->getDomainId(),
453 'Null domain ID handle used'
454 );
455 $this->assertEquals(
456 '',
457 $db->getDBname(),
458 'Null domain ID handle used'
459 );
460 $this->assertEquals(
461 '',
462 $db->tablePrefix(),
463 'Main domain ID handle used; prefix is empty though'
464 );
465 $this->assertEquals(
466 $this->quoteTable( $db, 'page' ),
467 $db->tableName( 'page' ),
468 "Correct full table name"
469 );
470 $this->assertEquals(
471 $this->quoteTable( $db, $wgDBname ) . '.' . $this->quoteTable( $db, 'page' ),
472 $db->tableName( "$wgDBname.page" ),
473 "Correct full table name"
474 );
475 $this->assertEquals(
476 $this->quoteTable( $db, 'nice_db' ) . '.' . $this->quoteTable( $db, 'page' ),
477 $db->tableName( 'nice_db.page' ),
478 "Correct full table name"
479 );
480
481 $lb->reuseConnection( $db ); // don't care
482
483 $db = $lb->getConnection( DB_MASTER ); // local domain connection
484 $factory->setDomainPrefix( 'my_' );
485
486 $this->assertEquals( $wgDBname, $db->getDBname() );
487 $this->assertEquals(
488 "$wgDBname-my_",
489 $db->getDomainID()
490 );
491 $this->assertEquals(
492 $this->quoteTable( $db, 'my_page' ),
493 $db->tableName( 'page' ),
494 "Correct full table name"
495 );
496 $this->assertEquals(
497 $this->quoteTable( $db, 'other_nice_db' ) . '.' . $this->quoteTable( $db, 'page' ),
498 $db->tableName( 'other_nice_db.page' ),
499 "Correct full table name"
500 );
501
502 $factory->closeAll();
503 $factory->destroy();
504 }
505
506 public function testTrickyDomain() {
507 global $wgDBname;
508
509 if ( wfGetDB( DB_MASTER )->databasesAreIndependent() ) {
510 self::markTestSkipped( "Skipping tests about selecting DBs: not applicable" );
511 return;
512 }
513
514 $dbname = 'unittest-domain'; // explodes if DB is selected
515 $factory = $this->newLBFactoryMulti(
516 [ 'localDomain' => ( new DatabaseDomain( $dbname, null, '' ) )->getId() ],
517 [
518 'dbName' => 'do_not_select_me' // explodes if DB is selected
519 ]
520 );
521 $lb = $factory->getMainLB();
522 /** @var Database $db */
523 $db = $lb->getConnection( DB_MASTER, [], '' );
524
525 $this->assertEquals( '', $db->getDomainID(), "Null domain used" );
526
527 $this->assertEquals(
528 $this->quoteTable( $db, 'page' ),
529 $db->tableName( 'page' ),
530 "Correct full table name"
531 );
532
533 $this->assertEquals(
534 $this->quoteTable( $db, $dbname ) . '.' . $this->quoteTable( $db, 'page' ),
535 $db->tableName( "$dbname.page" ),
536 "Correct full table name"
537 );
538
539 $this->assertEquals(
540 $this->quoteTable( $db, 'nice_db' ) . '.' . $this->quoteTable( $db, 'page' ),
541 $db->tableName( 'nice_db.page' ),
542 "Correct full table name"
543 );
544
545 $lb->reuseConnection( $db ); // don't care
546
547 $factory->setDomainPrefix( 'my_' );
548 $db = $lb->getConnection( DB_MASTER, [], "$wgDBname-my_" );
549
550 $this->assertEquals(
551 $this->quoteTable( $db, 'my_page' ),
552 $db->tableName( 'page' ),
553 "Correct full table name"
554 );
555 $this->assertEquals(
556 $this->quoteTable( $db, 'other_nice_db' ) . '.' . $this->quoteTable( $db, 'page' ),
557 $db->tableName( 'other_nice_db.page' ),
558 "Correct full table name"
559 );
560 $this->assertEquals(
561 $this->quoteTable( $db, 'garbage-db' ) . '.' . $this->quoteTable( $db, 'page' ),
562 $db->tableName( 'garbage-db.page' ),
563 "Correct full table name"
564 );
565
566 $lb->reuseConnection( $db ); // don't care
567
568 $factory->closeAll();
569 $factory->destroy();
570 }
571
572 public function testInvalidSelectDB() {
573 $dbname = 'unittest-domain'; // explodes if DB is selected
574 $factory = $this->newLBFactoryMulti(
575 [ 'localDomain' => ( new DatabaseDomain( $dbname, null, '' ) )->getId() ],
576 [
577 'dbName' => 'do_not_select_me' // explodes if DB is selected
578 ]
579 );
580 $lb = $factory->getMainLB();
581 /** @var Database $db */
582 $db = $lb->getConnection( DB_MASTER, [], '' );
583
584 if ( $db->getType() === 'sqlite' ) {
585 $this->assertFalse( $db->selectDB( 'garbage-db' ) );
586 } elseif ( $db->databasesAreIndependent() ) {
587 try {
588 $e = null;
589 $db->selectDB( 'garbage-db' );
590 } catch ( \Wikimedia\Rdbms\DBConnectionError $e ) {
591 // expected
592 }
593 $this->assertInstanceOf( \Wikimedia\Rdbms\DBConnectionError::class, $e );
594 $this->assertFalse( $db->isOpen() );
595 } else {
596 \Wikimedia\suppressWarnings();
597 $this->assertFalse( $db->selectDB( 'garbage-db' ) );
598 \Wikimedia\restoreWarnings();
599 }
600 }
601
602 private function quoteTable( Database $db, $table ) {
603 if ( $db->getType() === 'sqlite' ) {
604 return $table;
605 } else {
606 return $db->addIdentifierQuotes( $table );
607 }
608 }
609 }