3 * New version of the difference engine
5 * Copyright © 2008 Guy Van den Broeck <guy@guyvdb.eu>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 * http://www.gnu.org/copyleft/gpl.html
23 * @ingroup DifferenceEngine
25 use MediaWiki\Diff\ComplexityException
;
27 // FIXME: Don't use assert() in this file
28 // phpcs:disable MediaWiki.Usage.ForbiddenFunctions.assert
31 * This diff implementation is mainly lifted from the LCS algorithm of the Eclipse project which
32 * in turn is based on Myers' "An O(ND) difference algorithm and its variations"
33 * (http://citeseer.ist.psu.edu/myers86ond.html) with range compression (see Wu et al.'s
34 * "An O(NP) Sequence Comparison Algorithm").
36 * This implementation supports an upper bound on the execution time.
38 * Some ideas (and a bit of code) are from analyze.c, from GNU
39 * diffutils-2.7, which can be found at:
40 * ftp://gnudist.gnu.org/pub/gnu/diffutils/diffutils-2.7.tar.gz
42 * Complexity: O((M + N)D) worst case time, O(M + N + D^2) expected time, O(M + N) space
44 * @author Guy Van den Broeck, Geoffrey T. Dairiki, Tim Starling
45 * @ingroup DifferenceEngine
60 protected $bailoutComplexity = 0;
63 private $maxDifferences;
64 private $lcsLengthCorrectedForHeuristic = false;
70 public $heuristicUsed;
72 function __construct( $tooLong = 2000000, $powLimit = 1.45 ) {
73 $this->tooLong
= $tooLong;
74 $this->powLimit
= $powLimit;
80 * @param string[] $from_lines
81 * @param string[] $to_lines
82 * @throws ComplexityException
86 public function diff( $from_lines, $to_lines ) {
87 // Diff and store locally
88 $this->diffInternal( $from_lines, $to_lines );
90 // Merge edits when possible
91 $this->shiftBoundaries( $from_lines, $this->removed
, $this->added
);
92 $this->shiftBoundaries( $to_lines, $this->added
, $this->removed
);
94 // Compute the edit operations.
95 $n_from = count( $from_lines );
96 $n_to = count( $to_lines );
100 while ( $xi < $n_from ||
$yi < $n_to ) {
101 assert( $yi < $n_to ||
$this->removed
[$xi] );
102 assert( $xi < $n_from ||
$this->added
[$yi] );
104 // Skip matching "snake".
106 while ( $xi < $n_from && $yi < $n_to
107 && !$this->removed
[$xi] && !$this->added
[$yi]
109 $copy[] = $from_lines[$xi++
];
113 $edits[] = new DiffOpCopy( $copy );
116 // Find deletes & adds.
118 while ( $xi < $n_from && $this->removed
[$xi] ) {
119 $delete[] = $from_lines[$xi++
];
123 while ( $yi < $n_to && $this->added
[$yi] ) {
124 $add[] = $to_lines[$yi++
];
127 if ( $delete && $add ) {
128 $edits[] = new DiffOpChange( $delete, $add );
129 } elseif ( $delete ) {
130 $edits[] = new DiffOpDelete( $delete );
132 $edits[] = new DiffOpAdd( $add );
140 * Sets the complexity (in comparison operations) that can't be exceeded
143 public function setBailoutComplexity( $value ) {
144 $this->bailoutComplexity
= $value;
148 * Adjust inserts/deletes of identical lines to join changes
149 * as much as possible.
151 * We do something when a run of changed lines include a
152 * line at one end and has an excluded, identical line at the other.
153 * We are free to choose which identical line is included.
154 * `compareseq' usually chooses the one at the beginning,
155 * but usually it is cleaner to consider the following identical line
156 * to be the "change".
158 * This is extracted verbatim from analyze.c (GNU diffutils-2.7).
160 * @param string[] $lines
161 * @param string[] $changed
162 * @param string[] $other_changed
164 private function shiftBoundaries( array $lines, array &$changed, array $other_changed ) {
168 assert( count( $lines ) == count( $changed ) );
169 $len = count( $lines );
170 $other_len = count( $other_changed );
174 * Scan forwards to find beginning of another run of changes.
175 * Also keep track of the corresponding point in the other file.
177 * Throughout this code, $i and $j are adjusted together so that
178 * the first $i elements of $changed and the first $j elements
179 * of $other_changed both contain the same number of zeros
181 * Furthermore, $j is always kept so that $j == $other_len or
182 * $other_changed[$j] == false.
184 while ( $j < $other_len && $other_changed[$j] ) {
188 while ( $i < $len && !$changed[$i] ) {
189 assert( $j < $other_len && !$other_changed[$j] );
192 while ( $j < $other_len && $other_changed[$j] ) {
203 // Find the end of this run of changes.
204 while ( ++
$i < $len && $changed[$i] ) {
210 * Record the length of this run of changes, so that
211 * we can later determine whether the run has grown.
213 $runlength = $i - $start;
216 * Move the changed region back, so long as the
217 * previous unchanged line matches the last changed one.
218 * This merges with previous changed regions.
220 while ( $start > 0 && $lines[$start - 1] == $lines[$i - 1] ) {
221 $changed[--$start] = 1;
222 $changed[--$i] = false;
223 while ( $start > 0 && $changed[$start - 1] ) {
227 while ( $other_changed[--$j] ) {
230 assert( $j >= 0 && !$other_changed[$j] );
234 * Set CORRESPONDING to the end of the changed run, at the last
235 * point where it corresponds to a changed run in the other file.
236 * CORRESPONDING == LEN means no such point has been found.
238 $corresponding = $j < $other_len ?
$i : $len;
241 * Move the changed region forward, so long as the
242 * first changed line matches the following unchanged one.
243 * This merges with following changed regions.
244 * Do this second, so that if there are no merges,
245 * the changed region is moved forward as far as possible.
247 while ( $i < $len && $lines[$start] == $lines[$i] ) {
248 $changed[$start++
] = false;
250 while ( $i < $len && $changed[$i] ) {
254 assert( $j < $other_len && !$other_changed[$j] );
256 if ( $j < $other_len && $other_changed[$j] ) {
258 while ( $j < $other_len && $other_changed[$j] ) {
263 } while ( $runlength != $i - $start );
266 * If possible, move the fully-merged run of changes
267 * back to a corresponding run in the other file.
269 while ( $corresponding < $i ) {
270 $changed[--$start] = 1;
273 while ( $other_changed[--$j] ) {
276 assert( $j >= 0 && !$other_changed[$j] );
282 * @param string[] $from
283 * @param string[] $to
284 * @throws ComplexityException
286 protected function diffInternal( array $from, array $to ) {
287 // remember initial lengths
291 $this->heuristicUsed
= false;
294 $removed = $m > 0 ?
array_fill( 0, $m, true ) : [];
295 $added = $n > 0 ?
array_fill( 0, $n, true ) : [];
297 // reduce the complexity for the next step (intentionally done twice)
298 // remove common tokens at the start
300 while ( $i < $m && $i < $n && $from[$i] === $to[$i] ) {
301 $removed[$i] = $added[$i] = false;
302 unset( $from[$i], $to[$i] );
306 // remove common tokens at the end
308 while ( $i +
$j <= $m && $i +
$j <= $n && $from[$m - $j] === $to[$n - $j] ) {
309 $removed[$m - $j] = $added[$n - $j] = false;
310 unset( $from[$m - $j], $to[$n - $j] );
314 $this->from
= $newFromIndex = $this->to
= $newToIndex = [];
316 // remove tokens not in both sequences
318 foreach ( $from as $key ) {
319 $shared[$key] = false;
322 foreach ( $to as $index => &$el ) {
323 if ( array_key_exists( $el, $shared ) ) {
327 $newToIndex[] = $index;
330 foreach ( $from as $index => &$el ) {
331 if ( $shared[$el] ) {
334 $newFromIndex[] = $index;
338 unset( $shared, $from, $to );
340 $this->m
= count( $this->from
);
341 $this->n
= count( $this->to
);
343 if ( $this->bailoutComplexity
> 0 && $this->m
* $this->n
> $this->bailoutComplexity
) {
344 throw new ComplexityException();
347 $this->removed
= $this->m
> 0 ?
array_fill( 0, $this->m
, true ) : [];
348 $this->added
= $this->n
> 0 ?
array_fill( 0, $this->n
, true ) : [];
350 if ( $this->m
== 0 ||
$this->n
== 0 ) {
353 $this->maxDifferences
= ceil( ( $this->m +
$this->n
) / 2.0 );
354 if ( $this->m
* $this->n
> $this->tooLong
) {
355 // limit complexity to D^POW_LIMIT for long sequences
356 $this->maxDifferences
= floor( $this->maxDifferences
** ( $this->powLimit
- 1.0 ) );
357 wfDebug( "Limiting max number of differences to $this->maxDifferences\n" );
361 * The common prefixes and suffixes are always part of some LCS, include
362 * them now to reduce our search space
364 $max = min( $this->m
, $this->n
);
365 for ( $forwardBound = 0; $forwardBound < $max
366 // @phan-suppress-next-line PhanTypeInvalidDimOffset
367 && $this->from
[$forwardBound] === $this->to
[$forwardBound];
370 $this->removed
[$forwardBound] = $this->added
[$forwardBound] = false;
373 $backBoundL1 = $this->m
- 1;
374 $backBoundL2 = $this->n
- 1;
376 while ( $backBoundL1 >= $forwardBound && $backBoundL2 >= $forwardBound
377 && $this->from
[$backBoundL1] === $this->to
[$backBoundL2]
379 $this->removed
[$backBoundL1--] = $this->added
[$backBoundL2--] = false;
382 $temp = array_fill( 0, $this->m +
$this->n +
1, 0 );
383 $V = [ $temp, $temp ];
384 $snake = [ 0, 0, 0 ];
386 $this->length
= $forwardBound +
$this->m
- $backBoundL1 - 1
400 $this->length +
= $i +
$j - 1;
402 foreach ( $this->removed
as $key => &$removed_elem ) {
403 if ( !$removed_elem ) {
404 $removed[$newFromIndex[$key]] = false;
407 foreach ( $this->added
as $key => &$added_elem ) {
408 if ( !$added_elem ) {
409 $added[$newToIndex[$key]] = false;
412 $this->removed
= $removed;
413 $this->added
= $added;
416 function diff_range( $from_lines, $to_lines ) {
417 // Diff and store locally
418 $this->diff( $from_lines, $to_lines );
419 unset( $from_lines, $to_lines );
423 while ( $xi < $this->m ||
$yi < $this->n
) {
425 while ( $xi < $this->m
&& $yi < $this->n
426 && !$this->removed
[$xi]
427 && !$this->added
[$yi]
432 // Find deletes & adds.
434 while ( $xi < $this->m
&& $this->removed
[$xi] ) {
439 while ( $yi < $this->n
&& $this->added
[$yi] ) {
443 if ( $xi > $xstart ||
$yi > $ystart ) {
444 $ranges[] = new RangeDifference( $xstart, $xi, $ystart, $yi );
451 private function lcs_rec( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
452 // check that both sequences are non-empty
453 if ( $bottoml1 > $topl1 ||
$bottoml2 > $topl2 ) {
457 $d = $this->find_middle_snake( $bottoml1, $topl1, $bottoml2,
458 $topl2, $V, $snake );
460 // need to store these so we don't lose them when they're
461 // overwritten by the recursion
462 list( $startx, $starty, $len ) = $snake;
464 // the middle snake is part of the LCS, store it
465 for ( $i = 0; $i < $len; ++
$i ) {
466 $this->removed
[$startx +
$i] = $this->added
[$starty +
$i] = false;
471 +
$this->lcs_rec( $bottoml1, $startx - 1, $bottoml2,
472 $starty - 1, $V, $snake )
473 +
$this->lcs_rec( $startx +
$len, $topl1, $starty +
$len,
474 $topl2, $V, $snake );
475 } elseif ( $d == 1 ) {
477 * In this case the sequences differ by exactly 1 line. We have
478 * already saved all the lines after the difference in the for loop
479 * above, now we need to save all the lines before the difference.
481 $max = min( $startx - $bottoml1, $starty - $bottoml2 );
482 for ( $i = 0; $i < $max; ++
$i ) {
483 $this->removed
[$bottoml1 +
$i] =
484 $this->added
[$bottoml2 +
$i] = false;
493 private function find_middle_snake( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
494 $from = &$this->from
;
498 $snake0 = &$snake[0];
499 $snake1 = &$snake[1];
500 $snake2 = &$snake[2];
501 $bottoml1_min_1 = $bottoml1 - 1;
502 $bottoml2_min_1 = $bottoml2 - 1;
503 $N = $topl1 - $bottoml1_min_1;
504 $M = $topl2 - $bottoml2_min_1;
506 $maxabsx = $N +
$bottoml1;
507 $maxabsy = $M +
$bottoml2;
508 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
510 // value_to_add_forward: a 0 or 1 that we add to the start
511 // offset to make it odd/even
513 $value_to_add_forward = 1;
515 $value_to_add_forward = 0;
519 $value_to_add_backward = 1;
521 $value_to_add_backward = 0;
524 $start_forward = -$M;
526 $start_backward = -$N;
529 $limit_min_1 = $limit - 1;
530 $limit_plus_1 = $limit +
1;
532 $V0[$limit_plus_1] = 0;
533 $V1[$limit_min_1] = $N;
534 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
537 for ( $d = 0; $d <= $limit; ++
$d ) {
538 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
539 $end_diag = min( $end_forward, $d );
540 $value_to_add_forward = 1 - $value_to_add_forward;
542 // compute forward furthest reaching paths
543 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
544 if ( $k == -$d ||
( $k < $d
545 && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
547 $x = $V0[$limit_plus_1 +
$k];
549 $x = $V0[$limit_min_1 +
$k] +
1;
552 $absx = $snake0 = $x +
$bottoml1;
553 $absy = $snake1 = $x - $k +
$bottoml2;
555 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
559 $x = $absx - $bottoml1;
561 $snake2 = $absx - $snake0;
562 $V0[$limit +
$k] = $x;
563 if ( $k >= $delta - $d +
1 && $k <= $delta +
$d - 1
564 && $x >= $V1[$limit +
$k - $delta]
569 // check to see if we can cut down the diagonal range
570 if ( $x >= $N && $end_forward > $k - 1 ) {
571 $end_forward = $k - 1;
572 } elseif ( $absy - $bottoml2 >= $M ) {
573 $start_forward = $k +
1;
574 $value_to_add_forward = 0;
578 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
579 $end_diag = min( $end_backward, $d );
580 $value_to_add_backward = 1 - $value_to_add_backward;
582 // compute backward furthest reaching paths
583 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
585 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
587 $x = $V1[$limit_min_1 +
$k];
589 $x = $V1[$limit_plus_1 +
$k] - 1;
592 $y = $x - $k - $delta;
595 while ( $x > 0 && $y > 0
596 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
602 $V1[$limit +
$k] = $x;
604 // check to see if we can cut down our diagonal range
606 $start_backward = $k +
1;
607 $value_to_add_backward = 0;
608 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
609 $end_backward = $k - 1;
614 for ( $d = 0; $d <= $limit; ++
$d ) {
615 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
616 $end_diag = min( $end_forward, $d );
617 $value_to_add_forward = 1 - $value_to_add_forward;
619 // compute forward furthest reaching paths
620 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
622 ||
( $k < $d && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
624 $x = $V0[$limit_plus_1 +
$k];
626 $x = $V0[$limit_min_1 +
$k] +
1;
629 $absx = $snake0 = $x +
$bottoml1;
630 $absy = $snake1 = $x - $k +
$bottoml2;
632 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
636 $x = $absx - $bottoml1;
637 $snake2 = $absx - $snake0;
638 $V0[$limit +
$k] = $x;
640 // check to see if we can cut down the diagonal range
641 if ( $x >= $N && $end_forward > $k - 1 ) {
642 $end_forward = $k - 1;
643 } elseif ( $absy - $bottoml2 >= $M ) {
644 $start_forward = $k +
1;
645 $value_to_add_forward = 0;
649 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
650 $end_diag = min( $end_backward, $d );
651 $value_to_add_backward = 1 - $value_to_add_backward;
653 // compute backward furthest reaching paths
654 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
656 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
658 $x = $V1[$limit_min_1 +
$k];
660 $x = $V1[$limit_plus_1 +
$k] - 1;
663 $y = $x - $k - $delta;
666 while ( $x > 0 && $y > 0
667 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
673 $V1[$limit +
$k] = $x;
675 if ( $k >= -$delta - $d && $k <= $d - $delta
676 && $x <= $V0[$limit +
$k +
$delta]
678 $snake0 = $bottoml1 +
$x;
679 $snake1 = $bottoml2 +
$y;
684 // check to see if we can cut down our diagonal range
686 $start_backward = $k +
1;
687 $value_to_add_backward = 0;
688 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
689 $end_backward = $k - 1;
695 * computing the true LCS is too expensive, instead find the diagonal
696 * with the most progress and pretend a midle snake of length 0 occurs
700 $most_progress = self
::findMostProgress( $M, $N, $limit, $V );
702 $snake0 = $bottoml1 +
$most_progress[0];
703 $snake1 = $bottoml2 +
$most_progress[1];
705 wfDebug( "Computing the LCS is too expensive. Using a heuristic.\n" );
706 $this->heuristicUsed
= true;
709 * HACK: since we didn't really finish the LCS computation
710 * we don't really know the length of the SES. We don't do
711 * anything with the result anyway, unless it's <=1. We know
712 * for a fact SES > 1 so 5 is as good a number as any to
717 private static function findMostProgress( $M, $N, $limit, $V ) {
720 if ( ( $M & 1 ) == ( $limit & 1 ) ) {
721 $forward_start_diag = max( -$M, -$limit );
723 $forward_start_diag = max( 1 - $M, -$limit );
726 $forward_end_diag = min( $N, $limit );
728 if ( ( $N & 1 ) == ( $limit & 1 ) ) {
729 $backward_start_diag = max( -$N, -$limit );
731 $backward_start_diag = max( 1 - $N, -$limit );
734 $backward_end_diag = -min( $M, $limit );
738 $max_progress = array_fill( 0, ceil( max( $forward_end_diag - $forward_start_diag,
739 $backward_end_diag - $backward_start_diag ) / 2 ), $temp );
740 $num_progress = 0; // the 1st entry is current, it is initialized
743 // first search the forward diagonals
744 for ( $k = $forward_start_diag; $k <= $forward_end_diag; $k +
= 2 ) {
745 $x = $V[0][$limit +
$k];
747 if ( $x > $N ||
$y > $M ) {
752 if ( $progress > $max_progress[0][2] ) {
754 $max_progress[0][0] = $x;
755 $max_progress[0][1] = $y;
756 $max_progress[0][2] = $progress;
757 } elseif ( $progress == $max_progress[0][2] ) {
759 $max_progress[$num_progress][0] = $x;
760 $max_progress[$num_progress][1] = $y;
761 $max_progress[$num_progress][2] = $progress;
765 $max_progress_forward = true; // initially the maximum
766 // progress is in the forward
769 // now search the backward diagonals
770 for ( $k = $backward_start_diag; $k <= $backward_end_diag; $k +
= 2 ) {
771 $x = $V[1][$limit +
$k];
772 $y = $x - $k - $delta;
773 if ( $x < 0 ||
$y < 0 ) {
777 $progress = $N - $x +
$M - $y;
778 if ( $progress > $max_progress[0][2] ) {
780 $max_progress_forward = false;
781 $max_progress[0][0] = $x;
782 $max_progress[0][1] = $y;
783 $max_progress[0][2] = $progress;
784 } elseif ( $progress == $max_progress[0][2] && !$max_progress_forward ) {
786 $max_progress[$num_progress][0] = $x;
787 $max_progress[$num_progress][1] = $y;
788 $max_progress[$num_progress][2] = $progress;
792 // return the middle diagonal with maximal progress.
793 return $max_progress[(int)floor( $num_progress / 2 )];
799 public function getLcsLength() {
800 if ( $this->heuristicUsed
&& !$this->lcsLengthCorrectedForHeuristic
) {
801 $this->lcsLengthCorrectedForHeuristic
= true;
802 $this->length
= $this->m
- array_sum( $this->added
);
805 return $this->length
;