3 * New version of the difference engine
5 * Copyright © 2008 Guy Van den Broeck <guy@guyvdb.eu>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 * http://www.gnu.org/copyleft/gpl.html
23 * @ingroup DifferenceEngine
27 * This diff implementation is mainly lifted from the LCS algorithm of the Eclipse project which
28 * in turn is based on Myers' "An O(ND) difference algorithm and its variations"
29 * (http://citeseer.ist.psu.edu/myers86ond.html) with range compression (see Wu et al.'s
30 * "An O(NP) Sequence Comparison Algorithm").
32 * This implementation supports an upper bound on the execution time.
34 * Some ideas (and a bit of code) are from analyze.c, from GNU
35 * diffutils-2.7, which can be found at:
36 * ftp://gnudist.gnu.org/pub/gnu/diffutils/diffutils-2.7.tar.gz
38 * Complexity: O((M + N)D) worst case time, O(M + N + D^2) expected time, O(M + N) space
40 * @author Guy Van den Broeck, Geoffrey T. Dairiki, Tim Starling
41 * @ingroup DifferenceEngine
55 private $maxDifferences;
56 private $lcsLengthCorrectedForHeuristic = false;
62 public $heuristicUsed;
64 function __construct( $tooLong = 2000000, $powLimit = 1.45 ) {
65 $this->tooLong
= $tooLong;
66 $this->powLimit
= $powLimit;
72 * @param string[] $from_lines
73 * @param string[] $to_lines
77 public function diff( $from_lines, $to_lines ) {
79 // Diff and store locally
80 $this->diffInternal( $from_lines, $to_lines );
82 // Merge edits when possible
83 $this->shiftBoundaries( $from_lines, $this->removed
, $this->added
);
84 $this->shiftBoundaries( $to_lines, $this->added
, $this->removed
);
86 // Compute the edit operations.
87 $n_from = count( $from_lines );
88 $n_to = count( $to_lines );
92 while ( $xi < $n_from ||
$yi < $n_to ) {
93 assert( $yi < $n_to ||
$this->removed
[$xi] );
94 assert( $xi < $n_from ||
$this->added
[$yi] );
96 // Skip matching "snake".
98 while ( $xi < $n_from && $yi < $n_to
99 && !$this->removed
[$xi] && !$this->added
[$yi]
101 $copy[] = $from_lines[$xi++
];
105 $edits[] = new DiffOpCopy( $copy );
108 // Find deletes & adds.
110 while ( $xi < $n_from && $this->removed
[$xi] ) {
111 $delete[] = $from_lines[$xi++
];
115 while ( $yi < $n_to && $this->added
[$yi] ) {
116 $add[] = $to_lines[$yi++
];
119 if ( $delete && $add ) {
120 $edits[] = new DiffOpChange( $delete, $add );
121 } elseif ( $delete ) {
122 $edits[] = new DiffOpDelete( $delete );
124 $edits[] = new DiffOpAdd( $add );
132 * Adjust inserts/deletes of identical lines to join changes
133 * as much as possible.
135 * We do something when a run of changed lines include a
136 * line at one end and has an excluded, identical line at the other.
137 * We are free to choose which identical line is included.
138 * `compareseq' usually chooses the one at the beginning,
139 * but usually it is cleaner to consider the following identical line
140 * to be the "change".
142 * This is extracted verbatim from analyze.c (GNU diffutils-2.7).
144 * @param string[] $lines
145 * @param string[] $changed
146 * @param string[] $other_changed
148 private function shiftBoundaries( array $lines, array &$changed, array $other_changed ) {
152 assert( count( $lines ) == count( $changed ) );
153 $len = count( $lines );
154 $other_len = count( $other_changed );
158 * Scan forwards to find beginning of another run of changes.
159 * Also keep track of the corresponding point in the other file.
161 * Throughout this code, $i and $j are adjusted together so that
162 * the first $i elements of $changed and the first $j elements
163 * of $other_changed both contain the same number of zeros
165 * Furthermore, $j is always kept so that $j == $other_len or
166 * $other_changed[$j] == false.
168 while ( $j < $other_len && $other_changed[$j] ) {
172 while ( $i < $len && !$changed[$i] ) {
173 assert( $j < $other_len && ! $other_changed[$j] );
176 while ( $j < $other_len && $other_changed[$j] ) {
187 // Find the end of this run of changes.
188 while ( ++
$i < $len && $changed[$i] ) {
194 * Record the length of this run of changes, so that
195 * we can later determine whether the run has grown.
197 $runlength = $i - $start;
200 * Move the changed region back, so long as the
201 * previous unchanged line matches the last changed one.
202 * This merges with previous changed regions.
204 while ( $start > 0 && $lines[$start - 1] == $lines[$i - 1] ) {
205 $changed[--$start] = 1;
206 $changed[--$i] = false;
207 while ( $start > 0 && $changed[$start - 1] ) {
211 while ( $other_changed[--$j] ) {
214 assert( $j >= 0 && !$other_changed[$j] );
218 * Set CORRESPONDING to the end of the changed run, at the last
219 * point where it corresponds to a changed run in the other file.
220 * CORRESPONDING == LEN means no such point has been found.
222 $corresponding = $j < $other_len ?
$i : $len;
225 * Move the changed region forward, so long as the
226 * first changed line matches the following unchanged one.
227 * This merges with following changed regions.
228 * Do this second, so that if there are no merges,
229 * the changed region is moved forward as far as possible.
231 while ( $i < $len && $lines[$start] == $lines[$i] ) {
232 $changed[$start++
] = false;
234 while ( $i < $len && $changed[$i] ) {
238 assert( $j < $other_len && ! $other_changed[$j] );
240 if ( $j < $other_len && $other_changed[$j] ) {
242 while ( $j < $other_len && $other_changed[$j] ) {
247 } while ( $runlength != $i - $start );
250 * If possible, move the fully-merged run of changes
251 * back to a corresponding run in the other file.
253 while ( $corresponding < $i ) {
254 $changed[--$start] = 1;
257 while ( $other_changed[--$j] ) {
260 assert( $j >= 0 && !$other_changed[$j] );
266 * @param string[] $from
267 * @param string[] $to
269 protected function diffInternal( array $from, array $to ) {
270 // remember initial lengths
274 $this->heuristicUsed
= false;
277 $removed = $m > 0 ?
array_fill( 0, $m, true ) : [];
278 $added = $n > 0 ?
array_fill( 0, $n, true ) : [];
280 // reduce the complexity for the next step (intentionally done twice)
281 // remove common tokens at the start
283 while ( $i < $m && $i < $n && $from[$i] === $to[$i] ) {
284 $removed[$i] = $added[$i] = false;
285 unset( $from[$i], $to[$i] );
289 // remove common tokens at the end
291 while ( $i +
$j <= $m && $i +
$j <= $n && $from[$m - $j] === $to[$n - $j] ) {
292 $removed[$m - $j] = $added[$n - $j] = false;
293 unset( $from[$m - $j], $to[$n - $j] );
297 $this->from
= $newFromIndex = $this->to
= $newToIndex = [];
299 // remove tokens not in both sequences
301 foreach ( $from as $key ) {
302 $shared[$key] = false;
305 foreach ( $to as $index => &$el ) {
306 if ( array_key_exists( $el, $shared ) ) {
310 $newToIndex[] = $index;
313 foreach ( $from as $index => &$el ) {
314 if ( $shared[$el] ) {
317 $newFromIndex[] = $index;
321 unset( $shared, $from, $to );
323 $this->m
= count( $this->from
);
324 $this->n
= count( $this->to
);
326 $this->removed
= $this->m
> 0 ?
array_fill( 0, $this->m
, true ) : [];
327 $this->added
= $this->n
> 0 ?
array_fill( 0, $this->n
, true ) : [];
329 if ( $this->m
== 0 ||
$this->n
== 0 ) {
332 $this->maxDifferences
= ceil( ( $this->m +
$this->n
) / 2.0 );
333 if ( $this->m
* $this->n
> $this->tooLong
) {
334 // limit complexity to D^POW_LIMIT for long sequences
335 $this->maxDifferences
= floor( pow( $this->maxDifferences
, $this->powLimit
- 1.0 ) );
336 wfDebug( "Limiting max number of differences to $this->maxDifferences\n" );
340 * The common prefixes and suffixes are always part of some LCS, include
341 * them now to reduce our search space
343 $max = min( $this->m
, $this->n
);
344 for ( $forwardBound = 0; $forwardBound < $max
345 && $this->from
[$forwardBound] === $this->to
[$forwardBound];
348 $this->removed
[$forwardBound] = $this->added
[$forwardBound] = false;
351 $backBoundL1 = $this->m
- 1;
352 $backBoundL2 = $this->n
- 1;
354 while ( $backBoundL1 >= $forwardBound && $backBoundL2 >= $forwardBound
355 && $this->from
[$backBoundL1] === $this->to
[$backBoundL2]
357 $this->removed
[$backBoundL1--] = $this->added
[$backBoundL2--] = false;
360 $temp = array_fill( 0, $this->m +
$this->n +
1, 0 );
361 $V = [ $temp, $temp ];
362 $snake = [ 0, 0, 0 ];
364 $this->length
= $forwardBound +
$this->m
- $backBoundL1 - 1
378 $this->length +
= $i +
$j - 1;
380 foreach ( $this->removed
as $key => &$removed_elem ) {
381 if ( !$removed_elem ) {
382 $removed[$newFromIndex[$key]] = false;
385 foreach ( $this->added
as $key => &$added_elem ) {
386 if ( !$added_elem ) {
387 $added[$newToIndex[$key]] = false;
390 $this->removed
= $removed;
391 $this->added
= $added;
394 function diff_range( $from_lines, $to_lines ) {
395 // Diff and store locally
396 $this->diff( $from_lines, $to_lines );
397 unset( $from_lines, $to_lines );
401 while ( $xi < $this->m ||
$yi < $this->n
) {
403 while ( $xi < $this->m
&& $yi < $this->n
404 && !$this->removed
[$xi]
405 && !$this->added
[$yi]
410 // Find deletes & adds.
412 while ( $xi < $this->m
&& $this->removed
[$xi] ) {
417 while ( $yi < $this->n
&& $this->added
[$yi] ) {
421 if ( $xi > $xstart ||
$yi > $ystart ) {
422 $ranges[] = new RangeDifference( $xstart, $xi, $ystart, $yi );
429 private function lcs_rec( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
430 // check that both sequences are non-empty
431 if ( $bottoml1 > $topl1 ||
$bottoml2 > $topl2 ) {
435 $d = $this->find_middle_snake( $bottoml1, $topl1, $bottoml2,
436 $topl2, $V, $snake );
438 // need to store these so we don't lose them when they're
439 // overwritten by the recursion
444 // the middle snake is part of the LCS, store it
445 for ( $i = 0; $i < $len; ++
$i ) {
446 $this->removed
[$startx +
$i] = $this->added
[$starty +
$i] = false;
451 +
$this->lcs_rec( $bottoml1, $startx - 1, $bottoml2,
452 $starty - 1, $V, $snake )
453 +
$this->lcs_rec( $startx +
$len, $topl1, $starty +
$len,
454 $topl2, $V, $snake );
455 } elseif ( $d == 1 ) {
457 * In this case the sequences differ by exactly 1 line. We have
458 * already saved all the lines after the difference in the for loop
459 * above, now we need to save all the lines before the difference.
461 $max = min( $startx - $bottoml1, $starty - $bottoml2 );
462 for ( $i = 0; $i < $max; ++
$i ) {
463 $this->removed
[$bottoml1 +
$i] =
464 $this->added
[$bottoml2 +
$i] = false;
473 private function find_middle_snake( $bottoml1, $topl1, $bottoml2, $topl2, &$V, &$snake ) {
474 $from = &$this->from
;
478 $snake0 = &$snake[0];
479 $snake1 = &$snake[1];
480 $snake2 = &$snake[2];
481 $bottoml1_min_1 = $bottoml1 - 1;
482 $bottoml2_min_1 = $bottoml2 - 1;
483 $N = $topl1 - $bottoml1_min_1;
484 $M = $topl2 - $bottoml2_min_1;
486 $maxabsx = $N +
$bottoml1;
487 $maxabsy = $M +
$bottoml2;
488 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
490 // value_to_add_forward: a 0 or 1 that we add to the start
491 // offset to make it odd/even
492 if ( ( $M & 1 ) == 1 ) {
493 $value_to_add_forward = 1;
495 $value_to_add_forward = 0;
498 if ( ( $N & 1 ) == 1 ) {
499 $value_to_add_backward = 1;
501 $value_to_add_backward = 0;
504 $start_forward = -$M;
506 $start_backward = -$N;
509 $limit_min_1 = $limit - 1;
510 $limit_plus_1 = $limit +
1;
512 $V0[$limit_plus_1] = 0;
513 $V1[$limit_min_1] = $N;
514 $limit = min( $this->maxDifferences
, ceil( ( $N +
$M ) / 2 ) );
516 if ( ( $delta & 1 ) == 1 ) {
517 for ( $d = 0; $d <= $limit; ++
$d ) {
518 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
519 $end_diag = min( $end_forward, $d );
520 $value_to_add_forward = 1 - $value_to_add_forward;
522 // compute forward furthest reaching paths
523 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
524 if ( $k == -$d ||
( $k < $d
525 && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
527 $x = $V0[$limit_plus_1 +
$k];
529 $x = $V0[$limit_min_1 +
$k] +
1;
532 $absx = $snake0 = $x +
$bottoml1;
533 $absy = $snake1 = $x - $k +
$bottoml2;
535 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
539 $x = $absx - $bottoml1;
541 $snake2 = $absx - $snake0;
542 $V0[$limit +
$k] = $x;
543 if ( $k >= $delta - $d +
1 && $k <= $delta +
$d - 1
544 && $x >= $V1[$limit +
$k - $delta]
549 // check to see if we can cut down the diagonal range
550 if ( $x >= $N && $end_forward > $k - 1 ) {
551 $end_forward = $k - 1;
552 } elseif ( $absy - $bottoml2 >= $M ) {
553 $start_forward = $k +
1;
554 $value_to_add_forward = 0;
558 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
559 $end_diag = min( $end_backward, $d );
560 $value_to_add_backward = 1 - $value_to_add_backward;
562 // compute backward furthest reaching paths
563 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
565 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
567 $x = $V1[$limit_min_1 +
$k];
569 $x = $V1[$limit_plus_1 +
$k] - 1;
572 $y = $x - $k - $delta;
575 while ( $x > 0 && $y > 0
576 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
582 $V1[$limit +
$k] = $x;
584 // check to see if we can cut down our diagonal range
586 $start_backward = $k +
1;
587 $value_to_add_backward = 0;
588 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
589 $end_backward = $k - 1;
594 for ( $d = 0; $d <= $limit; ++
$d ) {
595 $start_diag = max( $value_to_add_forward +
$start_forward, -$d );
596 $end_diag = min( $end_forward, $d );
597 $value_to_add_forward = 1 - $value_to_add_forward;
599 // compute forward furthest reaching paths
600 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
602 ||
( $k < $d && $V0[$limit_min_1 +
$k] < $V0[$limit_plus_1 +
$k] )
604 $x = $V0[$limit_plus_1 +
$k];
606 $x = $V0[$limit_min_1 +
$k] +
1;
609 $absx = $snake0 = $x +
$bottoml1;
610 $absy = $snake1 = $x - $k +
$bottoml2;
612 while ( $absx < $maxabsx && $absy < $maxabsy && $from[$absx] === $to[$absy] ) {
616 $x = $absx - $bottoml1;
617 $snake2 = $absx - $snake0;
618 $V0[$limit +
$k] = $x;
620 // check to see if we can cut down the diagonal range
621 if ( $x >= $N && $end_forward > $k - 1 ) {
622 $end_forward = $k - 1;
623 } elseif ( $absy - $bottoml2 >= $M ) {
624 $start_forward = $k +
1;
625 $value_to_add_forward = 0;
629 $start_diag = max( $value_to_add_backward +
$start_backward, -$d );
630 $end_diag = min( $end_backward, $d );
631 $value_to_add_backward = 1 - $value_to_add_backward;
633 // compute backward furthest reaching paths
634 for ( $k = $start_diag; $k <= $end_diag; $k +
= 2 ) {
636 ||
( $k != -$d && $V1[$limit_min_1 +
$k] < $V1[$limit_plus_1 +
$k] )
638 $x = $V1[$limit_min_1 +
$k];
640 $x = $V1[$limit_plus_1 +
$k] - 1;
643 $y = $x - $k - $delta;
646 while ( $x > 0 && $y > 0
647 && $from[$x +
$bottoml1_min_1] === $to[$y +
$bottoml2_min_1]
653 $V1[$limit +
$k] = $x;
655 if ( $k >= -$delta - $d && $k <= $d - $delta
656 && $x <= $V0[$limit +
$k +
$delta]
658 $snake0 = $bottoml1 +
$x;
659 $snake1 = $bottoml2 +
$y;
664 // check to see if we can cut down our diagonal range
666 $start_backward = $k +
1;
667 $value_to_add_backward = 0;
668 } elseif ( $y <= 0 && $end_backward > $k - 1 ) {
669 $end_backward = $k - 1;
675 * computing the true LCS is too expensive, instead find the diagonal
676 * with the most progress and pretend a midle snake of length 0 occurs
680 $most_progress = self
::findMostProgress( $M, $N, $limit, $V );
682 $snake0 = $bottoml1 +
$most_progress[0];
683 $snake1 = $bottoml2 +
$most_progress[1];
685 wfDebug( "Computing the LCS is too expensive. Using a heuristic.\n" );
686 $this->heuristicUsed
= true;
689 * HACK: since we didn't really finish the LCS computation
690 * we don't really know the length of the SES. We don't do
691 * anything with the result anyway, unless it's <=1. We know
692 * for a fact SES > 1 so 5 is as good a number as any to
697 private static function findMostProgress( $M, $N, $limit, $V ) {
700 if ( ( $M & 1 ) == ( $limit & 1 ) ) {
701 $forward_start_diag = max( -$M, -$limit );
703 $forward_start_diag = max( 1 - $M, -$limit );
706 $forward_end_diag = min( $N, $limit );
708 if ( ( $N & 1 ) == ( $limit & 1 ) ) {
709 $backward_start_diag = max( -$N, -$limit );
711 $backward_start_diag = max( 1 - $N, -$limit );
714 $backward_end_diag = -min( $M, $limit );
718 $max_progress = array_fill( 0, ceil( max( $forward_end_diag - $forward_start_diag,
719 $backward_end_diag - $backward_start_diag ) / 2 ), $temp );
720 $num_progress = 0; // the 1st entry is current, it is initialized
723 // first search the forward diagonals
724 for ( $k = $forward_start_diag; $k <= $forward_end_diag; $k +
= 2 ) {
725 $x = $V[0][$limit +
$k];
727 if ( $x > $N ||
$y > $M ) {
732 if ( $progress > $max_progress[0][2] ) {
734 $max_progress[0][0] = $x;
735 $max_progress[0][1] = $y;
736 $max_progress[0][2] = $progress;
737 } elseif ( $progress == $max_progress[0][2] ) {
739 $max_progress[$num_progress][0] = $x;
740 $max_progress[$num_progress][1] = $y;
741 $max_progress[$num_progress][2] = $progress;
745 $max_progress_forward = true; // initially the maximum
746 // progress is in the forward
749 // now search the backward diagonals
750 for ( $k = $backward_start_diag; $k <= $backward_end_diag; $k +
= 2 ) {
751 $x = $V[1][$limit +
$k];
752 $y = $x - $k - $delta;
753 if ( $x < 0 ||
$y < 0 ) {
757 $progress = $N - $x +
$M - $y;
758 if ( $progress > $max_progress[0][2] ) {
760 $max_progress_forward = false;
761 $max_progress[0][0] = $x;
762 $max_progress[0][1] = $y;
763 $max_progress[0][2] = $progress;
764 } elseif ( $progress == $max_progress[0][2] && !$max_progress_forward ) {
766 $max_progress[$num_progress][0] = $x;
767 $max_progress[$num_progress][1] = $y;
768 $max_progress[$num_progress][2] = $progress;
772 // return the middle diagonal with maximal progress.
773 return $max_progress[(int)floor( $num_progress / 2 )];
779 public function getLcsLength() {
780 if ( $this->heuristicUsed
&& !$this->lcsLengthCorrectedForHeuristic
) {
781 $this->lcsLengthCorrectedForHeuristic
= true;
782 $this->length
= $this->m
- array_sum( $this->added
);
785 return $this->length
;
791 * Alternative representation of a set of changes, by the index
792 * ranges that are changed.
794 * @ingroup DifferenceEngine
796 class RangeDifference
{
816 function __construct( $leftstart, $leftend, $rightstart, $rightend ) {
817 $this->leftstart
= $leftstart;
818 $this->leftend
= $leftend;
819 $this->leftlength
= $leftend - $leftstart;
820 $this->rightstart
= $rightstart;
821 $this->rightend
= $rightend;
822 $this->rightlength
= $rightend - $rightstart;