Add a oneChangePerToken option (#460)

* Add a oneChangePerToken option to emit one change object per token instead of combining consecutive tokens into one change object

* Document oneChangePerToken

* Add release notes

* Add test (which fails; needs changes from #439)

* Add test of case with identical texts

* Add another test

* Move release notes to correct place now that I'm planning this for 6.0.0

Author: ExplodingCabbage

Diff for: README.md

@@ -167,6 +167,7 @@ Certain options can be provided in the `options` object of *any* method that cal
 
   (Note that if the ONLY option you want to provide is a callback, you can pass the callback function directly as the `options` parameter instead of passing an object with a `callback` property.)
 * `maxEditLength`: a number specifying the maximum edit distance to consider between the old and new texts. If the edit distance is higher than this, jsdiff will return `undefined` instead of a diff. You can use this to limit the computational cost of diffing large, very different texts by giving up early if the cost will be huge. Works for functions that return change objects and also for `structuredPatch`, but not other patch-generation functions.
+* `oneChangePerToken`: if `true`, the array of change objects returned will contain one change object per token (e.g. one per line if calling `diffLines`), instead of runs of consecutive tokens that are all added / all removed / all conserved being combined into a single change object.
 
 ### Defining custom diffing behaviors
 

Diff for: release-notes.md

@@ -8,6 +8,7 @@
 - [#439](https://github.com/kpdecker/jsdiff/pull/439) Prefer diffs that order deletions before insertions. When faced with a choice between two diffs with an equal total edit distance, the Myers diff algorithm generally prefers one that does deletions before insertions rather than insertions before deletions. For instance, when diffing `abcd` against `acbd`, it will prefer a diff that says to delete the `b` and then insert a new `b` after the `c`, over a diff that says to insert a `c` before the `b` and then delete the existing `c`. JsDiff deviated from the published Myers algorithm in a way that led to it having the opposite preference in many cases, including that example. This is now fixed, meaning diffs output by JsDiff will more accurately reflect what the published Myers diff algorithm would output.
 - [#455](https://github.com/kpdecker/jsdiff/pull/455) The `added` and `removed` properties of change objects are now guaranteed to be set to a boolean value. (Previously, they would be set to `undefined` or omitted entirely instead of setting them to false.)
 - [#464](https://github.com/kpdecker/jsdiff/pull/464) Specifying `{maxEditLength: 0}` now sets a max edit length of 0 instead of no maximum.
+- [#460][https://github.com/kpdecker/jsdiff/pull/460] Added `oneChangePerToken` option.
 
 ## Development
 

Diff for: src/diff/base.js

@@ -40,7 +40,7 @@ Diff.prototype = {
     let newPos = this.extractCommon(bestPath[0], newString, oldString, 0);
     if (bestPath[0].oldPos + 1 >= oldLen && newPos + 1 >= newLen) {
       // Identity per the equality and tokenizer
-      return done([{value: this.join(newString), count: newString.length, added: false, removed: false}]);
+      return done(buildValues(self, bestPath[0].lastComponent, newString, oldString, self.useLongestToken));
     }
 
     // Once we hit the right edge of the edit graph on some diagonal k, we can
@@ -147,7 +147,7 @@ Diff.prototype = {
 
   addToPath(path, added, removed, oldPosInc) {
     let last = path.lastComponent;
-    if (last && last.added === added && last.removed === removed) {
+    if (last && !this.options.oneChangePerToken && last.added === added && last.removed === removed) {
       return {
         oldPos: path.oldPos + oldPosInc,
         lastComponent: {count: last.count + 1, added: added, removed: removed, previousComponent: last.previousComponent }
@@ -170,9 +170,12 @@ Diff.prototype = {
       newPos++;
       oldPos++;
       commonCount++;
+      if (this.options.oneChangePerToken) {
+        basePath.lastComponent = {count: 1, previousComponent: basePath.lastComponent, added: false, removed: false};
+      }
     }
 
-    if (commonCount) {
+    if (commonCount && !this.options.oneChangePerToken) {
       basePath.lastComponent = {count: commonCount, previousComponent: basePath.lastComponent, added: false, removed: false};
     }
 

Diff for: test/diff/character.js

@@ -6,8 +6,25 @@ import {expect} from 'chai';
 describe('diff/character', function() {
   describe('#diffChars', function() {
     it('Should diff chars', function() {
-      const diffResult = diffChars('New Value.', 'New ValueMoreData.');
-      expect(convertChangesToXML(diffResult)).to.equal('New Value<ins>MoreData</ins>.');
+      const diffResult = diffChars('Old Value.', 'New ValueMoreData.');
+      expect(convertChangesToXML(diffResult)).to.equal('<del>Old</del><ins>New</ins> Value<ins>MoreData</ins>.');
+    });
+
+    describe('oneChangePerToken option', function() {
+      it('emits one change per character', function() {
+        const diffResult = diffChars('Old Value.', 'New ValueMoreData.', {oneChangePerToken: true});
+        expect(diffResult.length).to.equal(21);
+        expect(convertChangesToXML(diffResult)).to.equal('<del>O</del><del>l</del><del>d</del><ins>N</ins><ins>e</ins><ins>w</ins> Value<ins>M</ins><ins>o</ins><ins>r</ins><ins>e</ins><ins>D</ins><ins>a</ins><ins>t</ins><ins>a</ins>.');
+      });
+
+      it('correctly handles the case where the texts are identical', function() {
+        const diffResult = diffChars('foo bar baz qux', 'foo bar baz qux', {oneChangePerToken: true});
+        expect(diffResult).to.deep.equal(
+          ['f', 'o', 'o', ' ', 'b', 'a', 'r', ' ', 'b', 'a', 'z', ' ', 'q', 'u', 'x'].map(
+            char => ({value: char, count: 1, added: false, removed: false})
+          )
+        );
+      });
     });
 
     describe('case insensitivity', function() {

Diff for: test/diff/line.js

@@ -101,6 +101,25 @@ describe('diff/line', function() {
     });
   });
 
+  describe('oneChangePerToken option', function() {
+    it('emits one change per line', function() {
+      const diffResult = diffLines(
+        'foo\nbar\nbaz\nqux\n',
+        'fox\nbar\nbaz\nqux\n',
+        { oneChangePerToken: true }
+      );
+      expect(diffResult).to.deep.equal(
+        [
+          {value: 'foo\n', count: 1, added: false, removed: true},
+          {value: 'fox\n', count: 1, added: true, removed: false},
+          {value: 'bar\n', count: 1, added: false, removed: false},
+          {value: 'baz\n', count: 1, added: false, removed: false},
+          {value: 'qux\n', count: 1, added: false, removed: false}
+        ]
+      );
+    });
+  });
+
   // Trimmed Line Diff
   describe('#TrimmedLineDiff', function() {
     it('should diff lines', function() {