Fix text token offsets

Text tokens now accurately reflect their content's exact position
in the original template.

CHANGES

@@ -3,7 +3,8 @@
   * Rename Renderer => Writer.
   * Fixed a bug with higher-order sections that prevented them from being
     passed the raw text of the section from the original template.
-  * More concise token format.
+  * More concise token format. Tokens also include start/end indices in the
+    original template.
   * High-level API is consistent with the Writer API.
   * Allow partials to be passed to the pre-compiled function (thanks @fallenice).
   * Don't use eval (thanks @cweider).

mustache.js

@@ -315,9 +315,9 @@ var Mustache;
 
     function subRender(i, tokens, template) {
       if (!subRenders[i]) {
-        var render = compileTokens(tokens);
+        var fn = compileTokens(tokens);
         subRenders[i] = function (writer, context) {
-          return render(writer, context, template);
+          return fn(writer, context, template);
         };
       }
 
@@ -360,21 +360,10 @@ var Mustache;
     return renderFunction;
   }
 
-  function escapeTags(tags) {
-    if (tags.length !== 2) {
-      throw new Error("Invalid tags: " + tags.join(" "));
-    }
-
-    return [
-      new RegExp(escapeRe(tags[0]) + "\\s*"),
-      new RegExp("\\s*" + escapeRe(tags[1]))
-    ];
-  }
-
   /**
-   * Forms the given linear array of `tokens` into a nested tree structure
-   * where tokens that represent a section have a fifth item: an array that
-   * contains all tokens in that section.
+   * Forms the given array of `tokens` into a nested tree structure where
+   * tokens that represent a section have a fifth item: an array that contains
+   * all tokens in that section.
    */
   function nestTokens(tokens) {
     var tree = [];
@@ -437,6 +426,7 @@ var Mustache;
 
       if (lastToken && lastToken[0] === "text" && token[0] === "text") {
         lastToken[1] += token[1];
+        lastToken[3] = token[3];
         tokens.splice(i--, 1); // Remove this token from the array.
       } else {
         lastToken = token;
@@ -444,6 +434,17 @@ var Mustache;
     }
   }
 
+  function escapeTags(tags) {
+    if (tags.length !== 2) {
+      throw new Error("Invalid tags: " + tags.join(" "));
+    }
+
+    return [
+      new RegExp(escapeRe(tags[0]) + "\\s*"),
+      new RegExp("\\s*" + escapeRe(tags[1]))
+    ];
+  }
+
   /**
    * Breaks up the given `template` string into a tree of token objects. If
    * `tags` is given here it must be an array with two string values: the
@@ -492,7 +493,8 @@ var Mustache;
             nonSpace = true;
           }
 
-          tokens.push(["text", chr, start, scanner.pos]);
+          tokens.push(["text", chr, start, start + 1]);
+          start += 1;
 
           if (chr === "\n") {
             stripSpace(); // Check for whitespace on the current line.

test/parse_test.js

@@ -18,6 +18,7 @@ var expectations = {
   '{{ !hi}}'                                : [ [ '!', 'hi', 0, 8 ] ],
   '{{ ! hi}}'                               : [ [ '!', 'hi', 0, 9 ] ],
   '{{ ! hi }}'                              : [ [ '!', 'hi', 0, 10 ] ],
+  'a\n b'                                   : [ [ 'text', 'a\n b', 0, 4 ] ],
   'a{{hi}}'                                 : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ] ],
   'a {{hi}}'                                : [ [ 'text', 'a ', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
   ' a{{hi}}'                                : [ [ 'text', ' a', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
@@ -27,19 +28,19 @@ var expectations = {
   'a{{hi}}b '                               : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b ', 7, 9 ] ],
   'a\n{{hi}} b \n'                          : [ [ 'text', 'a\n', 0, 2 ], [ 'name', 'hi', 2, 8 ], [ 'text', ' b \n', 8, 12 ] ],
   'a\n {{hi}} \nb'                          : [ [ 'text', 'a\n ', 0, 3 ], [ 'name', 'hi', 3, 9 ], [ 'text', ' \nb', 9, 12 ] ],
-  'a\n {{!hi}} \nb'                         : [ [ 'text', 'a\n', 0, 3 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 10, 13 ] ],
-  'a\n{{#a}}{{/a}}\nb'                      : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 14, 16 ] ],
-  'a\n {{#a}}{{/a}}\nb'                     : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 17 ] ],
-  'a\n {{#a}}{{/a}} \nb'                    : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 18 ] ],
-  'a\n{{#a}}\n{{/a}}\nb'                    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 17 ] ],
-  'a\n {{#a}}\n{{/a}}\nb'                   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 18 ] ],
-  'a\n {{#a}}\n{{/a}} \nb'                  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 19 ] ],
-  'a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 29, 31 ] ],
-  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 32 ] ],
-  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb'  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 33 ] ],
-  'a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 29, 31 ] ],
-  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 32 ] ],
-  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb'  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 33 ] ],
+  'a\n {{!hi}} \nb'                         : [ [ 'text', 'a\n', 0, 2 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 12, 13 ] ],
+  'a\n{{#a}}{{/a}}\nb'                      : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 16 ] ],
+  'a\n {{#a}}{{/a}}\nb'                     : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 17 ] ],
+  'a\n {{#a}}{{/a}} \nb'                    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 17, 18 ] ],
+  'a\n{{#a}}\n{{/a}}\nb'                    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 16, 17 ] ],
+  'a\n {{#a}}\n{{/a}}\nb'                   : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 17, 18 ] ],
+  'a\n {{#a}}\n{{/a}} \nb'                  : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 18, 19 ] ],
+  'a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 30, 31 ] ],
+  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'   : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 31, 32 ] ],
+  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb'  : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 32, 33 ] ],
+  'a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 30, 31 ] ],
+  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'   : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 31, 32 ] ],
+  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb'  : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 32, 33 ] ],
   '{{>abc}}'                                : [ [ '>', 'abc', 0, 8 ] ],
   '{{> abc }}'                              : [ [ '>', 'abc', 0, 10 ] ],
   '{{ > abc }}'                             : [ [ '>', 'abc', 0, 11 ] ],
@@ -50,7 +51,7 @@ var expectations = {
   '{{#a}}{{/a}}hi{{#b}}{{/b}}\n'            : [ [ '#', 'a', 0, 6, [] ], [ 'text', 'hi', 12, 14 ], [ '#', 'b', 14, 20, [] ], [ 'text', '\n', 26, 27 ] ],
   '{{a}}\n{{b}}\n\n{{#c}}\n{{/c}}\n'        : [ [ 'name', 'a', 0, 5 ], [ 'text', '\n', 5, 6 ], [ 'name', 'b', 6, 11 ], [ 'text', '\n\n', 11, 13 ], [ '#', 'c', 13, 19, [] ] ],
   '{{#foo}}\n  {{#a}}\n    {{b}}\n  {{/a}}\n{{/foo}}\n'
-                                            : [ [ '#', 'foo', 0, 8, [ [ '#', 'a', 11, 17, [ [ 'text', '    ', 17, 22 ], [ 'name', 'b', 22, 27 ], [ 'text', '\n', 27, 30 ] ] ] ] ] ]
+                                            : [ [ '#', 'foo', 0, 8, [ [ '#', 'a', 11, 17, [ [ 'text', '    ', 18, 22 ], [ 'name', 'b', 22, 27 ], [ 'text', '\n', 27, 28 ] ] ] ] ] ]
 };
 
 var spec = {};