More concise token format

CHANGES

@@ -2,6 +2,7 @@
 
   * Fixed a bug with higher-order sections that prevented them from being
     passed the raw text of the section from the original template.
+  * More concise token format.
 
 = 0.6.0 / 31 Aug 2012
 

mustache.js

@@ -321,13 +321,13 @@ var Mustache;
    * last token that is part of that section. Returns an array of [start, end].
    */
   function sectionBounds(token) {
-    var start = token.end;
+    var start = token[3];
     var end = start;
 
     var tokens;
-    while ((tokens = token.tokens) && tokens.length) {
+    while ((tokens = token[4]) && tokens.length) {
       token = tokens[tokens.length - 1];
-      end = token.end;
+      end = token[3];
     }
 
     return [start, end];
@@ -346,30 +346,30 @@ var Mustache;
     for (var i = 0, len = tokens.length; i < len; ++i) {
       token = tokens[i];
 
-      switch (token.type) {
+      switch (token[0]) {
       case "#":
         bounds = sectionBounds(token);
         text = "t.slice(" + bounds[0] + ", " + bounds[1] + ")";
-        body.push("r._section(" + quote(token.value) + ", c, " + text + ", function (c, r) {\n" +
-          "  " + compileTokens(token.tokens, true) + "\n" +
+        body.push("r._section(" + quote(token[1]) + ", c, " + text + ", function (c, r) {\n" +
+          "  " + compileTokens(token[4], true) + "\n" +
           "})");
         break;
       case "^":
-        body.push("r._inverted(" + quote(token.value) + ", c, function (c, r) {\n" +
-          "  " + compileTokens(token.tokens, true) + "\n" +
+        body.push("r._inverted(" + quote(token[1]) + ", c, function (c, r) {\n" +
+          "  " + compileTokens(token[4], true) + "\n" +
           "})");
         break;
       case "{":
       case "&":
       case "name":
-        escape = String(token.type === "name");
-        body.push("r._name(" + quote(token.value) + ", c, " + escape + ")");
+        escape = String(token[0] === "name");
+        body.push("r._name(" + quote(token[1]) + ", c, " + escape + ")");
         break;
       case ">":
-        body.push("r._partial(" + quote(token.value) + ", c)");
+        body.push("r._partial(" + quote(token[1]) + ", c)");
         break;
       case "text":
-        body.push(quote(token.value));
+        body.push(quote(token[1]));
         break;
       }
     }
@@ -413,27 +413,27 @@ var Mustache;
     for (var i = 0; i < tokens.length; ++i) {
       token = tokens[i];
 
-      switch (token.type) {
+      switch (token[0]) {
       case "#":
       case "^":
-        token.tokens = [];
+        token[4] = [];
         sections.push(token);
         collector.push(token);
-        collector = token.tokens;
+        collector = token[4];
         break;
       case "/":
         if (sections.length === 0) {
-          throw new Error("Unopened section: " + token.value);
+          throw new Error("Unopened section: " + token[1]);
         }
 
         section = sections.pop();
 
-        if (section.value !== token.value) {
-          throw new Error("Unclosed section: " + section.value);
+        if (section[1] !== token[1]) {
+          throw new Error("Unclosed section: " + section[1]);
         }
 
         if (sections.length > 0) {
-          collector = sections[sections.length - 1].tokens;
+          collector = sections[sections.length - 1][4];
         } else {
           collector = tree;
         }
@@ -447,7 +447,7 @@ var Mustache;
     section = sections.pop();
 
     if (section) {
-      throw new Error("Unclosed section: " + section.value);
+      throw new Error("Unclosed section: " + section[1]);
     }
 
     return tree;
@@ -458,13 +458,13 @@ var Mustache;
    * to a single token.
    */
   function squashTokens(tokens) {
-    var lastToken;
+    var token, lastToken;
 
     for (var i = 0; i < tokens.length; ++i) {
-      var token = tokens[i];
+      token = tokens[i];
 
-      if (lastToken && lastToken.type === "text" && token.type === "text") {
-        lastToken.value += token.value;
+      if (lastToken && lastToken[0] === "text" && token[0] === "text") {
+        lastToken[1] += token[1];
         tokens.splice(i--, 1); // Remove this token from the array.
       } else {
         lastToken = token;
@@ -520,7 +520,7 @@ var Mustache;
             nonSpace = true;
           }
 
-          tokens.push({type: "text", value: chr, start: start, end: scanner.pos});
+          tokens.push(["text", chr, start, scanner.pos]);
 
           if (chr === "\n") {
             stripSpace(); // Check for whitespace on the current line.
@@ -560,7 +560,7 @@ var Mustache;
         throw new Error("Unclosed tag at " + scanner.pos);
       }
 
-      tokens.push({type: type, value: value, start: start, end: scanner.pos});
+      tokens.push([type, value, start, scanner.pos]);
 
       if (type === "name" || type === "{" || type === "&") {
         nonSpace = true;

test/parse_test.js

@@ -1,82 +1,68 @@
-var assert = require("assert");
-var vows = require("vows");
-var parse = require("./../mustache").parse;
+var assert = require('assert');
+var vows = require('vows');
+var parse = require('./../mustache').parse;
 
-// A map of templates to their expected token output.
+// A map of templates to their expected token output. Tokens are in the format:
+// [type, value, startIndex, endIndex].
 var expectations = {
-  "{{hi}}"                                  : [ [ 'name', 'hi', 0, 6 ] ],
-  "{{hi.world}}"                            : [ [ 'name', 'hi.world', 0, 12 ] ],
-  "{{hi . world}}"                          : [ [ 'name', 'hi . world', 0, 14 ] ],
-  "{{ hi}}"                                 : [ [ 'name', 'hi', 0, 7 ] ],
-  "{{hi }}"                                 : [ [ 'name', 'hi', 0, 7 ] ],
-  "{{ hi }}"                                : [ [ 'name', 'hi', 0, 8 ] ],
-  "{{{hi}}}"                                : [ [ '{', 'hi', 0, 8 ] ],
-  "{{!hi}}"                                 : [ [ '!', 'hi', 0, 7 ] ],
-  "{{! hi}}"                                : [ [ '!', 'hi', 0, 8 ] ],
-  "{{! hi }}"                               : [ [ '!', 'hi', 0, 9 ] ],
-  "{{ !hi}}"                                : [ [ '!', 'hi', 0, 8 ] ],
-  "{{ ! hi}}"                               : [ [ '!', 'hi', 0, 9 ] ],
-  "{{ ! hi }}"                              : [ [ '!', 'hi', 0, 10 ] ],
-  "a{{hi}}"                                 : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ] ],
-  "a {{hi}}"                                : [ [ 'text', 'a ', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
-  " a{{hi}}"                                : [ [ 'text', ' a', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
-  " a {{hi}}"                               : [ [ 'text', ' a ', 0, 3 ], [ 'name', 'hi', 3, 9 ] ],
-  "a{{hi}}b"                                : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b', 7, 8 ] ],
-  "a{{hi}} b"                               : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', ' b', 7, 9 ] ],
-  "a{{hi}}b "                               : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b ', 7, 9 ] ],
-  "a\n{{hi}} b \n"                          : [ [ 'text', 'a\n', 0, 2 ], [ 'name', 'hi', 2, 8 ], [ 'text', ' b \n', 8, 12 ] ],
-  "a\n {{hi}} \nb"                          : [ [ 'text', 'a\n ', 0, 3 ], [ 'name', 'hi', 3, 9 ], [ 'text', ' \nb', 9, 12 ] ],
-  "a\n {{!hi}} \nb"                         : [ [ 'text', 'a\n', 0, 3 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 10, 13 ] ],
-  "a\n{{#a}}{{/a}}\nb"                      : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 14, 16 ] ],
-  "a\n {{#a}}{{/a}}\nb"                     : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 17 ] ],
-  "a\n {{#a}}{{/a}} \nb"                    : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 18 ] ],
-  "a\n{{#a}}\n{{/a}}\nb"                    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 17 ] ],
-  "a\n {{#a}}\n{{/a}}\nb"                   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 18 ] ],
-  "a\n {{#a}}\n{{/a}} \nb"                  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 19 ] ],
-  "a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb"    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 29, 31 ] ],
-  "a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb"   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 32 ] ],
-  "a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb"  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 33 ] ],
-  "a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb"    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 29, 31 ] ],
-  "a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb"   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 32 ] ],
-  "a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb"  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 33 ] ],
-  "{{>abc}}"                                : [ [ '>', 'abc', 0, 8 ] ],
-  "{{> abc }}"                              : [ [ '>', 'abc', 0, 10 ] ],
-  "{{ > abc }}"                             : [ [ '>', 'abc', 0, 11 ] ],
-  "{{=<% %>=}}"                             : [ [ '=', '<% %>', 0, 11 ] ],
-  "{{= <% %> =}}"                           : [ [ '=', '<% %>', 0, 13 ] ],
-  "{{=<% %>=}}<%={{ }}=%>"                  : [ [ '=', '<% %>', 0, 11 ], [ '=', '{{ }}', 11, 22 ] ],
-  "{{=<% %>=}}<%hi%>"                       : [ [ '=', '<% %>', 0, 11 ], [ 'name', 'hi', 11, 17 ] ],
-  "{{#a}}{{/a}}hi{{#b}}{{/b}}\n"            : [ [ '#', 'a', 0, 6, [] ], [ 'text', 'hi', 12, 14 ], [ '#', 'b', 14, 20, [] ], [ 'text', '\n', 26, 27 ] ],
-  "{{a}}\n{{b}}\n\n{{#c}}\n{{/c}}\n"        : [ [ 'name', 'a', 0, 5 ], [ 'text', '\n', 5, 6 ], [ 'name', 'b', 6, 11 ], [ 'text', '\n\n', 11, 13 ], [ '#', 'c', 13, 19, [] ] ],
-  "{{#foo}}\n  {{#a}}\n    {{b}}\n  {{/a}}\n{{/foo}}\n"
+  '{{hi}}'                                  : [ [ 'name', 'hi', 0, 6 ] ],
+  '{{hi.world}}'                            : [ [ 'name', 'hi.world', 0, 12 ] ],
+  '{{hi . world}}'                          : [ [ 'name', 'hi . world', 0, 14 ] ],
+  '{{ hi}}'                                 : [ [ 'name', 'hi', 0, 7 ] ],
+  '{{hi }}'                                 : [ [ 'name', 'hi', 0, 7 ] ],
+  '{{ hi }}'                                : [ [ 'name', 'hi', 0, 8 ] ],
+  '{{{hi}}}'                                : [ [ '{', 'hi', 0, 8 ] ],
+  '{{!hi}}'                                 : [ [ '!', 'hi', 0, 7 ] ],
+  '{{! hi}}'                                : [ [ '!', 'hi', 0, 8 ] ],
+  '{{! hi }}'                               : [ [ '!', 'hi', 0, 9 ] ],
+  '{{ !hi}}'                                : [ [ '!', 'hi', 0, 8 ] ],
+  '{{ ! hi}}'                               : [ [ '!', 'hi', 0, 9 ] ],
+  '{{ ! hi }}'                              : [ [ '!', 'hi', 0, 10 ] ],
+  'a{{hi}}'                                 : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ] ],
+  'a {{hi}}'                                : [ [ 'text', 'a ', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
+  ' a{{hi}}'                                : [ [ 'text', ' a', 0, 2 ], [ 'name', 'hi', 2, 8 ] ],
+  ' a {{hi}}'                               : [ [ 'text', ' a ', 0, 3 ], [ 'name', 'hi', 3, 9 ] ],
+  'a{{hi}}b'                                : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b', 7, 8 ] ],
+  'a{{hi}} b'                               : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', ' b', 7, 9 ] ],
+  'a{{hi}}b '                               : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b ', 7, 9 ] ],
+  'a\n{{hi}} b \n'                          : [ [ 'text', 'a\n', 0, 2 ], [ 'name', 'hi', 2, 8 ], [ 'text', ' b \n', 8, 12 ] ],
+  'a\n {{hi}} \nb'                          : [ [ 'text', 'a\n ', 0, 3 ], [ 'name', 'hi', 3, 9 ], [ 'text', ' \nb', 9, 12 ] ],
+  'a\n {{!hi}} \nb'                         : [ [ 'text', 'a\n', 0, 3 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 10, 13 ] ],
+  'a\n{{#a}}{{/a}}\nb'                      : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 14, 16 ] ],
+  'a\n {{#a}}{{/a}}\nb'                     : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 17 ] ],
+  'a\n {{#a}}{{/a}} \nb'                    : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 18 ] ],
+  'a\n{{#a}}\n{{/a}}\nb'                    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 17 ] ],
+  'a\n {{#a}}\n{{/a}}\nb'                   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 18 ] ],
+  'a\n {{#a}}\n{{/a}} \nb'                  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 19 ] ],
+  'a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 29, 31 ] ],
+  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb'   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 32 ] ],
+  'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb'  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 33 ] ],
+  'a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'    : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 29, 31 ] ],
+  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb'   : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 32 ] ],
+  'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb'  : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 33 ] ],
+  '{{>abc}}'                                : [ [ '>', 'abc', 0, 8 ] ],
+  '{{> abc }}'                              : [ [ '>', 'abc', 0, 10 ] ],
+  '{{ > abc }}'                             : [ [ '>', 'abc', 0, 11 ] ],
+  '{{=<% %>=}}'                             : [ [ '=', '<% %>', 0, 11 ] ],
+  '{{= <% %> =}}'                           : [ [ '=', '<% %>', 0, 13 ] ],
+  '{{=<% %>=}}<%={{ }}=%>'                  : [ [ '=', '<% %>', 0, 11 ], [ '=', '{{ }}', 11, 22 ] ],
+  '{{=<% %>=}}<%hi%>'                       : [ [ '=', '<% %>', 0, 11 ], [ 'name', 'hi', 11, 17 ] ],
+  '{{#a}}{{/a}}hi{{#b}}{{/b}}\n'            : [ [ '#', 'a', 0, 6, [] ], [ 'text', 'hi', 12, 14 ], [ '#', 'b', 14, 20, [] ], [ 'text', '\n', 26, 27 ] ],
+  '{{a}}\n{{b}}\n\n{{#c}}\n{{/c}}\n'        : [ [ 'name', 'a', 0, 5 ], [ 'text', '\n', 5, 6 ], [ 'name', 'b', 6, 11 ], [ 'text', '\n\n', 11, 13 ], [ '#', 'c', 13, 19, [] ] ],
+  '{{#foo}}\n  {{#a}}\n    {{b}}\n  {{/a}}\n{{/foo}}\n'
                                             : [ [ '#', 'foo', 0, 8, [ [ '#', 'a', 11, 17, [ [ 'text', '    ', 17, 22 ], [ 'name', 'b', 22, 27 ], [ 'text', '\n', 27, 30 ] ] ] ] ] ]
 };
 
-function makeToken(tokenArray) {
-  var token = {
-    type: tokenArray[0],
-    value: tokenArray[1],
-    start: tokenArray[2],
-    end: tokenArray[3]
-  };
-
-  if (tokenArray[4]) {
-    token.tokens = tokenArray[4].map(makeToken);
-  }
-
-  return token;
-}
-
 var spec = {};
 
 for (var template in expectations) {
   (function (template, tokens) {
-    spec["knows how to parse " + JSON.stringify(template)] = function () {
-      assert.deepEqual(parse(template), tokens.map(makeToken));
+    spec['knows how to parse ' + JSON.stringify(template)] = function () {
+      assert.deepEqual(parse(template), tokens);
     };
   })(template, expectations[template]);
 }
 
-vows.describe("Mustache.parse").addBatch({
-  "parse": spec
+vows.describe('Mustache.parse').addBatch({
+  'parse': spec
 }).export(module);