From 3fb61fc0562ea9b92db20fc56960035306f018a4 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Fri, 7 Sep 2012 18:25:31 -0700 Subject: [PATCH] More concise token format --- CHANGES | 1 + mustache.js | 52 +++++++++---------- test/parse_test.js | 122 ++++++++++++++++++++------------------------- 3 files changed, 81 insertions(+), 94 deletions(-) diff --git a/CHANGES b/CHANGES index cc0f1f8..34d23e2 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,7 @@ * Fixed a bug with higher-order sections that prevented them from being passed the raw text of the section from the original template. + * More concise token format. = 0.6.0 / 31 Aug 2012 diff --git a/mustache.js b/mustache.js index 16e85ba..390fb0c 100644 --- a/mustache.js +++ b/mustache.js @@ -321,13 +321,13 @@ var Mustache; * last token that is part of that section. Returns an array of [start, end]. */ function sectionBounds(token) { - var start = token.end; + var start = token[3]; var end = start; var tokens; - while ((tokens = token.tokens) && tokens.length) { + while ((tokens = token[4]) && tokens.length) { token = tokens[tokens.length - 1]; - end = token.end; + end = token[3]; } return [start, end]; @@ -346,30 +346,30 @@ var Mustache; for (var i = 0, len = tokens.length; i < len; ++i) { token = tokens[i]; - switch (token.type) { + switch (token[0]) { case "#": bounds = sectionBounds(token); text = "t.slice(" + bounds[0] + ", " + bounds[1] + ")"; - body.push("r._section(" + quote(token.value) + ", c, " + text + ", function (c, r) {\n" + - " " + compileTokens(token.tokens, true) + "\n" + + body.push("r._section(" + quote(token[1]) + ", c, " + text + ", function (c, r) {\n" + + " " + compileTokens(token[4], true) + "\n" + "})"); break; case "^": - body.push("r._inverted(" + quote(token.value) + ", c, function (c, r) {\n" + - " " + compileTokens(token.tokens, true) + "\n" + + body.push("r._inverted(" + quote(token[1]) + ", c, function (c, r) {\n" + + " " + compileTokens(token[4], true) + "\n" + "})"); break; case "{": case "&": case "name": - escape = String(token.type === "name"); - body.push("r._name(" + quote(token.value) + ", c, " + escape + ")"); + escape = String(token[0] === "name"); + body.push("r._name(" + quote(token[1]) + ", c, " + escape + ")"); break; case ">": - body.push("r._partial(" + quote(token.value) + ", c)"); + body.push("r._partial(" + quote(token[1]) + ", c)"); break; case "text": - body.push(quote(token.value)); + body.push(quote(token[1])); break; } } @@ -413,27 +413,27 @@ var Mustache; for (var i = 0; i < tokens.length; ++i) { token = tokens[i]; - switch (token.type) { + switch (token[0]) { case "#": case "^": - token.tokens = []; + token[4] = []; sections.push(token); collector.push(token); - collector = token.tokens; + collector = token[4]; break; case "/": if (sections.length === 0) { - throw new Error("Unopened section: " + token.value); + throw new Error("Unopened section: " + token[1]); } section = sections.pop(); - if (section.value !== token.value) { - throw new Error("Unclosed section: " + section.value); + if (section[1] !== token[1]) { + throw new Error("Unclosed section: " + section[1]); } if (sections.length > 0) { - collector = sections[sections.length - 1].tokens; + collector = sections[sections.length - 1][4]; } else { collector = tree; } @@ -447,7 +447,7 @@ var Mustache; section = sections.pop(); if (section) { - throw new Error("Unclosed section: " + section.value); + throw new Error("Unclosed section: " + section[1]); } return tree; @@ -458,13 +458,13 @@ var Mustache; * to a single token. */ function squashTokens(tokens) { - var lastToken; + var token, lastToken; for (var i = 0; i < tokens.length; ++i) { - var token = tokens[i]; + token = tokens[i]; - if (lastToken && lastToken.type === "text" && token.type === "text") { - lastToken.value += token.value; + if (lastToken && lastToken[0] === "text" && token[0] === "text") { + lastToken[1] += token[1]; tokens.splice(i--, 1); // Remove this token from the array. } else { lastToken = token; @@ -520,7 +520,7 @@ var Mustache; nonSpace = true; } - tokens.push({type: "text", value: chr, start: start, end: scanner.pos}); + tokens.push(["text", chr, start, scanner.pos]); if (chr === "\n") { stripSpace(); // Check for whitespace on the current line. @@ -560,7 +560,7 @@ var Mustache; throw new Error("Unclosed tag at " + scanner.pos); } - tokens.push({type: type, value: value, start: start, end: scanner.pos}); + tokens.push([type, value, start, scanner.pos]); if (type === "name" || type === "{" || type === "&") { nonSpace = true; diff --git a/test/parse_test.js b/test/parse_test.js index c54b57f..e3e06dc 100644 --- a/test/parse_test.js +++ b/test/parse_test.js @@ -1,82 +1,68 @@ -var assert = require("assert"); -var vows = require("vows"); -var parse = require("./../mustache").parse; +var assert = require('assert'); +var vows = require('vows'); +var parse = require('./../mustache').parse; -// A map of templates to their expected token output. +// A map of templates to their expected token output. Tokens are in the format: +// [type, value, startIndex, endIndex]. var expectations = { - "{{hi}}" : [ [ 'name', 'hi', 0, 6 ] ], - "{{hi.world}}" : [ [ 'name', 'hi.world', 0, 12 ] ], - "{{hi . world}}" : [ [ 'name', 'hi . world', 0, 14 ] ], - "{{ hi}}" : [ [ 'name', 'hi', 0, 7 ] ], - "{{hi }}" : [ [ 'name', 'hi', 0, 7 ] ], - "{{ hi }}" : [ [ 'name', 'hi', 0, 8 ] ], - "{{{hi}}}" : [ [ '{', 'hi', 0, 8 ] ], - "{{!hi}}" : [ [ '!', 'hi', 0, 7 ] ], - "{{! hi}}" : [ [ '!', 'hi', 0, 8 ] ], - "{{! hi }}" : [ [ '!', 'hi', 0, 9 ] ], - "{{ !hi}}" : [ [ '!', 'hi', 0, 8 ] ], - "{{ ! hi}}" : [ [ '!', 'hi', 0, 9 ] ], - "{{ ! hi }}" : [ [ '!', 'hi', 0, 10 ] ], - "a{{hi}}" : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ] ], - "a {{hi}}" : [ [ 'text', 'a ', 0, 2 ], [ 'name', 'hi', 2, 8 ] ], - " a{{hi}}" : [ [ 'text', ' a', 0, 2 ], [ 'name', 'hi', 2, 8 ] ], - " a {{hi}}" : [ [ 'text', ' a ', 0, 3 ], [ 'name', 'hi', 3, 9 ] ], - "a{{hi}}b" : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b', 7, 8 ] ], - "a{{hi}} b" : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', ' b', 7, 9 ] ], - "a{{hi}}b " : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b ', 7, 9 ] ], - "a\n{{hi}} b \n" : [ [ 'text', 'a\n', 0, 2 ], [ 'name', 'hi', 2, 8 ], [ 'text', ' b \n', 8, 12 ] ], - "a\n {{hi}} \nb" : [ [ 'text', 'a\n ', 0, 3 ], [ 'name', 'hi', 3, 9 ], [ 'text', ' \nb', 9, 12 ] ], - "a\n {{!hi}} \nb" : [ [ 'text', 'a\n', 0, 3 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 10, 13 ] ], - "a\n{{#a}}{{/a}}\nb" : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 14, 16 ] ], - "a\n {{#a}}{{/a}}\nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 17 ] ], - "a\n {{#a}}{{/a}} \nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 18 ] ], - "a\n{{#a}}\n{{/a}}\nb" : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 17 ] ], - "a\n {{#a}}\n{{/a}}\nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 18 ] ], - "a\n {{#a}}\n{{/a}} \nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 19 ] ], - "a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb" : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 29, 31 ] ], - "a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 32 ] ], - "a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 33 ] ], - "a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb" : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 29, 31 ] ], - "a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 32 ] ], - "a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb" : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 33 ] ], - "{{>abc}}" : [ [ '>', 'abc', 0, 8 ] ], - "{{> abc }}" : [ [ '>', 'abc', 0, 10 ] ], - "{{ > abc }}" : [ [ '>', 'abc', 0, 11 ] ], - "{{=<% %>=}}" : [ [ '=', '<% %>', 0, 11 ] ], - "{{= <% %> =}}" : [ [ '=', '<% %>', 0, 13 ] ], - "{{=<% %>=}}<%={{ }}=%>" : [ [ '=', '<% %>', 0, 11 ], [ '=', '{{ }}', 11, 22 ] ], - "{{=<% %>=}}<%hi%>" : [ [ '=', '<% %>', 0, 11 ], [ 'name', 'hi', 11, 17 ] ], - "{{#a}}{{/a}}hi{{#b}}{{/b}}\n" : [ [ '#', 'a', 0, 6, [] ], [ 'text', 'hi', 12, 14 ], [ '#', 'b', 14, 20, [] ], [ 'text', '\n', 26, 27 ] ], - "{{a}}\n{{b}}\n\n{{#c}}\n{{/c}}\n" : [ [ 'name', 'a', 0, 5 ], [ 'text', '\n', 5, 6 ], [ 'name', 'b', 6, 11 ], [ 'text', '\n\n', 11, 13 ], [ '#', 'c', 13, 19, [] ] ], - "{{#foo}}\n {{#a}}\n {{b}}\n {{/a}}\n{{/foo}}\n" + '{{hi}}' : [ [ 'name', 'hi', 0, 6 ] ], + '{{hi.world}}' : [ [ 'name', 'hi.world', 0, 12 ] ], + '{{hi . world}}' : [ [ 'name', 'hi . world', 0, 14 ] ], + '{{ hi}}' : [ [ 'name', 'hi', 0, 7 ] ], + '{{hi }}' : [ [ 'name', 'hi', 0, 7 ] ], + '{{ hi }}' : [ [ 'name', 'hi', 0, 8 ] ], + '{{{hi}}}' : [ [ '{', 'hi', 0, 8 ] ], + '{{!hi}}' : [ [ '!', 'hi', 0, 7 ] ], + '{{! hi}}' : [ [ '!', 'hi', 0, 8 ] ], + '{{! hi }}' : [ [ '!', 'hi', 0, 9 ] ], + '{{ !hi}}' : [ [ '!', 'hi', 0, 8 ] ], + '{{ ! hi}}' : [ [ '!', 'hi', 0, 9 ] ], + '{{ ! hi }}' : [ [ '!', 'hi', 0, 10 ] ], + 'a{{hi}}' : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ] ], + 'a {{hi}}' : [ [ 'text', 'a ', 0, 2 ], [ 'name', 'hi', 2, 8 ] ], + ' a{{hi}}' : [ [ 'text', ' a', 0, 2 ], [ 'name', 'hi', 2, 8 ] ], + ' a {{hi}}' : [ [ 'text', ' a ', 0, 3 ], [ 'name', 'hi', 3, 9 ] ], + 'a{{hi}}b' : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b', 7, 8 ] ], + 'a{{hi}} b' : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', ' b', 7, 9 ] ], + 'a{{hi}}b ' : [ [ 'text', 'a', 0, 1 ], [ 'name', 'hi', 1, 7 ], [ 'text', 'b ', 7, 9 ] ], + 'a\n{{hi}} b \n' : [ [ 'text', 'a\n', 0, 2 ], [ 'name', 'hi', 2, 8 ], [ 'text', ' b \n', 8, 12 ] ], + 'a\n {{hi}} \nb' : [ [ 'text', 'a\n ', 0, 3 ], [ 'name', 'hi', 3, 9 ], [ 'text', ' \nb', 9, 12 ] ], + 'a\n {{!hi}} \nb' : [ [ 'text', 'a\n', 0, 3 ], [ '!', 'hi', 3, 10 ], [ 'text', 'b', 10, 13 ] ], + 'a\n{{#a}}{{/a}}\nb' : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 14, 16 ] ], + 'a\n {{#a}}{{/a}}\nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 17 ] ], + 'a\n {{#a}}{{/a}} \nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 15, 18 ] ], + 'a\n{{#a}}\n{{/a}}\nb' : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ 'text', 'b', 15, 17 ] ], + 'a\n {{#a}}\n{{/a}}\nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 18 ] ], + 'a\n {{#a}}\n{{/a}} \nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ 'text', 'b', 16, 19 ] ], + 'a\n{{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb' : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [] ], [ '#', 'b', 16, 22, [] ], [ 'text', 'b', 29, 31 ] ], + 'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}}\nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 32 ] ], + 'a\n {{#a}}\n{{/a}}\n{{#b}}\n{{/b}} \nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [] ], [ '#', 'b', 17, 23, [] ], [ 'text', 'b', 30, 33 ] ], + 'a\n{{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb' : [ [ 'text', 'a\n', 0, 2 ], [ '#', 'a', 2, 8, [ [ '#', 'b', 9, 15, [] ] ] ], [ 'text', 'b', 29, 31 ] ], + 'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}}\nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 32 ] ], + 'a\n {{#a}}\n{{#b}}\n{{/b}}\n{{/a}} \nb' : [ [ 'text', 'a\n', 0, 3 ], [ '#', 'a', 3, 9, [ [ '#', 'b', 10, 16, [] ] ] ], [ 'text', 'b', 30, 33 ] ], + '{{>abc}}' : [ [ '>', 'abc', 0, 8 ] ], + '{{> abc }}' : [ [ '>', 'abc', 0, 10 ] ], + '{{ > abc }}' : [ [ '>', 'abc', 0, 11 ] ], + '{{=<% %>=}}' : [ [ '=', '<% %>', 0, 11 ] ], + '{{= <% %> =}}' : [ [ '=', '<% %>', 0, 13 ] ], + '{{=<% %>=}}<%={{ }}=%>' : [ [ '=', '<% %>', 0, 11 ], [ '=', '{{ }}', 11, 22 ] ], + '{{=<% %>=}}<%hi%>' : [ [ '=', '<% %>', 0, 11 ], [ 'name', 'hi', 11, 17 ] ], + '{{#a}}{{/a}}hi{{#b}}{{/b}}\n' : [ [ '#', 'a', 0, 6, [] ], [ 'text', 'hi', 12, 14 ], [ '#', 'b', 14, 20, [] ], [ 'text', '\n', 26, 27 ] ], + '{{a}}\n{{b}}\n\n{{#c}}\n{{/c}}\n' : [ [ 'name', 'a', 0, 5 ], [ 'text', '\n', 5, 6 ], [ 'name', 'b', 6, 11 ], [ 'text', '\n\n', 11, 13 ], [ '#', 'c', 13, 19, [] ] ], + '{{#foo}}\n {{#a}}\n {{b}}\n {{/a}}\n{{/foo}}\n' : [ [ '#', 'foo', 0, 8, [ [ '#', 'a', 11, 17, [ [ 'text', ' ', 17, 22 ], [ 'name', 'b', 22, 27 ], [ 'text', '\n', 27, 30 ] ] ] ] ] ] }; -function makeToken(tokenArray) { - var token = { - type: tokenArray[0], - value: tokenArray[1], - start: tokenArray[2], - end: tokenArray[3] - }; - - if (tokenArray[4]) { - token.tokens = tokenArray[4].map(makeToken); - } - - return token; -} - var spec = {}; for (var template in expectations) { (function (template, tokens) { - spec["knows how to parse " + JSON.stringify(template)] = function () { - assert.deepEqual(parse(template), tokens.map(makeToken)); + spec['knows how to parse ' + JSON.stringify(template)] = function () { + assert.deepEqual(parse(template), tokens); }; })(template, expectations[template]); } -vows.describe("Mustache.parse").addBatch({ - "parse": spec +vows.describe('Mustache.parse').addBatch({ + 'parse': spec }).export(module);