Treat formats and buffers as interchangeable when they are mixed toge…

…ther.

During the compiler's optimization pass, "format" globs (basically whitespace) are stripped out. The remaining strings, "buffers", are concatenated into a single entity.
If whitespace compression is disabled, formats and buffers are all written, but the two types are not concatenated together. Because most code is broken up by newlines, this results in highly-inefficient chunking of writes that-- in the case of large templates-- causes Dust to exceed the call stack limit.

This patch adds dust.config.whitespace (set by default for now to false for backwards compatibility; by default there is zero change with or without the patch) to toggle the whitespace stripping behavior. When the flag is set to true (respect whitespace), the new format/buffer concat logic is invoked.

Also adds --whitespace=true|false (default false) flag to dustc.

Closes linkedin#498

.jshintrc

@@ -12,7 +12,7 @@
   "eqnull": true,
   "browser": true,
   "globals": {
-  "jQuery": true,
+    "jQuery": true,
     "require": true,
     "module": true
   }

bin/dustc

@@ -3,26 +3,39 @@
 var path = require('path'),
     fs = require('fs'),
     sys = require('util'),
-    dust = require('../lib/server'),
+    dust = require('../lib/server');
+
+var ARG_REGEX = /^--?([a-z][0-9a-z-]*)(?:=([^\s]+))?$/i,
     args = process.argv.slice(1),
     name = null;
 
-args = args.filter(function (arg) {
-    var match;
+args = args.filter(function(arg) {
+    var match = arg.match(ARG_REGEX),
+        val;
+
+    if (!match) {
+        return true;
+    }
 
-    if (match = arg.match(/^--?([a-z][0-9a-z-]*)(?:=([^\s]+))?$/i)) { arg = match[1] }
-    else { return arg }
+    arg = match[1];
+    val = match[2];
 
     switch (arg) {
         case 'h':
         case 'help':
-            sys.puts("usage: dustc [{-n|--name}=<template_name>] {sourcefilename|-} [destination]");
+            sys.puts("usage: dustc [{-n|--name}=<template_name>] [--whitespace=true|false] {source|-} [destination]");
             process.exit(0);
+            break;
         case 'n':
         case 'name':
-            name = match[2];
+            name = val;
+            break;
+        case 'whitespace':
+            dust.config.whitespace = (val === 'true');
             break;
     }
+
+    return false;
 });
 
 var input = args[1];

lib/compiler.js

@@ -11,7 +11,7 @@
   var compiler = {},
       isArray = dust.isArray;
 
-  
+
   compiler.compile = function(source, name) {
     // the name parameter is optional.
     // this can happen for templates that are rendered immediately (renderSource which calls compileFn) or
@@ -21,7 +21,7 @@
     if (!name && name !== null) {
       throw new Error('Template name parameter cannot be undefined when calling dust.compile');
     }
- 
+
     try {
       var ast = filterAST(parse(source));
       return compile(ast, name);
@@ -48,7 +48,7 @@
     body:      compactBuffers,
     buffer:    noop,
     special:   convertSpecial,
-    format:    nullify,        // TODO: convert format
+    format:    format,
     reference: visit,
     '#':       visit,
     '?':       visit,
@@ -105,9 +105,10 @@
     for (i=1, len=node.length; i<len; i++) {
       res = compiler.filterNode(context, node[i]);
       if (res) {
-        if (res[0] === 'buffer') {
+        if (res[0] === 'buffer' || res[0] === 'format') {
           if (memo) {
-            memo[1] += res[1];
+            memo[0] = (res[0] === 'buffer') ? 'buffer' : memo[0];
+            memo[1] += res.slice(1, -2).join('');
           } else {
             memo = res;
             out.push(res);
@@ -130,7 +131,7 @@
   };
 
   function convertSpecial(context, node) {
-    return ['buffer', specialChars[node[1]]];
+    return ['buffer', specialChars[node[1]], node[2], node[3]];
   }
 
   function noop(context, node) {
@@ -139,6 +140,10 @@
 
   function nullify(){}
 
+  function format(context, node) {
+    return dust.config.whitespace ? node : null;
+  }
+
   function compile(ast, name) {
     var context = {
       name: name,
@@ -208,15 +213,15 @@
     },
 
     buffer: function(context, node) {
-      return '.write(' + escape(node[1]) + ')';
+      return '.w(' + escape(node[1]) + ')';
     },
 
     format: function(context, node) {
-      return '.write(' + escape(node[1] + node[2]) + ')';
+      return '.w(' + escape(node[1] + node[2]) + ')';
     },
 
     reference: function(context, node) {
-      return '.reference(' + compiler.compileNode(context, node[1]) +
+      return '.f(' + compiler.compileNode(context, node[1]) +
         ',ctx,' + compiler.compileNode(context, node[2]) + ')';
     },
 
@@ -263,7 +268,7 @@
     },
 
     '@': function(context, node) {
-      return '.helper(' +
+      return '.h(' +
         escape(node[1].text) +
         ',' + compiler.compileNode(context, node[2]) + ',' +
         compiler.compileNode(context, node[4]) + ',' +
@@ -303,7 +308,7 @@
     },
 
     partial: function(context, node) {
-      return '.partial(' +
+      return '.p(' +
           compiler.compileNode(context, node[1]) +
           ',' + compiler.compileNode(context, node[2]) +
           ',' + compiler.compileNode(context, node[3]) + ')';
@@ -372,12 +377,12 @@
       return escape(node[1]);
     },
     raw: function(context, node) {
-      return ".write(" + escape(node[1]) + ")";
+      return ".w(" + escape(node[1]) + ")";
     }
   };
 
   function compileSection(context, node, cmd) {
-    return '.' + cmd + '(' +
+    return '.' + (dust._aliases[cmd] || cmd) + '(' +
       compiler.compileNode(context, node[1]) +
       ',' + compiler.compileNode(context, node[2]) + ',' +
       compiler.compileNode(context, node[4]) + ',' +
@@ -411,8 +416,7 @@
   dust.pragmas = compiler.pragmas;
   dust.compileNode = compiler.compileNode;
   dust.nodes = compiler.nodes;
-  
+
   return compiler;
 
 }));
-

lib/dust.js

@@ -14,6 +14,25 @@
 
   dust.debugLevel = NONE;
 
+  dust.config = {
+    whitespace: false,
+  };
+
+  // Directive aliases to minify code
+  dust._aliases = {
+    "write": "w",
+    "end": "e",
+    "map": "m",
+    "render": "r",
+    "reference": "f",
+    "section": "s",
+    "exists": "x",
+    "notexists": "nx",
+    "block": "b",
+    "partial": "p",
+    "helper": "h"
+  };
+
   // Try to find the console in global scope
   if (root && root.console && root.console.log) {
     loggerContext = root.console;
@@ -795,6 +814,13 @@
     return this;
   };
 
+  // Chunk aliases
+  for(var f in Chunk.prototype) {
+    if(dust._aliases[f]) {
+      Chunk.prototype[dust._aliases[f]] = Chunk.prototype[f];
+    }
+  }
+
   function Tap(head, tail) {
     this.head = head;
     this.tail = tail;