support template literals (#4601)

lib/ast.js

@@ -1418,6 +1418,34 @@ var AST_This = DEFNODE("This", null, {
     },
 }, AST_Symbol);
 
+var AST_Template = DEFNODE("Template", "expressions strings tag", {
+    $documentation: "A template literal, i.e. tag`str1${expr1}...strN${exprN}strN+1`",
+    $propdoc: {
+        expressions: "[AST_Node*] the placeholder expressions",
+        strings: "[string*] the interpolating text segments",
+        tag: "[AST_Node] tag function, or null if absent",
+    },
+    walk: function(visitor) {
+        var node = this;
+        visitor.visit(node, function() {
+            if (node.tag) node.tag.walk(visitor);
+            node.expressions.forEach(function(expr) {
+                expr.walk(visitor);
+            });
+        });
+    },
+    _validate: function() {
+        if (this.expressions.length + 1 != this.strings.length) {
+            throw new Error("malformed template with " + this.expressions.length + " placeholder(s) but " + this.strings.length + " text segment(s)");
+        }
+        must_be_expressions(this, "expressions");
+        this.strings.forEach(function(string) {
+            if (typeof string != "string") throw new Error("strings must contain string");
+        });
+        if (this.tag != null) must_be_expression(this, "tag");
+    },
+});
+
 var AST_Constant = DEFNODE("Constant", null, {
     $documentation: "Base class for all constants",
 });

lib/compress.js

@@ -4663,6 +4663,9 @@ merge(Compressor.prototype, {
         def(AST_SymbolRef, function(compressor) {
             return !this.is_declared(compressor) || !can_drop_symbol(this);
         });
+        def(AST_Template, function(compressor) {
+            return any(this.expressions, compressor);
+        });
         def(AST_This, return_false);
         def(AST_Try, function(compressor) {
             return any(this.body, compressor)
@@ -4673,7 +4676,7 @@ merge(Compressor.prototype, {
             return unary_side_effects[this.operator]
                 || this.expression.has_side_effects(compressor);
         });
-        def(AST_VarDef, function(compressor) {
+        def(AST_VarDef, function() {
             return this.value;
         });
     })(function(node, func) {
@@ -7015,6 +7018,11 @@ merge(Compressor.prototype, {
         def(AST_SymbolRef, function(compressor) {
             return this.is_declared(compressor) && can_drop_symbol(this) ? null : this;
         });
+        def(AST_Template, function(compressor, first_in_statement) {
+            var expressions = this.expressions;
+            if (expressions.length == 0) return null;
+            return make_sequence(this, expressions).drop_side_effect_free(compressor, first_in_statement);
+        });
         def(AST_This, return_null);
         def(AST_Unary, function(compressor, first_in_statement) {
             var exp = this.expression;

lib/output.js

@@ -1486,6 +1486,19 @@ function OutputStream(options) {
     DEFPRINT(AST_This, function(output) {
         output.print("this");
     });
+    DEFPRINT(AST_Template, function(output) {
+        var self = this;
+        if (self.tag) self.tag.print(output);
+        output.print("`");
+        for (var i = 0; i < self.expressions.length; i++) {
+            output.print(self.strings[i]);
+            output.print("${");
+            self.expressions[i].print(output);
+            output.print("}");
+        }
+        output.print(self.strings[i]);
+        output.print("`");
+    });
     DEFPRINT(AST_Constant, function(output) {
         output.print(this.value);
     });

lib/parse.js

@@ -113,7 +113,7 @@ var OPERATORS = makePredicate([
 var NEWLINE_CHARS = "\n\r\u2028\u2029";
 var OPERATOR_CHARS = "+-*&%=<>!?|~^";
 var PUNC_BEFORE_EXPRESSION = "[{(,;:";
-var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + ")}]";
+var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + "`)}]";
 var WHITESPACE_CHARS = NEWLINE_CHARS + " \u00a0\t\f\u000b\u200b\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF";
 var NON_IDENTIFIER_CHARS = makePredicate(characters("./'\"" + OPERATOR_CHARS + PUNC_CHARS + WHITESPACE_CHARS));
 
@@ -191,7 +191,28 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         regex_allowed   : false,
         comments_before : [],
         directives      : {},
-        directive_stack : []
+        directive_stack : [],
+        read_template   : with_eof_error("Unterminated template literal", function(strings) {
+            var s = "";
+            for (;;) {
+                var ch = next(true, true);
+                switch (ch) {
+                  case "\\":
+                    ch += next(true, true);
+                    break;
+                  case "`":
+                    strings.push(s);
+                    return;
+                  case "$":
+                    if (peek() == "{") {
+                        next();
+                        strings.push(s);
+                        return true;
+                    }
+                }
+                s += ch;
+            }
+        }),
     };
     var prev_was_dot = false;
 
@@ -816,6 +837,7 @@ function parse($TEXT, options) {
                 });
               case "[":
               case "(":
+              case "`":
                 return simple_statement();
               case ";":
                 S.in_directives = false;
@@ -1401,6 +1423,11 @@ function parse($TEXT, options) {
         var start = S.token;
         if (is("punc")) {
             switch (start.value) {
+              case "`":
+                var tmpl = template(null);
+                tmpl.start = start;
+                tmpl.end = prev();
+                return subscripts(tmpl, allow_calls);
               case "(":
                 next();
                 if (is("punc", ")")) {
@@ -1771,6 +1798,23 @@ function parse($TEXT, options) {
         }
     }
 
+    function template(tag) {
+        var read = S.input.context().read_template;
+        var strings = [];
+        var expressions = [];
+        while (read(strings)) {
+            next();
+            expressions.push(expression());
+            if (!is("punc", "}")) unexpected();
+        }
+        next();
+        return new AST_Template({
+            expressions: expressions,
+            strings: strings,
+            tag: tag,
+        });
+    }
+
     var subscripts = function(expr, allow_calls) {
         var start = expr.start;
         if (is("punc", ".")) {
@@ -1804,6 +1848,12 @@ function parse($TEXT, options) {
             mark_pure(call);
             return subscripts(call, true);
         }
+        if (is("punc", "`")) {
+            var tmpl = template(expr);
+            tmpl.start = expr.start;
+            tmpl.end = prev();
+            return subscripts(tmpl, allow_calls);
+        }
         return expr;
     };
 

lib/transform.js

@@ -201,6 +201,10 @@ TreeTransformer.prototype = new TreeWalker;
         if (self.key instanceof AST_Node) self.key = self.key.transform(tw);
         self.value = self.value.transform(tw);
     });
+    DEF(AST_Template, function(self, tw) {
+        if (self.tag) self.tag = self.tag.transform(tw);
+        self.expressions = do_list(self.expressions, tw);
+    });
 })(function(node, descend) {
     node.DEFMETHOD("transform", function(tw, in_list) {
         var x, y;

lib/utils.js

@@ -255,6 +255,8 @@ function first_in_statement(stack, arrow) {
             if (p.expressions[0] === node) continue;
         } else if (p instanceof AST_Statement) {
             return p.body === node;
+        } else if (p instanceof AST_Template) {
+            if (p.tag === node) continue;
         } else if (p instanceof AST_UnaryPostfix) {
             if (p.expression === node) continue;
         }