support template literals (#4601)

lib/ast.js

@@ -1418,6 +1418,34 @@ var AST_This = DEFNODE("This", null, {
     },
 }, AST_Symbol);
 
+var AST_Template = DEFNODE("Template", "expressions strings tag", {
+    $documentation: "A template literal, i.e. tag`str1${expr1}...strN${exprN}strN+1`",
+    $propdoc: {
+        expressions: "[AST_Node*] the placeholder expressions",
+        strings: "[string*] the interpolating text segments",
+        tag: "[AST_Node] tag function, or null if absent",
+    },
+    walk: function(visitor) {
+        var node = this;
+        visitor.visit(node, function() {
+            if (node.tag) node.tag.walk(visitor);
+            node.expressions.forEach(function(expr) {
+                expr.walk(visitor);
+            });
+        });
+    },
+    _validate: function() {
+        if (this.expressions.length + 1 != this.strings.length) {
+            throw new Error("malformed template with " + this.expressions.length + " placeholder(s) but " + this.strings.length + " text segment(s)");
+        }
+        must_be_expressions(this, "expressions");
+        this.strings.forEach(function(string) {
+            if (typeof string != "string") throw new Error("strings must contain string");
+        });
+        if (this.tag != null) must_be_expression(this, "tag");
+    },
+});
+
 var AST_Constant = DEFNODE("Constant", null, {
     $documentation: "Base class for all constants",
 });

lib/compress.js

@@ -4663,6 +4663,9 @@ merge(Compressor.prototype, {
         def(AST_SymbolRef, function(compressor) {
             return !this.is_declared(compressor) || !can_drop_symbol(this);
         });
+        def(AST_Template, function(compressor) {
+            return any(this.expressions, compressor);
+        });
         def(AST_This, return_false);
         def(AST_Try, function(compressor) {
             return any(this.body, compressor)
@@ -4673,7 +4676,7 @@ merge(Compressor.prototype, {
             return unary_side_effects[this.operator]
                 || this.expression.has_side_effects(compressor);
         });
-        def(AST_VarDef, function(compressor) {
+        def(AST_VarDef, function() {
             return this.value;
         });
     })(function(node, func) {
@@ -7015,6 +7018,11 @@ merge(Compressor.prototype, {
         def(AST_SymbolRef, function(compressor) {
             return this.is_declared(compressor) && can_drop_symbol(this) ? null : this;
         });
+        def(AST_Template, function(compressor, first_in_statement) {
+            var expressions = this.expressions;
+            if (expressions.length == 0) return null;
+            return make_sequence(this, expressions).drop_side_effect_free(compressor, first_in_statement);
+        });
         def(AST_This, return_null);
         def(AST_Unary, function(compressor, first_in_statement) {
             var exp = this.expression;

lib/output.js

@@ -1486,6 +1486,19 @@ function OutputStream(options) {
     DEFPRINT(AST_This, function(output) {
         output.print("this");
     });
+    DEFPRINT(AST_Template, function(output) {
+        var self = this;
+        if (self.tag) self.tag.print(output);
+        output.print("`");
+        for (var i = 0; i < self.expressions.length; i++) {
+            output.print(self.strings[i]);
+            output.print("${");
+            self.expressions[i].print(output);
+            output.print("}");
+        }
+        output.print(self.strings[i]);
+        output.print("`");
+    });
     DEFPRINT(AST_Constant, function(output) {
         output.print(this.value);
     });

lib/parse.js

@@ -113,7 +113,7 @@ var OPERATORS = makePredicate([
 var NEWLINE_CHARS = "\n\r\u2028\u2029";
 var OPERATOR_CHARS = "+-*&%=<>!?|~^";
 var PUNC_BEFORE_EXPRESSION = "[{(,;:";
-var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + ")}]";
+var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + "`)}]";
 var WHITESPACE_CHARS = NEWLINE_CHARS + " \u00a0\t\f\u000b\u200b\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF";
 var NON_IDENTIFIER_CHARS = makePredicate(characters("./'\"" + OPERATOR_CHARS + PUNC_CHARS + WHITESPACE_CHARS));
 
@@ -191,7 +191,28 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         regex_allowed   : false,
         comments_before : [],
         directives      : {},
-        directive_stack : []
+        directive_stack : [],
+        read_template   : with_eof_error("Unterminated template literal", function(strings) {
+            var s = "";
+            for (;;) {
+                var ch = next(true, true);
+                switch (ch) {
+                  case "\\":
+                    ch += next(true, true);
+                    break;
+                  case "`":
+                    strings.push(s);
+                    return;
+                  case "$":
+                    if (peek() == "{") {
+                        next();
+                        strings.push(s);
+                        return true;
+                    }
+                }
+                s += ch;
+            }
+        }),
     };
     var prev_was_dot = false;
 
@@ -816,6 +837,7 @@ function parse($TEXT, options) {
                 });
               case "[":
               case "(":
+              case "`":
                 return simple_statement();
               case ";":
                 S.in_directives = false;
@@ -1401,6 +1423,11 @@ function parse($TEXT, options) {
         var start = S.token;
         if (is("punc")) {
             switch (start.value) {
+              case "`":
+                var tmpl = template(null);
+                tmpl.start = start;
+                tmpl.end = prev();
+                return subscripts(tmpl, allow_calls);
               case "(":
                 next();
                 if (is("punc", ")")) {
@@ -1771,6 +1798,23 @@ function parse($TEXT, options) {
         }
     }
 
+    function template(tag) {
+        var read = S.input.context().read_template;
+        var strings = [];
+        var expressions = [];
+        while (read(strings)) {
+            next();
+            expressions.push(expression());
+            if (!is("punc", "}")) unexpected();
+        }
+        next();
+        return new AST_Template({
+            expressions: expressions,
+            strings: strings,
+            tag: tag,
+        });
+    }
+
     var subscripts = function(expr, allow_calls) {
         var start = expr.start;
         if (is("punc", ".")) {
@@ -1804,6 +1848,12 @@ function parse($TEXT, options) {
             mark_pure(call);
             return subscripts(call, true);
         }
+        if (is("punc", "`")) {
+            var tmpl = template(expr);
+            tmpl.start = expr.start;
+            tmpl.end = prev();
+            return subscripts(tmpl, allow_calls);
+        }
         return expr;
     };
 

lib/transform.js

@@ -201,6 +201,10 @@ TreeTransformer.prototype = new TreeWalker;
         if (self.key instanceof AST_Node) self.key = self.key.transform(tw);
         self.value = self.value.transform(tw);
     });
+    DEF(AST_Template, function(self, tw) {
+        if (self.tag) self.tag = self.tag.transform(tw);
+        self.expressions = do_list(self.expressions, tw);
+    });
 })(function(node, descend) {
     node.DEFMETHOD("transform", function(tw, in_list) {
         var x, y;

lib/utils.js

@@ -255,6 +255,8 @@ function first_in_statement(stack, arrow) {
             if (p.expressions[0] === node) continue;
         } else if (p instanceof AST_Statement) {
             return p.body === node;
+        } else if (p instanceof AST_Template) {
+            if (p.tag === node) continue;
         } else if (p instanceof AST_UnaryPostfix) {
             if (p.expression === node) continue;
         }

test/compress/templates.js

@@ -0,0 +1,82 @@
+simple: {
+    input: {
+        console.log(`foo
+        bar\nbaz`);
+    }
+    expect_exact: "console.log(`foo\n        bar\\nbaz`);"
+    expect_stdout: [
+        "foo",
+        "        bar",
+        "baz",
+    ]
+    node_version: ">=4"
+}
+
+placeholder: {
+    input: {
+        console.log(`foo ${ function(a, b) {
+            return a * b;
+        }(6, 7) }`);
+    }
+    expect_exact: "console.log(`foo ${function(a,b){return a*b}(6,7)}`);"
+    expect_stdout: "foo 42"
+    node_version: ">=4"
+}
+
+nested: {
+    input: {
+        console.log(`P${`A${"S"}`}S`);
+    }
+    expect_exact: 'console.log(`P${`A${"S"}`}S`);'
+    expect_stdout: "PASS"
+    node_version: ">=4"
+}
+
+tagged: {
+    input: {
+        console.log(String.raw`foo\nbar`);
+    }
+    expect_exact: "console.log(String.raw`foo\\nbar`);"
+    expect_stdout: "foo\\nbar"
+    node_version: ">=4"
+}
+
+tagged_chain: {
+    input: {
+        function f(strings) {
+            return strings.join("") || f;
+        }
+        console.log(f```${42}``pass`.toUpperCase());
+    }
+    expect_exact: 'function f(strings){return strings.join("")||f}console.log(f```${42}``pass`.toUpperCase());'
+    expect_stdout: "PASS"
+    node_version: ">=4"
+}
+
+malformed_escape: {
+    input: {
+        (function(s) {
+            s.forEach((c, i) => console.log(i, c, s.raw[i]));
+            return () => console.log(arguments);
+        })`\uFo${42}`();
+    }
+    expect_exact: "(function(s){s.forEach((c,i)=>console.log(i,c,s.raw[i]));return()=>console.log(arguments)})`\\uFo${42}`();"
+    expect_stdout: true
+    node_version: ">=4"
+}
+
+evaluate: {
+    options = {
+        evaluate: true,
+    }
+    input: {
+        console.log(`foo ${ function(a, b) {
+            return a * b;
+        }(6, 7) }`);
+    }
+    expect: {
+        console.log(`foo ${42}`);
+    }
+    expect_stdout: "foo 42"
+    node_version: ">=4"
+}

test/mocha/templates.js

@@ -0,0 +1,64 @@
+var assert = require("assert");
+var run_code = require("../sandbox").run_code;
+var semver = require("semver");
+var UglifyJS = require("../node");
+
+describe("Template literals", function() {
+    it("Should reject invalid literal", function() {
+        [
+            "`foo\\`",
+            "`foo${bar`",
+            "`foo${bar}",
+        ].forEach(function(input) {
+            assert.throws(function() {
+                UglifyJS.parse(input);
+            }, function(e) {
+                return e instanceof UglifyJS.JS_Parse_Error
+                    && e.message === "Unterminated template literal";
+            }, input);
+        });
+    });
+    it("Should reject invalid expression", function() {
+        [
+            "`foo${bar;}`",
+            "`foo${42bar}`",
+        ].forEach(function(input) {
+            assert.throws(function() {
+                UglifyJS.parse(input);
+            }, function(e) {
+                return e instanceof UglifyJS.JS_Parse_Error;
+            }, input);
+        });
+    });
+    it("Should process line-break characters correctly", function() {
+        [
+            // native line breaks
+            [ "`foo\nbar`", "`foo\nbar`" ],
+            [ "`foo\rbar`", "`foo\rbar`" ],
+            [ "`foo\r\nbar`", "`foo\nbar`" ],
+            [ "`foo\r\n\rbar`", "`foo\n\rbar`" ],
+            // escaped line breaks
+            [ "`foo\\nbar`", "`foo\\nbar`" ],
+            [ "`foo\\rbar`", "`foo\\rbar`" ],
+            [ "`foo\r\\nbar`", "`foo\r\\nbar`" ],
+            [ "`foo\\r\nbar`", "`foo\\r\nbar`" ],
+            [ "`foo\\r\\nbar`", "`foo\\r\\nbar`" ],
+            // continuation
+            [ "`foo\\\nbar`", "`foo\\\nbar`" ],
+            [ "`foo\\\rbar`", "`foo\\\rbar`" ],
+            [ "`foo\\\r\nbar`", "`foo\\\nbar`" ],
+            [ "`foo\\\r\n\rbar`", "`foo\\\n\rbar`" ],
+            [ "`foo\\\\nbar`", "`foo\\\\nbar`" ],
+            [ "`foo\\\\rbar`", "`foo\\\\rbar`" ],
+            [ "`foo\\\\r\nbar`", "`foo\\\\r\nbar`" ],
+        ].forEach(function(test) {
+            var input = "console.log(" + test[0] + ");";
+            var result = UglifyJS.minify(input);
+            if (result.error) throw result.error;
+            var expected = "console.log(" + test[1] + ");";
+            assert.strictEqual(result.code, expected, test[0]);
+            if (semver.satisfies(process.version, "<4")) return;
+            assert.strictEqual(run_code(result.code), run_code(input), test[0]);
+        });
+    });
+});

test/ufuzz/index.js

@@ -146,6 +146,7 @@ var SUPPORT = function(matrix) {
     rest_object: "var {...a} = {};",
     spread: "[...[]];",
     spread_object: "({...0});",
+    template: "``",
     trailing_comma: "function f(a,) {}",
 });
 
@@ -1038,6 +1039,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
       case p++:
         return rng(2) + " === 1 ? a : b";
       case p++:
+        if (SUPPORT.template && rng(20) == 0) return createTemplateLiteral(recurmax, stmtDepth, canThrow);
       case p++:
         return createValue();
       case p++:
@@ -1298,6 +1300,28 @@ function createArrayLiteral(recurmax, stmtDepth, canThrow) {
     return "[" + arr.join(", ") + "]";
 }
 
+function createTemplateLiteral(recurmax, stmtDepth, canThrow) {
+    recurmax--;
+    var s = [];
+    addText();
+    for (var i = rng(6); --i >= 0;) {
+        s.push("${", createExpression(recurmax, COMMA_OK, stmtDepth, canThrow), "}");
+        addText();
+    }
+    return (rng(10) ? "`" : "String.raw`") + s.join(rng(5) ? "" : "\n") + "`";
+
+    function addText() {
+        while (rng(5) == 0) s.push([
+            " ",
+            "$",
+            "}",
+            "\\`",
+            "\\\\",
+            "tmpl",
+        ][rng(6)]);
+    }
+}
+
 var SAFE_KEYS = [
     "length",
     "foo",