Template fixes

* Fixes #1147: template strings not obeying -b ascii_only true
* Allow evaluation of template expressions by adding optimizers and
  walkers
* Make sure tagged templates are never changed
* Remove template tokenizer in parser, add template tokenizer in
  tokenizer. It is using a brace counter to track brace position of
  templates
* Add tokens `template_head` and `template_substitution` but parsing
  tokens stays mostly the same
* Do not output strings anymore in AST_TemplateString, instead use
  AST_TemplateSegment
* Fix parsing tagged templates, allowing multiple templates behind
  as spec allows this

These changes don't influence tagged templates because raw content
may influence code execution, however they are safe to do in normal
templates:
* Allow basic string concatenation of templates where possible
* Allow custom character escape style similar to strings, except in
  tagged templates

Note that expressions are still compressed in tagged templates.

Optional things that may be improved later:
* Custom quote style for templates if it doesn't have expressions.
  Making it obey the quote_style option if this is the case.

lib/ast.js

@@ -538,7 +538,7 @@ var AST_PrefixedTemplateString = DEFNODE("PrefixedTemplateString", "template_str
 var AST_TemplateString = DEFNODE("TemplateString", "segments", {
     $documentation: "A template string literal",
     $propdoc: {
-        segments: "[string|AST_Expression]* One or more segments. They can be the parts that are evaluated, or the raw string parts."
+        segments: "[AST_TemplateSegment|AST_Expression]* One or more segments, starting with AST_TemplateSegment. AST_Expression may follow AST_TemplateSegment, but each AST_Expression must be followed by AST_TemplateSegment."
     },
     _walk: function(visitor) {
         return visitor._visit(this, function(){
@@ -551,6 +551,14 @@ var AST_TemplateString = DEFNODE("TemplateString", "segments", {
     }
 });
 
+var AST_TemplateSegment = DEFNODE("TemplateSegment", "value raw", {
+    $documentation: "A segment of a template string literal",
+    $propdoc: {
+        value: "Content of the segment",
+        raw: "Raw content of the segment"
+    }
+});
+
 /* -----[ JUMPS ]----- */
 
 var AST_Jump = DEFNODE("Jump", null, {

lib/compress.js

@@ -952,6 +952,9 @@ merge(Compressor.prototype, {
     (function (def){
         def(AST_Node, function(){ return false });
         def(AST_String, function(){ return true });
+        def(AST_TemplateString, function(){
+            return this.segments.length === 1;
+        });
         def(AST_UnaryPrefix, function(){
             return this.operator == "typeof";
         });
@@ -1056,6 +1059,10 @@ merge(Compressor.prototype, {
         def(AST_Constant, function(){
             return this.getValue();
         });
+        def(AST_TemplateString, function() {
+            if (this.segments.length !== 1) throw def;
+            return this.segments[0].value;
+        });
         def(AST_UnaryPrefix, function(compressor){
             var e = this.expression;
             switch (this.operator) {
@@ -2988,4 +2995,37 @@ merge(Compressor.prototype, {
         return self;
     });
 
+    OPT(AST_TemplateString, function(self, compressor){
+        if (!compressor.option("evaluate")
+        || compressor.parent() instanceof AST_PrefixedTemplateString)
+            return self;
+
+        var segments = [];
+        for (var i = 0; i < self.segments.length; i++) {
+            if (self.segments[i] instanceof AST_Node) {
+                var result = self.segments[i].evaluate(compressor);
+                // No result[1] means nothing to stringify
+                if (result.length === 1) {
+                    segments.push(result[0]);
+                    continue;
+                }
+                // Evaluate length
+                if (result[0].print_to_string().length + 3 /* ${} */ < (result[1]+"").length) {
+                    segments.push(result[0]);
+                    continue;
+                }
+                // There should always be a previous and next segment if segment is a node
+                segments[segments.length - 1].value = segments[segments.length - 1].value + result[1] + self.segments[++i].value;
+            } else {
+                segments.push(self.segments[i]);
+            }
+        }
+        self.segments = segments;
+
+        return self;
+    });
+
+    OPT(AST_PrefixedTemplateString, function(self, compressor){
+        return self;
+    });
 })();

lib/output.js

@@ -125,7 +125,22 @@ function OutputStream(options) {
         function quote_double() {
             return '"' + str.replace(/\x22/g, '\\"') + '"';
         }
+        function quote_template() {
+            if (!options.ascii_only) {
+                str = str.replace(/\\(n|r|u2028|u2029)/g, function(s, c) {
+                    switch(c) {
+                        case "n": return "\n";
+                        case "r": return "\r";
+                        case "u2028": return "\u2028";
+                        case "u2029": return "\u2029";
+                    }
+                    return s;
+                });
+            }
+            return '`' + str.replace(/`/g, '\\`') + '`';
+        }
         if (options.ascii_only) str = to_ascii(str);
+        if (quote === "`") return quote_template();
         switch (options.quote_style) {
           case 1:
             return quote_single();
@@ -387,6 +402,10 @@ function OutputStream(options) {
             }
             print(encoded);
         },
+        print_template_string_chars: function(str) {
+            var encoded = encode_string(str, '`');
+            return print(encoded.substr(1, encoded.length - 2));
+        },
         encode_string   : encode_string,
         next_indent     : next_indent,
         with_indent     : with_indent,
@@ -889,14 +908,18 @@ function OutputStream(options) {
         self.template_string.print(output);
     });
     DEFPRINT(AST_TemplateString, function(self, output) {
+        var is_tagged = output.parent() instanceof AST_PrefixedTemplateString;
+
         output.print("`");
         for (var i = 0; i < self.segments.length; i++) {
-            if (typeof self.segments[i] !== "string") {
+            if (!(self.segments[i] instanceof AST_TemplateSegment)) {
                 output.print("${");
                 self.segments[i].print(output);
                 output.print("}");
+            } else if (is_tagged) {
+                output.print(self.segments[i].raw);
             } else {
-                output.print(self.segments[i]);
+                output.print_template_string_chars(self.segments[i].value);
             }
         }
         output.print("`");

lib/parse.js

@@ -120,7 +120,7 @@ var PUNC_AFTER_EXPRESSION = makePredicate(characters(";]),:"));
 
 var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:"));
 
-var PUNC_CHARS = makePredicate(characters("[]{}(),;:`"));
+var PUNC_CHARS = makePredicate(characters("[]{}(),;:"));
 
 var REGEXP_MODIFIERS = makePredicate(characters("gmsiy"));
 
@@ -269,6 +269,8 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         tokcol          : 0,
         newline_before  : false,
         regex_allowed   : false,
+        brace_counter   : 0,
+        template_braces : [],
         comments_before : [],
         directives      : {},
         directive_stack : []
@@ -487,6 +489,40 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
         return tok;
     });
 
+    var read_template_characters = with_eof_error("SyntaxError: Unterminated template", function(begin){
+        if (begin) {
+            S.template_braces.push(S.brace_counter);
+        }
+        var content = "", raw = "", ch, tok;
+        next();
+        while ((ch = next(true)) !== "`") {
+            if (ch === "$" && peek() === "{") {
+                next();
+                S.brace_counter++;
+                tok = token(begin ? "template_head" : "template_substitution", content);
+                tok.begin = begin;
+                tok.raw = raw;
+                tok.end = false;
+                return tok;
+            }
+
+            raw += ch;
+            if (ch === "\\") {
+                var tmp = S.pos;
+                ch = read_escaped_char();
+                raw += S.text.substr(tmp, S.pos - tmp);
+            }
+
+            content += ch;
+        }
+        S.template_braces.pop();
+        tok = token(begin ? "template_head" : "template_substitution", content);
+        tok.begin = begin;
+        tok.raw = raw;
+        tok.end = true;
+        return tok;
+    });
+
     function skip_line_comment(type) {
         var regex_allowed = S.regex_allowed;
         var i = find_eol(), ret;
@@ -688,6 +724,16 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
                   return tok;
               }
               case 61: return handle_eq_sign();
+              case 96: return read_template_characters(true);
+              case 123:
+                S.brace_counter++;
+                break;
+              case 125:
+                S.brace_counter--;
+                if (S.template_braces.length > 0
+                    && S.template_braces[S.template_braces.length - 1] === S.brace_counter)
+                    return read_template_characters(false);
+                break;
             }
             if (is_digit(code)) return read_num();
             if (PUNC_CHARS(ch)) return token("punc", next());
@@ -939,6 +985,7 @@ function parse($TEXT, options) {
                 });
             }
             return stat;
+          case "template_head":
           case "num":
           case "regexp":
           case "operator":
@@ -960,7 +1007,6 @@ function parse($TEXT, options) {
                 });
               case "[":
               case "(":
-              case "`":
                 return simple_statement();
               case ";":
                 S.in_directives = false;
@@ -1600,8 +1646,6 @@ function parse($TEXT, options) {
                 return subscripts(array_(), allow_calls);
               case "{":
                 return subscripts(object_or_object_destructuring_(), allow_calls);
-              case "`":
-                return subscripts(template_string(), allow_calls);
             }
             unexpected();
         }
@@ -1619,6 +1663,9 @@ function parse($TEXT, options) {
             cls.end = prev();
             return subscripts(cls, allow_calls);
         }
+        if (is("template_head")) {
+            return subscripts(template_string(), allow_calls);
+        }
         if (ATOMIC_START_TOKEN[S.token.type]) {
             return subscripts(as_atom_node(), allow_calls);
         }
@@ -1626,28 +1673,29 @@ function parse($TEXT, options) {
     };
 
     function template_string() {
-        var tokenizer_S = S.input, start = S.token, segments = [], segment = "", ch;
+        var segments = [], start = S.token;
 
-        while ((ch = tokenizer_S.next()) !== "`")  {
+        segments.push(new AST_TemplateSegment({
-            if (ch === "$" && tokenizer_S.peek() === "{") {
+            start: S.token,
-                segments.push(segment); segment = "";
+            raw: S.token.raw,
-                tokenizer_S.next();
+            value: S.token.value,
+            end: S.token
+        }));
+        while (S.token.end === false) {
             next();
             segments.push(expression());
-                if (!is("punc", "}")) {
+
-                    // force error message
+            if (!is_token("template_substitution")) {
-                    expect("}");
+                unexpected();
-                }
-                continue;
-            }
-            segment += ch;
-            if (ch === "\\") {
-                segment += tokenizer_S.next();
-            }
             }
 
-        segments.push(segment);
+            segments.push(new AST_TemplateSegment({
-
+                start: S.token,
+                raw: S.token.raw,
+                value: S.token.value,
+                end: S.token
+            }));
+        }
         next();
 
         return new AST_TemplateString({
@@ -2033,6 +2081,13 @@ function parse($TEXT, options) {
                 end        : prev()
             }), true);
         }
+        if (is("template_head")) {
+            return subscripts(new AST_PrefixedTemplateString({
+                start: start,
+                prefix: expr,
+                template_string: template_string()
+            }), allow_calls);
+        }
         return expr;
     };
 
@@ -2189,13 +2244,6 @@ function parse($TEXT, options) {
             });
             return arrow_function(expr);
         }
-        if ((expr instanceof AST_SymbolRef || expr instanceof AST_PropAccess) && is("punc", "`")) {
-            return new AST_PrefixedTemplateString({
-                start: start,
-                prefix: expr,
-                template_string: template_string()
-            })
-        }
         if (commas && is("punc", ",")) {
             next();
             return new AST_Seq({

lib/transform.js

@@ -231,4 +231,16 @@ TreeTransformer.prototype = new TreeWalker;
         self.expression = self.expression.transform(tw);
     });
 
+    _(AST_TemplateString, function(self, tw) {
+        for (var i = 0; i < self.segments.length; i++) {
+            if (!(self.segments[i] instanceof AST_TemplateSegment)) {
+                self.segments[i] = self.segments[i].transform(tw);
+            }
+        }
+    });
+
+    _(AST_PrefixedTemplateString, function(self, tw) {
+        self.template_string = self.template_string.transform(tw);
+    });
+
 })();

test/compress/harmony.js

@@ -78,24 +78,6 @@ typeof_arrow_functions: {
     expect_exact: "var foo=\"function\";"
 }
 
-template_strings: {
-    input: {
-        ``;
-        `xx\`x`;
-        `${ foo + 2 }`;
-        ` foo ${ bar + `baz ${ qux }` }`;
-    }
-    expect_exact: "``;`xx\\`x`;`${foo+2}`;` foo ${bar+`baz ${qux}`}`;";
-}
-
-template_string_prefixes: {
-    input: {
-        String.raw`foo`;
-        foo `bar`;
-    }
-    expect_exact: "String.raw`foo`;foo`bar`;";
-}
-
 destructuring_arguments: {
     input: {
         (function ( a ) { });

test/compress/template-string.js

@@ -0,0 +1,331 @@
+template_strings: {
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        ``;
+        `xx\`x`;
+        `${ foo + 2 }`;
+        ` foo ${ bar + `baz ${ qux }` }`;
+    }
+    expect_exact: "``;`xx\\`x`;`${foo+2}`;` foo ${bar+`baz ${qux}`}`;";
+}
+
+template_string_prefixes: {
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        String.raw`foo`;
+        foo `bar`;
+    }
+    expect_exact: "String.raw`foo`;foo`bar`;";
+}
+
+template_strings_ascii_only: {
+    beautify = {
+        ascii_only: true,
+        quote_style: 3
+    }
+    input: {
+        var foo = `foo
+        bar
+        ↂωↂ`;
+        var bar = `\``;
+    }
+    expect_exact: "var foo=`foo\\n        bar\\n        \\u2182\\u03c9\\u2182`;var bar=`\\``;"
+}
+
+template_strings_without_ascii_only: {
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        var foo = `foo
+        bar
+        ↂωↂ`
+    }
+    expect_exact: "var foo=`foo\n        bar\n        ↂωↂ`;"
+}
+
+template_string_with_constant_expression: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        var foo = `${4 + 4} equals 4 + 4`;
+    }
+    expect: {
+        var foo = `8 equals 4 + 4`;
+    }
+}
+
+template_string_with_predefined_constants: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        var foo = `This is ${undefined}`;
+        var bar = `This is ${NaN}`;
+        var baz = `This is ${null}`;
+        var foofoo = `This is ${Infinity}`;
+        var foobar = "This is ${1/0}";
+        var foobaz = 'This is ${1/0}';
+        var barfoo = "This is ${NaN}";
+        var bazfoo = "This is ${null}";
+        var bazbaz = `This is ${1/0}`;
+        var barbar = `This is ${0/0}`;
+        var barbar = "This is ${0/0}";
+        var barber = 'This is ${0/0}';
+
+        var a = `${4**11}`; // 8 in template vs 7 chars - 4194304
+        var b = `${4**12}`; // 8 in template vs 8 chars - 16777216
+        var c = `${4**14}`; // 8 in template vs 9 chars - 268435456
+    }
+    expect: {
+        var foo = `This is undefined`;
+        var bar = `This is NaN`;
+        var baz = `This is null`;
+        var foofoo = `This is ${1/0}`;
+        var foobar = "This is ${1/0}";
+        var foobaz = 'This is ${1/0}';
+        var barfoo = "This is ${NaN}";
+        var bazfoo = "This is ${null}";
+        var bazbaz = `This is ${1/0}`;
+        var barbar = `This is NaN`;
+        var barbar = "This is ${0/0}";
+        var barber = 'This is ${0/0}';
+
+        var a = `4194304`;
+        var b = `16777216`; // Potential for further concatentation
+        var c = `${4**14}`; // Not worth converting
+    }
+}
+
+template_string_evaluate_with_many_segments: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        var foo = `Hello ${guest()}, welcome to ${location()}${"."}`;
+        var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`;
+        var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`;
+        var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`;
+    }
+    expect: {
+        var foo = `Hello ${guest()}, welcome to ${location()}.`;
+        var bar = `1234567890`;
+        var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`;
+        var buzz = `1${foobar()}2${foobar()}3${foobar()}`;
+    }
+}
+
+template_string_with_many_segments: {
+    beautify = {
+        quote_style: 3
+    }
+    input: {
+        var foo = `Hello ${guest()}, welcome to ${location()}${"."}`;
+        var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`;
+        var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`;
+        var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`;
+    }
+    expect: {
+        var foo = `Hello ${guest()}, welcome to ${location()}${"."}`;
+        var bar = `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`;
+        var baz = `${foobar()}${foobar()}${foobar()}${foobar()}`;
+        var buzz = `${1}${foobar()}${2}${foobar()}${3}${foobar()}`;
+    }
+}
+
+template_string_to_normal_string: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 0
+    }
+    input: {
+        var foo = `This is ${undefined}`;
+        var bar = "Decimals " + `${1}${2}${3}${4}${5}${6}${7}${8}${9}${0}`;
+    }
+    expect: {
+        var foo = `This is undefined`;
+        var bar = "Decimals 1234567890";
+    }
+}
+
+template_concattenating_string: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 3 // Yes, keep quotes
+    }
+    input: {
+        var foo = "Have a nice " + `day. ${`day. ` + `day.`}`;
+        var bar = "Have a nice " + `${day()}`;
+    }
+    expect: {
+        var foo = "Have a nice day. day. day.";
+        var bar = "Have a nice " + `${day()}`;
+    }
+}
+
+evaluate_nested_templates: {
+    options = {
+        evaluate: true
+    }
+    beautify = {
+        quote_style: 0
+    }
+    input: {
+        var baz = `${`${`${`foo`}`}`}`;
+    }
+    expect: {
+        var baz = `foo`;
+    }
+}
+
+enforce_double_quotes: {
+    beautify = {
+        quote_style: 1
+    }
+    input: {
+        var foo = `Hello world`;
+        var bar = `Hello ${'world'}`;
+        var baz = `Hello ${world()}`;
+    }
+    expect: {
+        var foo = `Hello world`;
+        var bar = `Hello ${"world"}`;
+        var baz = `Hello ${world()}`;
+    }
+}
+
+enforce_single_quotes: {
+    beautify = {
+        quote_style: 2
+    }
+    input: {
+        var foo = `Hello world`;
+        var bar = `Hello ${"world"}`;
+        var baz = `Hello ${world()}`;
+    }
+    expect: {
+        var foo = `Hello world`;
+        var bar = `Hello ${'world'}`;
+        var baz = `Hello ${world()}`;
+    }
+}
+
+enforce_double_quotes_and_evaluate: {
+    beautify = {
+        quote_style: 1
+    }
+    options = {
+        evaluate: true
+    }
+    input: {
+        var foo = `Hello world`;
+        var bar = `Hello ${'world'}`;
+        var baz = `Hello ${world()}`;
+    }
+    expect: {
+        var foo = `Hello world`;
+        var bar = `Hello world`;
+        var baz = `Hello ${world()}`;
+    }
+}
+
+enforce_single_quotes_and_evaluate: {
+    beautify = {
+        quote_style: 2
+    }
+    options = {
+        evaluate: true
+    }
+    input: {
+        var foo = `Hello world`;
+        var bar = `Hello ${"world"}`;
+        var baz = `Hello ${world()}`;
+    }
+    expect: {
+        var foo = `Hello world`;
+        var bar = `Hello world`;
+        var baz = `Hello ${world()}`;
+    }
+}
+
+respect_inline_script: {
+    beautify = {
+        inline_script: true,
+        quote_style: 3
+    }
+    input: {
+        var foo = `</script>${content}`;
+        var bar = `<!--`;
+        var baz = `-->`;
+    }
+    expect_exact: "var foo=`<\\/script>${content}`;var bar=`\\x3c!--`;var baz=`--\\x3e`;";
+}
+
+do_not_optimize_tagged_template_1: {
+    beautify = {
+        quote_style: 0
+    }
+    options = {
+        evaluate: true
+    }
+    input: {
+        var foo = tag`Shall not be optimized. ${"But " + "this " + "is " + "fine."}`;
+        var bar = tag`Don't even mind changing my quotes!`;
+    }
+    expect_exact:
+        'var foo=tag`Shall not be optimized. ${"But this is fine."}`;var bar=tag`Don\'t even mind changing my quotes!`;';
+}
+
+do_not_optimize_tagged_template_2: {
+    options = {
+        evaluate: true
+    }
+    input: {
+        var foo = tag`test` + " something out";
+    }
+    expect_exact: 'var foo=tag`test`+" something out";';
+}
+
+keep_raw_content_in_tagged_template: {
+    options = {
+        evaluate: true
+    }
+    input: {
+        var foo = tag`\u0020\u{20}\u{00020}\x20\40\040 `;
+    }
+    expect_exact: "var foo=tag`\\u0020\\u{20}\\u{00020}\\x20\\40\\040 `;";
+}
+
+allow_chained_templates: {
+    input: {
+        var foo = tag`a``b``c``d`;
+    }
+    expect: {
+        var foo = tag`a``b``c``d`;
+    }
+}
+
+check_escaped_chars: {
+    input: {
+        var foo = `\u0020\u{20}\u{00020}\x20\40\040 `;
+    }
+    expect_exact: "var foo=`       `;";
+}

test/mocha/template-string.js

@@ -0,0 +1,33 @@
+var assert = require("assert");
+var uglify = require("../../");
+
+describe("Template string", function() {
+    it("Should not accept invalid sequences", function() {
+        var tests = [
+            // Stress invalid expression
+            "var foo = `Hello ${]}`",
+            "var foo = `Test 123 ${>}`",
+            "var foo = `Blah ${;}`",
+
+            // Stress invalid template_substitution after expression
+            "var foo = `Blablabla ${123 456}`",
+            "var foo = `Blub ${123;}`",
+            "var foo = `Bleh ${a b}`"
+        ];
+
+        var exec = function(test) {
+            return function() {
+                uglify.parse(test);
+            }
+        };
+
+        var fail = function(e) {
+            return e instanceof uglify.JS_Parse_Error
+                && /^SyntaxError: Unexpected token: /.test(e.message);
+        };
+
+        for (var i = 0; i < tests.length; i++) {
+            assert.throws(exec(tests[i]), fail, tests[i]);
+        }
+    });
+});