improve parser under "use strict" (#1836)

- `const` without value
- `delete` of expression
- redefining `arguments` or `eval`

extend `test/ufuzz.js`
- optionally generate "use strict"
- improve handling of test cases with syntax errors
- group IIFE generation
- generate bare anonymous functions
- workaround `console.log()` for `new function()`
- generate expressions with `this`


fixes #1810

lib/parse.js

@@ -629,8 +629,7 @@ function tokenizer($TEXT, filename, html5_comments, shebang) {
     }
 
     next_token.has_directive = function(directive) {
-        return S.directives[directive] !== undefined &&
+        return S.directives[directive] > 0;
-            S.directives[directive] > 0;
     }
 
     return next_token;
@@ -1033,29 +1032,32 @@ function parse($TEXT, options) {
         if (in_statement && !name)
             unexpected();
         expect("(");
-        return new ctor({
+        var argnames = [];
-            name: name,
+        for (var first = true; !is("punc", ")");) {
-            argnames: (function(first, a){
-                while (!is("punc", ")")) {
             if (first) first = false; else expect(",");
-                    a.push(as_symbol(AST_SymbolFunarg));
+            argnames.push(as_symbol(AST_SymbolFunarg));
         }
         next();
-                return a;
+        var loop = S.in_loop;
-            })(true, []),
+        var labels = S.labels;
-            body: (function(loop, labels){
         ++S.in_function;
         S.in_directives = true;
         S.input.push_directives_stack();
         S.in_loop = 0;
         S.labels = [];
-                var a = block_();
+        var body = block_();
+        if (S.input.has_directive("use strict")) {
+            if (name) strict_verify_symbol(name);
+            argnames.forEach(strict_verify_symbol);
+        }
         S.input.pop_directives_stack();
         --S.in_function;
         S.in_loop = loop;
         S.labels = labels;
-                return a;
+        return new ctor({
-            })(S.in_loop, S.labels)
+            name: name,
+            argnames: argnames,
+            body: body
         });
     };
 
@@ -1157,7 +1159,10 @@ function parse($TEXT, options) {
             a.push(new AST_VarDef({
                 start : S.token,
                 name  : as_symbol(in_const ? AST_SymbolConst : AST_SymbolVar),
-                value : is("operator", "=") ? (next(), expression(false, no_in)) : null,
+                value : is("operator", "=")
+                    ? (next(), expression(false, no_in))
+                    : in_const && S.input.has_directive("use strict")
+                        ? croak("Missing initializer in const declaration") : null,
                 end   : prev()
             }));
             if (!is("punc", ","))
@@ -1384,12 +1389,20 @@ function parse($TEXT, options) {
         });
     };
 
+    function strict_verify_symbol(sym) {
+        if (sym.name == "arguments" || sym.name == "eval")
+            croak("Unexpected " + sym.name + " in strict mode", sym.start.line, sym.start.col, sym.start.pos);
+    }
+
     function as_symbol(type, noerror) {
         if (!is("name")) {
             if (!noerror) croak("Name expected");
             return null;
         }
         var sym = _make_symbol(type);
+        if (S.input.has_directive("use strict") && sym instanceof AST_SymbolDeclaration) {
+            strict_verify_symbol(sym);
+        }
         next();
         return sym;
     };
@@ -1450,8 +1463,17 @@ function parse($TEXT, options) {
 
     function make_unary(ctor, token, expr) {
         var op = token.value;
-        if ((op == "++" || op == "--") && !is_assignable(expr))
+        switch (op) {
+          case "++":
+          case "--":
+            if (!is_assignable(expr))
                 croak("Invalid use of " + op + " operator", token.line, token.col, token.pos);
+            break;
+          case "delete":
+            if (expr instanceof AST_SymbolRef && S.input.has_directive("use strict"))
+                croak("Calling delete on expression not allowed in strict mode", expr.start.line, expr.start.col, expr.start.pos);
+            break;
+        }
         return new ctor({ operator: op, expression: expr });
     };
 

test/input/invalid/const.js

@@ -0,0 +1,8 @@
+function f() {
+    const a;
+}
+
+function g() {
+    "use strict";
+    const a;
+}

test/input/invalid/delete.js

@@ -0,0 +1,14 @@
+function f(x) {
+    delete 42;
+    delete (0, x);
+    delete null;
+    delete x;
+}
+
+function g(x) {
+    "use strict";
+    delete 42;
+    delete (0, x);
+    delete null;
+    delete x;
+}

test/input/invalid/function_1.js

@@ -0,0 +1,6 @@
+function f(arguments) {
+}
+
+function g(arguments) {
+    "use strict";
+}

test/input/invalid/function_2.js

@@ -0,0 +1,6 @@
+function arguments() {
+}
+
+function eval() {
+    "use strict";
+}

test/input/invalid/function_3.js

@@ -0,0 +1,6 @@
+!function eval() {
+}();
+
+!function arguments() {
+    "use strict";
+}();

test/input/invalid/try.js

@@ -0,0 +1,8 @@
+function f() {
+    try {} catch (eval) {}
+}
+
+function g() {
+    "use strict";
+    try {} catch (eval) {}
+}

test/input/invalid/var.js

@@ -0,0 +1,8 @@
+function f() {
+    var eval;
+}
+
+function g() {
+    "use strict";
+    var eval;
+}

test/mocha/cli.js

@@ -379,6 +379,111 @@ describe("bin/uglifyjs", function () {
            done();
        });
     });
+    it("Should throw syntax error (const a)", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/const.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/const.js:7,11",
+               "    const a;",
+               "           ^",
+               "ERROR: Missing initializer in const declaration"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (delete x)", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/delete.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/delete.js:13,11",
+               "    delete x;",
+               "           ^",
+               "ERROR: Calling delete on expression not allowed in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (function g(arguments))", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/function_1.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/function_1.js:4,11",
+               "function g(arguments) {",
+               "           ^",
+               "ERROR: Unexpected arguments in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (function eval())", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/function_2.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/function_2.js:4,9",
+               "function eval() {",
+               "         ^",
+               "ERROR: Unexpected eval in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (iife arguments())", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/function_3.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/function_3.js:4,10",
+               "!function arguments() {",
+               "          ^",
+               "ERROR: Unexpected arguments in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (catch(eval))", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/try.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/try.js:7,18",
+               "    try {} catch (eval) {}",
+               "                  ^",
+               "ERROR: Unexpected eval in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
+    it("Should throw syntax error (var eval)", function(done) {
+       var command = uglifyjscmd + ' test/input/invalid/var.js';
+
+       exec(command, function (err, stdout, stderr) {
+           assert.ok(err);
+           assert.strictEqual(stdout, "");
+           assert.strictEqual(stderr.split(/\n/).slice(0, 4).join("\n"), [
+               "Parse error at test/input/invalid/var.js:7,8",
+               "    var eval;",
+               "        ^",
+               "ERROR: Unexpected eval in strict mode"
+           ].join("\n"));
+           done();
+       });
+    });
     it("Should handle literal string as source map input", function(done) {
         var command = [
             uglifyjscmd,

test/sandbox.js

@@ -1,15 +1,35 @@
 var vm = require("vm");
 
+function safe_log(arg) {
+    if (arg) switch (typeof arg) {
+      case "function":
+        return arg.toString();
+      case "object":
+        if (/Error$/.test(arg.name)) return arg.toString();
+        arg.constructor.toString();
+        for (var key in arg) {
+            arg[key] = safe_log(arg[key]);
+        }
+    }
+    return arg;
+}
+
 var FUNC_TOSTRING = [
     "Function.prototype.toString = Function.prototype.valueOf = function() {",
-    "    var ids = [];",
+    "    var id = 0;",
     "    return function() {",
-    "        var i = ids.indexOf(this);",
+    '        if (this === Array) return "[Function: Array]";',
-    "        if (i < 0) {",
+    '        if (this === Object) return "[Function: Object]";',
-    "            i = ids.length;",
+    "        var i = this.name;",
-    "            ids.push(this);",
+    '        if (typeof i != "number") {',
+    "            i = ++id;",
+    '            Object.defineProperty(this, "name", {',
+    "                get: function() {",
+    "                    return i;",
     "                }",
-    '        return "[Function: __func_" + i + "__]";',
+    "            });",
+    "        }",
+    '        return "[Function: " + i + "]";',
     "    }",
     "}();",
 ].join("\n");
@@ -21,16 +41,14 @@ exports.run_code = function(code) {
     };
     try {
         vm.runInNewContext([
-            "!function() {",
             FUNC_TOSTRING,
+            "!function() {",
             code,
             "}();",
         ].join("\n"), {
             console: {
                 log: function() {
-                    return console.log.apply(console, [].map.call(arguments, function(arg) {
+                    return console.log.apply(console, [].map.call(arguments, safe_log));
-                        return typeof arg == "function" || arg && /Error$/.test(arg.name) ? arg.toString() : arg;
-                    }));
                 }
             }
         }, { timeout: 5000 });

test/ufuzz.js

@@ -49,6 +49,7 @@ var num_iterations = +process.argv[2] || 1/0;
 var verbose = false; // log every generated test
 var verbose_interval = false; // log every 100 generated tests
 var verbose_error = false;
+var use_strict = false;
 for (var i = 2; i < process.argv.length; ++i) {
     switch (process.argv[i]) {
       case '-v':
@@ -78,6 +79,9 @@ for (var i = 2; i < process.argv.length; ++i) {
         STMT_SECOND_LEVEL_OVERRIDE = STMT_ARG_TO_ID[name];
         if (!(STMT_SECOND_LEVEL_OVERRIDE >= 0)) throw new Error('Unknown statement name; use -? to get a list');
         break;
+      case '--use-strict':
+        use_strict = true;
+        break;
       case '--stmt-depth-from-func':
         STMT_COUNT_FROM_GLOBAL = false;
         break;
@@ -104,6 +108,7 @@ for (var i = 2; i < process.argv.length; ++i) {
         console.log('-r <int>: maximum recursion depth for generator (higher takes longer)');
         console.log('-s1 <statement name>: force the first level statement to be this one (see list below)');
         console.log('-s2 <statement name>: force the second level statement to be this one (see list below)');
+        console.log('--use-strict: generate "use strict"');
         console.log('--stmt-depth-from-func: reset statement depth counter at each function, counts from global otherwise');
         console.log('--only-stmt <statement names>: a comma delimited white list of statements that may be generated');
         console.log('--without-stmt <statement names>: a comma delimited black list of statements never to generate');
@@ -280,9 +285,19 @@ function rng(max) {
     return Math.floor(max * r);
 }
 
+function strictMode() {
+    return use_strict && rng(4) == 0 ? '"use strict";' : '';
+}
+
 function createTopLevelCode() {
-    if (rng(2) === 0) return createStatements(3, MAX_GENERATION_RECURSION_DEPTH, CANNOT_THROW, CANNOT_BREAK, CANNOT_CONTINUE, CANNOT_RETURN, 0);
+    return [
-    return createFunctions(rng(MAX_GENERATED_TOPLEVELS_PER_RUN) + 1, MAX_GENERATION_RECURSION_DEPTH, IN_GLOBAL, ANY_TYPE, CANNOT_THROW, 0);
+        strictMode(),
+        'var a = 100, b = 10, c = 0;',
+        rng(2) == 0
+        ? createStatements(3, MAX_GENERATION_RECURSION_DEPTH, CANNOT_THROW, CANNOT_BREAK, CANNOT_CONTINUE, CANNOT_RETURN, 0)
+        : createFunctions(rng(MAX_GENERATED_TOPLEVELS_PER_RUN) + 1, MAX_GENERATION_RECURSION_DEPTH, IN_GLOBAL, ANY_TYPE, CANNOT_THROW, 0),
+        'console.log(null, a, b, c);' // preceding `null` makes for a cleaner output (empty string still shows up etc)
+    ].join('\n');
 }
 
 function createFunctions(n, recurmax, inGlobal, noDecl, canThrow, stmtDepth) {
@@ -320,10 +335,22 @@ function createFunction(recurmax, inGlobal, noDecl, canThrow, stmtDepth) {
     var s = '';
     if (rng(5) === 0) {
         // functions with functions. lower the recursion to prevent a mess.
-        s = 'function ' + name + '(' + createParams() + '){' + createFunctions(rng(5) + 1, Math.ceil(recurmax * 0.7), NOT_GLOBAL, ANY_TYPE, canThrow, stmtDepth) + '}\n';
+        s = [
+            'function ' + name + '(' + createParams() + '){',
+            strictMode(),
+            createFunctions(rng(5) + 1, Math.ceil(recurmax * 0.7), NOT_GLOBAL, ANY_TYPE, canThrow, stmtDepth),
+            '}',
+            ''
+        ].join('\n');
     } else {
         // functions with statements
-        s = 'function ' + name + '(' + createParams() + '){' + createStatements(3, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth) + '}\n';
+        s = [
+            'function ' + name + '(' + createParams() + '){',
+            strictMode(),
+            createStatements(3, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+            '}',
+            ''
+        ].join('\n');
     }
 
     VAR_NAMES.length = namesLenBefore;
@@ -423,7 +450,7 @@ function createStatement(recurmax, canThrow, canBreak, canContinue, cannotReturn
         }
         return '{var expr' + loop + ' = ' + createExpression(recurmax, COMMA_OK, stmtDepth, canThrow) + '; ' + label.target + ' for (var key' + loop + ' in expr' + loop + ') {' + optElementVar + createStatement(recurmax, canThrow, canBreak, canContinue, cannotReturn, stmtDepth) + '}}';
       case STMT_SEMI:
-        return ';';
+        return use_strict && rng(20) === 0 ? '"use strict";' : ';';
       case STMT_EXPR:
         return createExpression(recurmax, COMMA_OK, stmtDepth, canThrow) + ';';
       case STMT_SWITCH:
@@ -486,6 +513,7 @@ function createStatement(recurmax, canThrow, canBreak, canContinue, cannotReturn
             // we have to do go through some trouble here to prevent leaking it
             var nameLenBefore = VAR_NAMES.length;
             var catchName = createVarName(MANDATORY);
+            if (catchName == 'this') catchName = 'a';
             var freshCatchName = VAR_NAMES.length !== nameLenBefore;
             s += ' catch (' + catchName + ') { ' + createStatements(3, recurmax, canThrow, canBreak, canContinue, cannotReturn, stmtDepth) + ' }';
             if (freshCatchName) VAR_NAMES.splice(nameLenBefore, 1); // remove catch name
@@ -563,37 +591,63 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) {
         return createExpression(recurmax, COMMA_OK, stmtDepth, canThrow);
       case p++:
         return createExpression(recurmax, noComma, stmtDepth, canThrow) + '?' + createExpression(recurmax, NO_COMMA, stmtDepth, canThrow) + ':' + createExpression(recurmax, noComma, stmtDepth, canThrow);
+      case p++:
       case p++:
         var nameLenBefore = VAR_NAMES.length;
         var name = createVarName(MAYBE); // note: this name is only accessible from _within_ the function. and immutable at that.
-        if (name === 'c') name = 'a';
+        if (name == 'c') name = 'a';
-        var s = '';
+        var s = [];
-        switch(rng(4)) {
+        switch (rng(5)) {
           case 0:
-            s = '(function ' + name + '(){' + createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth) + '})()';
+            s.push(
+                '(function ' + name + '(){',
+                strictMode(),
+                createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+                '})()'
+            );
             break;
           case 1:
-            s = '+function ' + name + '(){' + createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth) + '}()';
+            s.push(
+                '+function ' + name + '(){',
+                strictMode(),
+                createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+                '}()'
+            );
             break;
           case 2:
-            s = '!function ' + name + '(){' + createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth) + '}()';
+            s.push(
+                '!function ' + name + '(){',
+                strictMode(),
+                createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+                '}()'
+            );
+            break;
+          case 3:
+            s.push(
+                'void function ' + name + '(){',
+                strictMode(),
+                createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+                '}()'
+            );
             break;
           default:
-            s = 'void function ' + name + '(){' + createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth) + '}()';
+            if (rng(4) == 0) s.push('function ' + name + '(){');
+            else {
+                VAR_NAMES.push('this');
+                s.push('new function ' + name + '(){');
+            }
+            s.push(
+                strictMode(),
+                createStatements(rng(5) + 1, recurmax, canThrow, CANNOT_BREAK, CANNOT_CONTINUE, CAN_RETURN, stmtDepth),
+                '}'
+            );
             break;
         }
         VAR_NAMES.length = nameLenBefore;
-        return s;
+        return s.join('\n');
       case p++:
       case p++:
         return createTypeofExpr(recurmax, stmtDepth, canThrow);
-      case p++:
-        return [
-            'new function() {',
-            rng(2) ? '' : createExpression(recurmax, COMMA_OK, stmtDepth, canThrow) + ';',
-            'return ' + createExpression(recurmax, COMMA_OK, stmtDepth, canThrow) + ';',
-            '}'
-        ].join('\n');
       case p++:
       case p++:
         // more like a parser test but perhaps comment nodes mess up the analysis?
@@ -715,22 +769,24 @@ function _createBinaryExpr(recurmax, noComma, stmtDepth, canThrow) {
 function _createSimpleBinaryExpr(recurmax, noComma, stmtDepth, canThrow) {
     // intentionally generate more hardcore ops
     if (--recurmax < 0) return createValue();
+    var assignee, expr;
     switch (rng(30)) {
       case 0:
         return '(c = c + 1, ' + _createSimpleBinaryExpr(recurmax, noComma, stmtDepth, canThrow) + ')';
       case 1:
         return '(' + createUnarySafePrefix() + '(' + _createSimpleBinaryExpr(recurmax, noComma, stmtDepth, canThrow) + '))';
       case 2:
-        var assignee = getVarName();
+        assignee = getVarName();
+        if (assignee == 'this') assignee = 'a';
         return '(' + assignee + createAssignment() + _createBinaryExpr(recurmax, noComma, stmtDepth, canThrow) + ')';
       case 3:
-        var assignee = getVarName();
+        assignee = getVarName();
-        var expr = '(' + assignee + '[' + createExpression(recurmax, COMMA_OK, stmtDepth, canThrow)
+        expr = '(' + assignee + '[' + createExpression(recurmax, COMMA_OK, stmtDepth, canThrow)
             + ']' + createAssignment() + _createBinaryExpr(recurmax, noComma, stmtDepth, canThrow) + ')';
         return canThrow && rng(10) == 0 ? expr : '(' + assignee + ' && ' + expr + ')';
       case 4:
-        var assignee = getVarName();
+        assignee = getVarName();
-        var expr = '(' + assignee + '.' + getDotKey() + createAssignment()
+        expr = '(' + assignee + '.' + getDotKey() + createAssignment()
             + _createBinaryExpr(recurmax, noComma, stmtDepth, canThrow) + ')';
         return canThrow && rng(10) == 0 ? expr : '(' + assignee + ' && ' + expr + ')';
       default:
@@ -890,6 +946,12 @@ function log(options) {
     } else {
         console.log("// !!! uglify failed !!!");
         console.log(uglify_code.stack);
+        if (typeof original_result != "string") {
+            console.log();
+            console.log();
+            console.log("original stacktrace:");
+            console.log(original_result.stack);
+        }
     }
     console.log("minify(options):");
     options = JSON.parse(options);
@@ -901,6 +963,10 @@ function log(options) {
     }
 }
 
+var fallback_options = [ JSON.stringify({
+    compress: false,
+    mangle: false
+}) ];
 var minify_options = require("./ufuzz.json").map(JSON.stringify);
 var original_code, original_result;
 var uglify_code, uglify_result, ok;
@@ -911,13 +977,9 @@ for (var round = 1; round <= num_iterations; round++) {
     loops = 0;
     funcs = 0;
 
-    original_code = [
+    original_code = createTopLevelCode();
-        "var a = 100, b = 10, c = 0;",
+    original_result = sandbox.run_code(original_code);
-        createTopLevelCode(),
+    (typeof original_result != "string" ? fallback_options : minify_options).forEach(function(options) {
-        "console.log(null, a, b, c);" // preceding `null` makes for a cleaner output (empty string still shows up etc)
-    ].join("\n");
-
-    minify_options.forEach(function(options) {
         try {
             uglify_code = UglifyJS.minify(original_code, JSON.parse(options)).code;
         } catch (e) {
@@ -926,9 +988,10 @@ for (var round = 1; round <= num_iterations; round++) {
 
         ok = typeof uglify_code == "string";
         if (ok) {
-            original_result = sandbox.run_code(original_code);
             uglify_result = sandbox.run_code(uglify_code);
             ok = sandbox.same_stdout(original_result, uglify_result);
+        } else if (typeof original_result != "string") {
+            ok = uglify_code.name == original_result.name;
         }
         if (verbose || (verbose_interval && !(round % INTERVAL_COUNT)) || !ok) log(options);
         else if (verbose_error && typeof original_result != "string") {