improve --reduce-test (#3742)

- ignore difference in error messages
- improve readability on trailing whitespace differences
- improve performance & quality via `console.log()` insertions

test/input/reduce/label.reduced.js

@@ -1,8 +1,7 @@
+// (beautified)
 var o = this;
 
-for (var k in o) {
+for (var k in o) {}
-    0;
-}
 
 var a;
 

test/input/reduce/setter.reduced.js

@@ -1,3 +1,4 @@
+// (beautified)
 console.log(function f(a) {
     ({
         set p(v) {

test/input/reduce/unsafe_math.reduced.js

@@ -1,3 +1,4 @@
+// (beautified)
 var b = 0;
 
 var expr2 = (0 - 1 - .1 - .1).toString();

test/mocha/reduce.js

@@ -53,8 +53,8 @@ describe("test/reduce.js", function() {
         });
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
-            "// Can't reproduce test failure with minify options provided:",
+            "// Can't reproduce test failure",
-            "// {",
+            "// minify options: {",
             '//   "toplevel": true',
             "// }",
         ].join("\n"));
@@ -70,8 +70,8 @@ describe("test/reduce.js", function() {
         });
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
-            "// Can't reproduce test failure with minify options provided:",
+            "// Can't reproduce test failure",
-            "// {",
+            "// minify options: {",
             '//   "compress": {',
             '//     "toplevel": true',
             "//   }",
@@ -89,8 +89,8 @@ describe("test/reduce.js", function() {
         });
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
-            "// Can't reproduce test failure with minify options provided:",
+            "// Can't reproduce test failure",
-            "// {",
+            "// minify options: {",
             '//   "mangle": {',
             '//     "toplevel": true',
             "//   }",
@@ -101,11 +101,8 @@ describe("test/reduce.js", function() {
         var result = reduce_test("throw 0 / 0;");
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
-            "// Can't reproduce test failure with minify options provided:",
+            "// Can't reproduce test failure",
-            "// {",
+            "// minify options: {}",
-            '//   "compress": {},',
-            '//   "mangle": false',
-            "// }",
         ].join("\n"));
     });
     it("Should print correct output for irreducible test case", function() {
@@ -121,6 +118,7 @@ describe("test/reduce.js", function() {
         });
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
+            "// (beautified)",
             "console.log(function f(a) {",
             "    return f.length;",
             "}());",
@@ -169,6 +167,7 @@ describe("test/reduce.js", function() {
         });
         if (result.error) throw result.error;
         assert.strictEqual(result.code, [
+            "// (beautified)",
             code,
             "// output: 0.8",
             "// 1.6",
@@ -189,11 +188,41 @@ describe("test/reduce.js", function() {
     it("Should reduce infinite loops with reasonable performance", function() {
         if (semver.satisfies(process.version, "0.10")) return;
         this.timeout(120000);
+        var result = reduce_test("while (/9/.test(1 - .8));", {
+            compress: {
+                unsafe_math: true,
+            },
+            mangle: false,
+        });
+        if (result.error) throw result.error;
+        assert.strictEqual(result.code.replace(/ timed out after [0-9]+ms/, " timed out."), [
+            "// (beautified)",
+            "while (/9/.test(1 - .8)) {}",
+            "// output: Error: Script execution timed out.",
+            "// minify: ",
+            "// options: {",
+            '//   "compress": {',
+            '//     "unsafe_math": true',
+            "//   },",
+            '//   "mangle": false',
+            "// }",
+        ].join("\n"));
+    });
+    it("Should ignore difference in Error.message", function() {
+        var result = reduce_test("null[function() {\n}];");
+        if (result.error) throw result.error;
+        assert.strictEqual(result.code, (semver.satisfies(process.version, "0.10") ? [
+            "// Can't reproduce test failure",
+            "// minify options: {}",
+        ] : [
+            "// No differences except in error message",
+            "// minify options: {}",
+        ]).join("\n"));
+    });
+    it("Should report trailing whitespace difference in stringified format", function() {
         var code = [
-            "var a = 9007199254740992, b = 1;",
+            "for (var a in (1 - .8).toString()) {",
-            "",
+            "    console.log();",
-            "while (a++ + (1 - b) < a) {",
-            "    0;",
             "}",
         ].join("\n");
         var result = reduce_test(code, {
@@ -203,14 +232,16 @@ describe("test/reduce.js", function() {
             mangle: false,
         });
         if (result.error) throw result.error;
-        assert.strictEqual(result.code.replace(/ timed out after [0-9]+ms/, " timed out."), [
+        assert.strictEqual(result.code, [
+            "// (beautified)",
             code,
-            "// output: ",
+            "// (stringified)",
-            "// minify: Error: Script execution timed out.",
+            '// output: "\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n"',
+            '// minify: "\\n\\n\\n"',
             "// options: {",
             '//   "compress": {',
             '//     "unsafe_math": true',
-            "//   },",
+            '//   },',
             '//   "mangle": false',
             "// }",
         ].join("\n"));

test/reduce.js

@@ -20,7 +20,7 @@ var sandbox = require("./sandbox");
 
 module.exports = function reduce_test(testcase, minify_options, reduce_options) {
     if (testcase instanceof U.AST_Node) testcase = testcase.print_to_string();
-    minify_options = minify_options || { compress: {}, mangle: false };
+    minify_options = minify_options || {};
     reduce_options = reduce_options || {};
     var max_iterations = reduce_options.max_iterations || 1000;
     var max_timeout = reduce_options.max_timeout || 10000;
@@ -36,16 +36,29 @@ module.exports = function reduce_test(testcase, minify_options, reduce_options)
     if (!differs) {
         // same stdout result produced when minified
         return {
-            code: "// Can't reproduce test failure with minify options provided:"
+            code: [
-                + "\n// " + to_comment(minify_options_json)
+                "// Can't reproduce test failure",
+                "// minify options: " + to_comment(minify_options_json)
+            ].join("\n")
         };
     } else if (differs.timed_out) {
         return {
-            code: "// Can't reproduce test failure within " + max_timeout + "ms:"
+            code: [
-                + "\n// " + to_comment(minify_options_json)
+                "// Can't reproduce test failure within " + max_timeout + "ms",
+                "// minify options: " + to_comment(minify_options_json)
+            ].join("\n")
         };
     } else if (differs.error) {
         return differs;
+    } else if (is_error(differs.unminified_result)
+        && is_error(differs.minified_result)
+        && differs.unminified_result.name == differs.minified_result.name) {
+        return {
+            code: [
+                "// No differences except in error message",
+                "// minify options: " + to_comment(minify_options_json)
+            ].join("\n")
+        };
     } else {
         max_timeout = Math.min(100 * differs.elapsed, max_timeout);
         // Replace expressions with constants that will be parsed into
@@ -71,6 +84,7 @@ module.exports = function reduce_test(testcase, minify_options, reduce_options)
             // quick ignores
             if (node instanceof U.AST_Accessor) return;
             if (node instanceof U.AST_Directive) return;
+            if (!in_list && node instanceof U.AST_EmptyStatement) return;
             if (node instanceof U.AST_Label) return;
             if (node instanceof U.AST_LabelRef) return;
             if (!in_list && node instanceof U.AST_SymbolDeclaration) return;
@@ -112,10 +126,25 @@ module.exports = function reduce_test(testcase, minify_options, reduce_options)
             }
             else if (node instanceof U.AST_Binary) {
                 CHANGED = true;
-                return [
+                var permute = ((node.start._permute += step) * steps | 0) % 4;
+                var expr = [
                     node.left,
                     node.right,
-                ][ ((node.start._permute += step) * steps | 0) % 2 ];
+                ][ permute & 1 ];
+                if (permute < 2) return expr;
+                // wrap with console.log()
+                return new U.AST_Call({
+                    expression: new U.AST_Dot({
+                        expression: new U.AST_SymbolRef({
+                            name: "console",
+                            start: {},
+                        }),
+                        property: "log",
+                        start: {},
+                    }),
+                    args: [ expr ],
+                    start: {},
+                });
             }
             else if (node instanceof U.AST_Catch || node instanceof U.AST_Finally) {
                 // drop catch or finally block
@@ -357,15 +386,11 @@ module.exports = function reduce_test(testcase, minify_options, reduce_options)
             }
 
             // replace this node
-            var newNode = U.parse(REPLACEMENTS[node.start._permute % REPLACEMENTS.length | 0], {
+            var newNode = is_statement(node) ? new U.AST_EmptyStatement({
+                start: {},
+            }) : U.parse(REPLACEMENTS[node.start._permute % REPLACEMENTS.length | 0], {
                 expression: true,
             });
-            if (is_statement(node)) {
-                newNode = new U.AST_SimpleStatement({
-                    body: newNode,
-                    start: {},
-                });
-            }
             newNode.start._permute = ++node.start._permute;
             CHANGED = true;
             return newNode;
@@ -445,29 +470,62 @@ module.exports = function reduce_test(testcase, minify_options, reduce_options)
                 console.error("// reduce test pass " + pass + ": " + testcase.length + " bytes");
             }
         }
-        testcase = U.minify(testcase, {
+        testcase = try_beautify(result_cache, testcase, minify_options, differs.unminified_result, max_timeout);
-            compress: false,
+        var lines = [ "" ];
-            mangle: false,
+        var unminified_result = strip_color_codes(differs.unminified_result);
-            output: {
+        var minified_result = strip_color_codes(differs.minified_result);
-                beautify: true,
+        if (trim_trailing_whitespace(unminified_result) == trim_trailing_whitespace(minified_result)) {
-                braces: true,
+            lines.push(
-                comments: true,
+                "// (stringified)",
-            },
+                "// output: " + JSON.stringify(unminified_result),
-        });
+                "// minify: " + JSON.stringify(minified_result)
-        testcase.code += [
+            );
-            "",
+        } else {
-            "// output: " + to_comment(differs.unminified_result),
+            lines.push(
-            "// minify: " + to_comment(differs.minified_result),
+                "// output: " + to_comment(unminified_result),
-            "// options: " + to_comment(minify_options_json),
+                "// minify: " + to_comment(minified_result)
-        ].join("\n").replace(/\u001b\[\d+m/g, "");
+            );
+        }
+        lines.push("// options: " + to_comment(minify_options_json));
+        testcase.code += lines.join("\n");
         return testcase;
     }
 };
 
+function strip_color_codes(value) {
+    return ("" + value).replace(/\u001b\[\d+m/g, "");
+}
+
 function to_comment(value) {
     return ("" + value).replace(/\n/g, "\n// ");
 }
 
+function trim_trailing_whitespace(value) {
+    return ("" + value).replace(/\s+$/, "");
+}
+
+function try_beautify(result_cache, testcase, minify_options, expected, timeout) {
+    var result = U.minify(testcase, {
+        compress: false,
+        mangle: false,
+        output: {
+            beautify: true,
+            braces: true,
+            comments: true,
+        },
+    });
+    if (result.error) return {
+        code: testcase,
+    };
+    var toplevel = sandbox.has_toplevel(minify_options);
+    var actual = run_code(result_cache, result.code, toplevel, timeout);
+    if (!sandbox.same_stdout(expected, actual)) return {
+        code: testcase,
+    };
+    result.code = "// (beautified)\n" + result.code;
+    return result;
+}
+
 function has_exit(fn) {
     var found = false;
     var tw = new U.TreeWalker(function(node) {