improve AST tests & tools (#4873)

.github/workflows/build.yml

@@ -8,7 +8,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        options: [ '-mb braces', '--ie8 -c', '-mc', '--toplevel -mc passes=3,pure_getters,unsafe' ]
+        options:
+          - '-mb braces'
+          - '--ie8 -c'
+          - '-mc'
+          - '--toplevel -mc passes=3,pure_getters,unsafe'
         script:
           - acorn.sh
           - bootstrap.sh

bin/uglifyjs

@@ -235,10 +235,11 @@ if (options.mangle && options.mangle.properties) {
         });
     }
 }
-if (output == "ast" || output == "spidermonkey") options.output = {
+if (/^ast|spidermonkey$/.test(output)) {
-    ast: true,
+    if (typeof options.output != "object") options.output = {};
-    code: false,
+    options.output.ast = true;
-};
+    options.output.code = false;
+}
 if (options.parse && (options.parse.acorn || options.parse.spidermonkey)
     && options.sourceMap && options.sourceMap.content == "inline") {
     fatal("inline source map only works with built-in parser");
@@ -265,7 +266,9 @@ if (specified["self"]) {
     paths = UglifyJS.FILES;
 }
 if (specified["in-situ"]) {
-    if (output || specified["reduce-test"] || specified["self"]) fatal("incompatible options specified");
+    if (output && output != "spidermonkey" || specified["reduce-test"] || specified["self"]) {
+        fatal("incompatible options specified");
+    }
     paths.forEach(function(name) {
         print(name);
         if (/^ast|spidermonkey$/.test(name)) fatal("invalid file name specified");
@@ -313,6 +316,7 @@ function run() {
             if (options.parse.acorn) {
                 files = convert_ast(function(toplevel, name) {
                     return require("acorn").parse(files[name], {
+                        allowHashBang: true,
                         ecmaVersion: "latest",
                         locations: true,
                         program: toplevel,
@@ -413,7 +417,17 @@ function run() {
     } else if (output == "spidermonkey") {
         print(JSON.stringify(result.ast.to_mozilla_ast(), null, 2));
     } else if (output) {
-        fs.writeFileSync(output, result.code);
+        var code;
+        if (result.ast) {
+            var opts = {};
+            for (var name in options.output) {
+                if (!/^ast|code$/.test(name)) opts[name] = options.output[name];
+            }
+            code = UglifyJS.AST_Node.from_mozilla_ast(result.ast.to_mozilla_ast()).print_to_string(opts);
+        } else {
+            code = result.code;
+        }
+        fs.writeFileSync(output, code);
         if (result.map) fs.writeFileSync(output + ".map", result.map);
     } else {
         print(result.code);

lib/minify.js

@@ -204,27 +204,29 @@ function minify(files, options) {
         if (options.mangle && options.mangle.properties) mangle_properties(toplevel, options.mangle.properties);
         if (timings) timings.output = Date.now();
         var result = {};
-        if (options.output.ast) {
+        var output = defaults(options.output, {
-            result.ast = toplevel;
+            ast: false,
-        }
+            code: true,
-        if (!HOP(options.output, "code") || options.output.code) {
+        });
+        if (output.ast) result.ast = toplevel;
+        if (output.code) {
             if (options.sourceMap) {
-                options.output.source_map = SourceMap(options.sourceMap);
+                output.source_map = SourceMap(options.sourceMap);
                 if (options.sourceMap.includeSources) {
                     if (files instanceof AST_Toplevel) {
                         throw new Error("original source content unavailable");
                     } else for (var name in files) if (HOP(files, name)) {
-                        options.output.source_map.setSourceContent(name, files[name]);
+                        output.source_map.setSourceContent(name, files[name]);
                     }
                 }
             }
-            delete options.output.ast;
+            delete output.ast;
-            delete options.output.code;
+            delete output.code;
-            var stream = OutputStream(options.output);
+            var stream = OutputStream(output);
             toplevel.print(stream);
             result.code = stream.get();
             if (options.sourceMap) {
-                result.map = options.output.source_map.toString();
+                result.map = output.source_map.toString();
                 var url = options.sourceMap.url;
                 if (url) {
                     result.code = result.code.replace(/\n\/\/# sourceMappingURL=\S+\s*$/, "");

lib/mozilla-ast.js

@@ -220,7 +220,7 @@
                 start: my_start_token(M),
                 end: my_end_token(M),
                 key: key,
-                value: from_moz(M[M.shorthand ? "key" : "value"]),
+                value: from_moz(M.value),
             };
             if (M.kind == "init") return new (M.method ? AST_ObjectMethod : AST_ObjectKeyVal)(args);
             args.value = new AST_Accessor(args.value);
@@ -395,6 +395,20 @@
                 path: M.source.value,
             });
         },
+        ImportExpression: function(M) {
+            var start = my_start_token(M);
+            var arg = from_moz(M.source);
+            return new AST_Call({
+                start: start,
+                end: my_end_token(M),
+                expression: new AST_SymbolRef({
+                    start: start,
+                    end: arg.start,
+                    name: "import",
+                }),
+                args: [ arg ],
+            });
+        },
         VariableDeclaration: function(M) {
             return new ({
                 const: AST_Const,
@@ -462,7 +476,7 @@
             } while (p.type == "ArrayPattern"
                 || p.type == "AssignmentPattern" && p.left === FROM_MOZ_STACK[level + 1]
                 || p.type == "ObjectPattern"
-                || p.type == "Property" && p[p.shorthand ? "key" : "value"] === FROM_MOZ_STACK[level + 1]
+                || p.type == "Property" && p.value === FROM_MOZ_STACK[level + 1]
                 || p.type == "VariableDeclarator" && p.id === FROM_MOZ_STACK[level + 1]);
             var ctor = AST_SymbolRef;
             switch (p.type) {
@@ -731,18 +745,9 @@
     });
 
     def_to_moz(AST_ExportDefault, function To_Moz_ExportDefaultDeclaration(M) {
-        var decl = to_moz(M.body);
-        switch (decl.type) {
-          case "ClassExpression":
-            decl.type = "ClassDeclaration";
-            break;
-          case "FunctionExpression":
-            decl.type = "FunctionDeclaration";
-            break;
-        }
         return {
             type: "ExportDefaultDeclaration",
-            declaration: decl,
+            declaration: to_moz(M.body),
         };
     });
 
@@ -1158,11 +1163,15 @@
 
     var FROM_MOZ_STACK = null;
 
-    function from_moz(node) {
+    function from_moz(moz) {
-        FROM_MOZ_STACK.push(node);
+        FROM_MOZ_STACK.push(moz);
-        var ret = node != null ? MOZ_TO_ME[node.type](node) : null;
+        var node = null;
+        if (moz) {
+            if (!HOP(MOZ_TO_ME, moz.type)) throw new Error("Unsupported type: " + moz.type);
+            node = MOZ_TO_ME[moz.type](moz);
+        }
         FROM_MOZ_STACK.pop();
-        return ret;
+        return node;
     }
 
     AST_Node.from_mozilla_ast = function(node) {

test/mozilla-ast.js

@@ -5,23 +5,23 @@ var acorn = require("acorn");
 var ufuzz = require("./ufuzz");
 var UglifyJS = require("..");
 
-function try_beautify(code) {
+function beautify(ast) {
-    var beautified = UglifyJS.minify(code, {
+    var beautified = UglifyJS.minify(ast, {
         compress: false,
         mangle: false,
         output: {
             beautify: true,
             braces: true,
-        }
+        },
+    });
+    if (beautified.error) return beautified;
+    return UglifyJS.minify(beautified.code, {
+        compress: false,
+        mangle: false,
+        output: {
+            ast: true,
+        },
     });
-    if (beautified.error) {
-        console.log("// !!! beautify failed !!!");
-        console.log(beautified.error.stack);
-        console.log(code);
-    } else {
-        console.log("// (beautified)");
-        console.log(beautified.code);
-    }
 }
 
 function validate(ast) {
@@ -35,22 +35,55 @@ function validate(ast) {
     return UglifyJS.minify(ast, {
         compress: false,
         mangle: false,
+        output: {
+            ast: true,
+        },
         validate: true,
     });
 }
 
-function fuzzy(code) {
+function patch_import(code) {
     return code.replace(/\bimport\s*\{\s*\}\s*from\s*(['"])/g, "import$1")
         .replace(/\b(import\b.*?)\s*,\s*\{\s*\}\s*(from\s*['"])/g, "$1 $2");
 }
 
-function test(original, estree, description) {
+function equals(input, transformed) {
-    var transformed = validate(UglifyJS.AST_Node.from_mozilla_ast(estree));
+    if (input.code === transformed.code) return true;
-    if (transformed.error || original !== transformed.code && fuzzy(original) !== fuzzy(transformed.code)) {
+    return patch_import(input.code) === patch_import(transformed.code);
+}
+
+function test(input, to_moz, description, skip_on_error, beautified) {
+    try {
+        var ast = UglifyJS.AST_Node.from_mozilla_ast(to_moz(input));
+    } catch (e) {
+        if (skip_on_error) return true;
+        console.log("//=============================================================");
+        console.log("//", description, "failed... round", round);
+        console.log(e);
+        console.log("// original code");
+        if (beautified === true) console.log("// (beautified)");
+        console.log(input.code);
+        return false;
+    }
+    var transformed = validate(ast);
+    if (transformed.error || !equals(input, transformed)) {
+        if (!beautified) {
+            beautified = beautify(input.ast);
+            if (!beautified.error) {
+                beautified.raw = beautified.code;
+                if (!test(beautified, to_moz, description, skip_on_error, true)) return false;
+            }
+        }
         console.log("//=============================================================");
         console.log("// !!!!!! Failed... round", round);
         console.log("// original code");
-        try_beautify(original);
+        if (beautified.error) {
+            console.log("// !!! beautify failed !!!");
+            console.log(beautified.error.stack);
+        } else if (beautified === true) {
+            console.log("// (beautified)");
+        }
+        console.log(input.raw);
         console.log();
         console.log();
         console.log("//-------------------------------------------------------------");
@@ -58,7 +91,15 @@ function test(original, estree, description) {
         if (transformed.error) {
             console.log(transformed.error.stack);
         } else {
-            try_beautify(transformed.code);
+            beautified = beautify(transformed.ast);
+            if (beautified.error) {
+                console.log("// !!! beautify failed !!!");
+                console.log(beautified.error.stack);
+                console.log(transformed.code);
+            } else {
+                console.log("// (beautified)");
+                console.log(beautified.code);
+            }
         }
         console.log("!!!!!! Failed... round", round);
         return false;
@@ -73,41 +114,33 @@ for (var round = 1; round <= num_iterations; round++) {
     process.stdout.write(round + " of " + num_iterations + "\r");
     var code = ufuzz.createTopLevelCode();
     minify_options.forEach(function(options) {
-        var input = options ? UglifyJS.minify(code, JSON.parse(options)).code : code;
+        var ok = true;
-        var uglified = UglifyJS.minify(input, {
+        var input = UglifyJS.minify(options ? UglifyJS.minify(code, JSON.parse(options)).code : code, {
             compress: false,
             mangle: false,
             output: {
                 ast: true,
             },
         });
-        var ok = true;
+        input.raw = options ? input.code : code;
-        try {
+        if (input.error) {
-            var estree = uglified.ast.to_mozilla_ast();
-        } catch (e) {
             ok = false;
             console.log("//=============================================================");
-            console.log("// AST_Node.to_mozilla_ast() failed... round", round);
+            console.log("// minify() failed... round", round);
-            console.log(e);
+            console.log(input.error);
             console.log("// original code");
-            console.log(input);
+            console.log(code);
         }
-        if (ok) ok = test(uglified.code, estree, "AST_Node.to_mozilla_ast()");
+        if (ok) ok = test(input, function(input) {
-        if (ok) try {
+            return input.ast.to_mozilla_ast();
-            ok = test(uglified.code, acorn.parse(input, {
+        }, "AST_Node.to_mozilla_ast()");
+        if (ok) ok = test(input, function(input) {
+            return acorn.parse(input.raw, {
                 ecmaVersion: "latest",
                 locations: true,
                 sourceType: "module",
-            }), "acorn.parse()");
+            });
-        } catch (e) {
+        }, "acorn.parse()", !ufuzz.verbose);
-            if (ufuzz.verbose) {
-                console.log("//=============================================================");
-                console.log("// acorn parser failed... round", round);
-                console.log(e);
-                console.log("// original code");
-                console.log(input);
-            }
-        }
         if (!ok) process.exit(1);
     });
 }

test/release/rollup-ts.sh

@@ -15,7 +15,7 @@ minify_in_situ() {
     for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'`
     do
         echo "$i"
-        node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \
+        node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \
             | uglify-js $UGLIFY_OPTIONS -o "$i"
     done
 }

test/release/sucrase.sh

@@ -19,7 +19,7 @@ minify_in_situ() {
     for i in `find $DIRS -type f -name '*.ts' | grep -v '\.d\.ts'`
     do
         echo "$i"
-        node_modules/.bin/esbuild --loader=ts --target=node14 < "$i" \
+        node_modules/.bin/esbuild --loader=ts --target=es2019 < "$i" \
             | uglify-js $UGLIFY_OPTIONS -o "$i"
     done
 }