fix corner cases with function inlining (#4613)

fixes #4612

lib/compress.js

@@ -4330,6 +4330,7 @@ merge(Compressor.prototype, {
                     if (ignore_side_effects) {
                         fn.walk(scan_modified);
                         var found = false;
+                        fn.evaluating = true;
                         walk_body(fn, new TreeWalker(function(node) {
                             if (found) return true;
                             if (node instanceof AST_Return) {
@@ -4340,6 +4341,7 @@ merge(Compressor.prototype, {
                             }
                             if (node instanceof AST_Scope && node !== fn) return true;
                         }));
+                        delete fn.evaluating;
                         if (!found) return;
                     }
                     return this;
@@ -4999,6 +5001,7 @@ merge(Compressor.prototype, {
             var stat = self.body[i];
             if (stat instanceof AST_Directive) continue;
             if (stat instanceof AST_Return) {
+                if (i != self.body.length - 1) break;
                 var call = stat.value;
                 if (!call || call.TYPE != "Call") break;
                 if (call.is_expr_pure(compressor)) break;
@@ -8323,9 +8326,12 @@ merge(Compressor.prototype, {
                             return parent ? maintain_this_binding(compressor, parent, node, arg) : arg;
                         }
                     })));
+                    var save_inlined = fn.inlined;
+                    if (exp !== fn) fn.inlined = true;
                     var node = make_sequence(self, args.filter(function(arg) {
                         return arg;
                     })).optimize(compressor);
+                    fn.inlined = save_inlined;
                     node = maintain_this_binding(compressor, compressor.parent(), compressor.self(), node);
                     if (replacing || best_of_expression(node, self) === node) {
                         refs.forEach(function(ref) {
@@ -9788,8 +9794,8 @@ merge(Compressor.prototype, {
                         single_use = fixed.is_constant_expression(self.scope);
                         if (single_use == "f") {
                             var scope = self.scope;
-                            do if (is_defun(scope) || is_function(scope)) {
-                                scope.inlined = true;
+                            do {
+                                if (is_defun(scope) || is_function(scope)) scope.inlined = true;
                             } while (scope = scope.parent_scope);
                         }
                     } else if (fixed.name && fixed.name.name == "await" && is_async(fixed)) {

test/compress/functions.js

@@ -5283,3 +5283,127 @@ issue_4471: {
         "PASS",
     ]
 }
+
+issue_4612_1: {
+    options = {
+        evaluate: true,
+        inline: true,
+        reduce_vars: true,
+        side_effects: true,
+        unused: true,
+    }
+    input: {
+        console.log(function() {
+            function f() {
+                return g();
+            }
+            function g(a) {
+                return a || f();
+            }
+            return g("PASS");
+        }());
+    }
+    expect: {
+        console.log("PASS");
+    }
+    expect_stdout: "PASS"
+}
+
+issue_4612_2: {
+    options = {
+        evaluate: true,
+        inline: true,
+        reduce_vars: true,
+        side_effects: true,
+        unused: true,
+    }
+    input: {
+        console.log(function() {
+            function fn() {
+                return h();
+            }
+            function g() {
+                return fn();
+            }
+            function h(a) {
+                return a || fn();
+            }
+            return h("PASS");
+        }());
+    }
+    expect: {
+        console.log("PASS");
+    }
+    expect_stdout: "PASS"
+}
+
+issue_4612_3: {
+    options = {
+        inline: true,
+        reduce_vars: true,
+    }
+    input: {
+        console.log(typeof function() {
+            return g();
+            function f() {
+                return g;
+            }
+            function g() {
+                {
+                    return f;
+                }
+            }
+        }());
+    }
+    expect: {
+        console.log(typeof function() {
+            return g();
+            function f() {
+                return g;
+            }
+            function g() {
+                return f;
+            }
+        }());
+    }
+    expect_stdout: "function"
+}
+
+issue_4612_4: {
+    options = {
+        booleans: true,
+        evaluate: true,
+        reduce_vars: true,
+    }
+    input: {
+        console.log(function() {
+            function f() {
+                return h();
+            }
+            function g() {
+                {
+                    return h();
+                }
+            }
+            function h() {
+                {
+                    return g();
+                }
+            }
+        }());
+    }
+    expect: {
+        console.log(function() {
+            function f() {
+                return h();
+            }
+            function g() {
+                return h();
+            }
+            function h() {
+                return g();
+            }
+        }());
+    }
+    expect_stdout: "undefined"
+}