fix corner cases with instanceof (#5509)

- enhance `evaluate`

test/compress/awaits.js

@@ -1293,6 +1293,21 @@ functions_inner_var: {
     node_version: ">=8"
 }
 
+instanceof_lambda: {
+    options = {
+        evaluate: true,
+        side_effects: true,
+    }
+    input: {
+        console.log(42 instanceof async function() {});
+    }
+    expect: {
+        console.log(false);
+    }
+    expect_stdout: "false"
+    node_version: ">=8"
+}
+
 issue_4335_1: {
     options = {
         inline: true,

test/compress/classes.js

@@ -1266,6 +1266,23 @@ keep_fnames: {
     node_version: ">=4"
 }
 
+instanceof_lambda: {
+    options = {
+        evaluate: true,
+        side_effects: true,
+    }
+    input: {
+        "use strict";
+        console.log(42 instanceof class {});
+    }
+    expect: {
+        "use strict";
+        console.log(false);
+    }
+    expect_stdout: "false"
+    node_version: ">=4"
+}
+
 issue_805_1: {
     options = {
         inline: true,

test/compress/evaluate.js

@@ -907,6 +907,20 @@ chained_side_effects: {
     ]
 }
 
+instanceof_lambda: {
+    options = {
+        evaluate: true,
+        side_effects: true,
+    }
+    input: {
+        console.log(42 instanceof function() {});
+    }
+    expect: {
+        console.log(false);
+    }
+    expect_stdout: "false"
+}
+
 issue_1649: {
     options = {
         evaluate: true,

test/compress/pure_funcs.js

@@ -147,7 +147,7 @@ relational: {
         "bar" >= "bar";
     }
     expect: {
-        bar();
+        0 instanceof bar();
         bar();
         bar(), bar();
         bar();

test/compress/regexp.js

@@ -50,6 +50,22 @@ regexp_properties: {
     expect_stdout: "abc true false 0 false"
 }
 
+instanceof_1: {
+    input: {
+        console.log(/foo/ instanceof RegExp);
+    }
+    expect_exact: "console.log(/foo/ instanceof RegExp);"
+    expect_stdout: "true"
+}
+
+instanceof_2: {
+    input: {
+        console.log(42 + /foo/ instanceof Object);
+    }
+    expect_exact: "console.log(42+/foo/ instanceof Object);"
+    expect_stdout: "false"
+}
+
 issue_3434_1: {
     options = {
         evaluate: true,

test/compress/side_effects.js

@@ -645,3 +645,56 @@ issue_4751: {
     }
     expect_stdout: "PASS"
 }
+
+drop_instanceof: {
+    options = {
+        side_effects: true,
+    }
+    input: {
+        42 instanceof function() {};
+        console.log("PASS");
+    }
+    expect: {
+        console.log("PASS");
+    }
+    expect_stdout: "PASS"
+}
+
+drop_instanceof_reference: {
+    options = {
+        reduce_vars: true,
+        side_effects: true,
+        toplevel: true,
+    }
+    input: {
+        function f() {}
+        42 instanceof f;
+        console.log("PASS");
+    }
+    expect: {
+        function f() {}
+        console.log("PASS");
+    }
+    expect_stdout: "PASS"
+}
+
+retain_instanceof: {
+    options = {
+        side_effects: true,
+    }
+    input: {
+        try {
+            42 instanceof "foo";
+        } catch (e) {
+            console.log("PASS");
+        }
+    }
+    expect: {
+        try {
+            0 instanceof "foo";
+        } catch (e) {
+            console.log("PASS");
+        }
+    }
+    expect_stdout: "PASS"
+}

test/compress/yields.js

@@ -934,6 +934,21 @@ drop_unused_call: {
     node_version: ">=4"
 }
 
+instanceof_lambda: {
+    options = {
+        evaluate: true,
+        side_effects: true,
+    }
+    input: {
+        console.log(42 instanceof function*() {});
+    }
+    expect: {
+        console.log(false);
+    }
+    expect_stdout: "false"
+    node_version: ">=4"
+}
+
 issue_4454_1: {
     rename = false
     options = {