Implement harmony generators and yield

Uses #716's implementation and adds tests.

Fixes #716.

lib/ast.js

@@ -438,9 +438,10 @@ var AST_ArrowParametersOrSeq = DEFNODE("ArrowParametersOrSeq", "expressions", {
     }
 });
 
-var AST_Lambda = DEFNODE("Lambda", "name argnames uses_arguments", {
+var AST_Lambda = DEFNODE("Lambda", "name argnames uses_arguments is_generator", {
     $documentation: "Base class for functions",
     $propdoc: {
+        is_generator: "is generatorFn or not",
         name: "[AST_SymbolDeclaration?] the name of this function",
         argnames: "[AST_SymbolFunarg|AST_Destructuring|AST_Expansion*] array of function arguments, destructurings, or expanding arguments",
         uses_arguments: "[boolean/S] tells whether this function accesses the arguments array"

lib/output.js

@@ -226,6 +226,10 @@ function OutputStream(options) {
         OUTPUT += str;
     };
 
+    var star = function(){
+        print("*");
+    }
+
     var space = options.beautify ? function() {
         print(" ");
     } : function() {
@@ -343,6 +347,7 @@ function OutputStream(options) {
         should_break    : function() { return options.width && this.current_width() >= options.width },
         newline         : newline,
         print           : print,
+        star            : star,
         space           : space,
         comma           : comma,
         colon           : colon,
@@ -783,6 +788,9 @@ function OutputStream(options) {
         var self = this;
         if (!nokeyword) {
             output.print("function");
+            if (this.is_generator) {
+                output.star();
+            }
             if (self.name) {
                 output.space();
             }
@@ -1203,8 +1211,13 @@ function OutputStream(options) {
         output.print(self.operator);
     });
     DEFPRINT(AST_Binary, function(self, output){
+        var isYield = (self.left.operator == "yield" || self.left.operator === "yield*");
         var op = self.operator;
+
+        isYield && output.print("(");
         self.left.print(output);
+        isYield && output.print(")");
+
         if (op[0] == ">" /* ">>" ">>>" ">" ">=" */
             && self.left instanceof AST_UnaryPostfix
             && self.left.operator == "--") {
@@ -1214,7 +1227,12 @@ function OutputStream(options) {
             // the space is optional depending on "beautify"
             output.space();
         }
+
+        isYield = (self.right.operator == "yield" || self.right.operator === "yield*");
+        isYield && output.print("(");
         output.print(op);
+        isYield && output.print(")");
+
         if ((op == "<" || op == "<<")
             && self.right instanceof AST_UnaryPrefix
             && self.right.operator == "!"

lib/parse.js

@@ -44,7 +44,7 @@
 
 "use strict";
 
-var KEYWORDS = 'break case catch class const continue debugger default delete do else extends finally for function if in instanceof new return switch throw try typeof var let void while with import export';
+var KEYWORDS = 'break case catch class const continue debugger default delete do else export extends finally for function if in instanceof new return switch throw try typeof var let void while with import yield';
 var KEYWORDS_ATOM = 'false null true';
 var RESERVED_WORDS = 'abstract boolean byte char double enum export final float goto implements int interface long native package private protected public short static super synchronized this throws transient volatile yield'
     + " " + KEYWORDS_ATOM + " " + KEYWORDS;
@@ -68,6 +68,7 @@ var OPERATORS = makePredicate([
     "instanceof",
     "typeof",
     "new",
+    "yield",
     "void",
     "delete",
     "++",
@@ -627,6 +628,7 @@ var UNARY_PREFIX = makePredicate([
     "typeof",
     "void",
     "delete",
+    "yield",
     "--",
     "++",
     "!",
@@ -1062,6 +1064,11 @@ function parse($TEXT, options) {
         var start = S.token
 
         var in_statement = ctor === AST_Defun;
+        var is_generator = is("operator", "*");
+        if (is_generator) {
+          next();
+        }
+
         var name = is("name") ? as_symbol(in_statement ? AST_SymbolDefun : AST_SymbolLambda) : null;
         if (in_statement && !name)
             unexpected();
@@ -1071,6 +1078,7 @@ function parse($TEXT, options) {
         return new ctor({
             start : args.start,
             end   : body.end,
+            is_generator: is_generator,
             name  : name,
             argnames: args,
             body  : body
@@ -1836,6 +1844,10 @@ function parse($TEXT, options) {
         var start = S.token;
         if (is("operator") && UNARY_PREFIX(start.value)) {
             next();
+            if  (start.type === "operator" && start.value === "yield" && is("operator", "*")) {
+                start.value = "yield*";
+                next();
+            }
             handle_regexp();
             var ex = make_unary(AST_UnaryPrefix, start.value, maybe_unary(allow_calls));
             ex.start = start;

test/compress/harmony.js

@@ -418,3 +418,19 @@ regression_cannot_use_of: {
         foo();  /* Label statement missing? No prob. */
     }
 }
+
+generators: {
+    input: {
+        function* fn() {};
+    }
+    expect_exact: "function*fn(){}"
+}
+
+generators_yield: {
+    input: {
+      function* fn() {
+        yield remote();
+      }
+    }
+    expect_exact: "function*fn(){yield remote()}"
+}

test/parser.js

@@ -118,6 +118,17 @@ module.exports = function () {
     ok.equal(expanding_def.name.names[0].TYPE, 'SymbolVar');
     ok.equal(expanding_def.name.names[1].TYPE, 'Expansion');
     ok.equal(expanding_def.name.names[1].symbol.TYPE, 'SymbolVar');
+
+    // generators
+    var generators_def = UglifyJS.parse('function* fn() {}').body[0];
+    ok.equal(generators_def.is_generator, true);
+
+    ok.throws(function () {
+        UglifyJS.parse('function* (){ }');
+    });
+
+    var generators_yield_def = UglifyJS.parse('function* fn() {\nyield remote();\}').body[0].body[0];
+    ok.equal(generators_yield_def.body.operator, 'yield');
 }
 
 // Run standalone