Node.js Flow (part 2) - Fibers and Generators

This post is the continuation of a previous post about asynchronous flow in JavaScript/node.js

This time we'll look at

  • fibers (fibrous.js)
  • generators (ES6)
  • generators + co + mz

We'll still use my (poor) example of an express route:

  • reading from a file
  • doing some processing (in 3 steps)
    • process* is just some arbitrary async operation that calls back with extended data
  • writing the result to a file
  • responding to the request with either a success or error message

Approach 1 - Using fibers

var fs = require('fs');  
var express = require('express');  
var fibrous = require('fibrous');

var app = express();

app.get('/', function(req, res) {  
  fibrous.run(function() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = fs.sync.readFile(inputFile);
      var processedData1 = process1.sync(inputData);
      var processedData2 = process2.sync(processedData1);
      var result = process3.sync(processedData2);

      fs.sync.writeFile(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  }, function(err) {...});
});

Using fibers out flow looks a lot like synchronous code, since our async functions now return a value. This is some of the "magic" of fibers. All functions/object-methods now have a .sync version that enables this.

Approach 2 - Generators (ES6)

Generators is a new construct introduced in ES6, giving you the ability to create generator functions using the function *() keyword. These generator functions returns an iterator when called (var iter = genFun();) which has the special ability of letting you step through your code until you hit the newly introduced yield keyword, which will pause the execution while waiting for a return value, without blocking the execution of your application.

Read more about them here

# example using partial application
var fs = require('fs');  
var express = require('express');  
var run = require('../../lib/runner');

var app = express();

app.get('/', function(req, res) {  
  run(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = yield fs.readFile.bind(fs, inputFile);
      var processedData1 = yield process1.bind(null, inputData);
      var processedData2 = yield process2.bind(null, processedData1);
      var result = yield process3.bind(null, processedData2);

      yield fs.writeFile.bind(fs, outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

where the iterator runner function run is:

function run(fn) {  
  var gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    var ret = gen.next(res);
    if (ret.done) return;
    ret.value(next);
  }

  next();
};
# example using currying (from lodash)
var fs = require('fs');  
var express = require('express');  
var run = require('../../lib/runner');  
var _ = require('lodash');

var app = express();

app.get('/', function(req, res) {  
  run(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = yield _.curry(fs.readFile.bind(fs))(inputFile);
      var processedData1 = yield _.curry(process1)(inputData);
      var processedData2 = yield _.curry(process2)(processedData1);
      var result = yield _.curry(process3)(processedData2);

      yield _.curry(fs.writeFile.bind(fs))(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  });
});
# example using thunks
var fs = require('fs');  
var express = require('express');  
var run = require('../../lib/runner');  
var thunkify = require('../../lib/thunkify');

var app = express();

app.get('/', function(req, res) {  
  run(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = yield thunkify(fs.readFile.bind(fs))(inputFile);
      var processedData1 = yield thunkify(process1)(inputData);
      var processedData2 = yield thunkify(process2)(processedData1);
      var result = yield thunkify(process3)(processedData2);

      yield thunkify(fs.writeFile.bind(fs))(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

Where thunkify is a function that basicly does a 2-step curry - meaning you pass input data as the first call, and a callback as the second:

function thunkify(fn) {  
  return function() {
    var args = Array.prototype.slice.call(arguments, 0, fn.length - 1);

    return function(done) {
      fn.apply(null, args.concat(done));
    };
  };
};
# example using promises
var Promise = require('bluebird');  
var fs = Promise.promisifyAll(require('fs'));  
var express = require('express');  
var run = require('../../lib/runner');

var app = express();

app.get('/', function(req, res) {  
  run(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = fs.readFileAsync(inputFile);
      var processedData1 = yield Promise.promisify(process1)(inputData);
      var processedData2 = yield Promise.promisify(process2)(processedData1);
      var result = yield Promise.promisify(process3)(processedData2);

      yield fs.writeFileAsync(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  });
});

To be able to use Promises when yielding in our generator function we'll need to modify out iterator runner function run to support them:

function run(fn) {  
  var gen = fn();

  function next(err, res) {
    if (err) return gen.throw(err);
    var ret = gen.next(res);
    if (ret.done) return;

    if (typeof ret.value.then === 'function') {
      try {
        ret.value.then(function(value) {
          next(null, value);
        }, next);
      } catch (e) {
        gen.throw(e);
      }
    } else {
      try {
        ret.value(next);
      } catch (e) {
        gen.throw(e);
      }
    }
  }

  next();
};

Approach 3 - Generators + co + mz

Instead of building out own iterator runner function we can use the module co by T.J Holowaychuck, which support yielding to a number of options (promises, thunks, arrays, objects, etc..) while wrapping the entire execution in a Promise.

# example using co
var Promise = require('bluebird');  
var fs = Promise.promisifyAll(require('fs'));  
var express = require('express');  
var co = require('co');

var app = express();

app.get('/', function(req, res) {  
  co(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = fs.readFileAsync(inputFile);
      var processedData1 = yield Promise.promisify(process1)(inputData);
      var processedData2 = yield Promise.promisify(process2)(processedData1);
      var result = yield Promise.promisify(process3)(processedData2);

      yield fs.writeFileAsync(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(funtion(err) {...});
});

To get example even cleaner we can use a helper library called mz to shim all native async API's to return a Promise.

# example using co + mz
var Promise = require('bluebird');  
var fs = require('mz/fs');  
var express = require('express');  
var co = require('co');

var app = express();

app.get('/', function(req, res) {  
  co(function *() {
    var inputFile = 'input.txt';
    var outputFile = 'output.txt';

    try {
      var inputData = yield fs.readFile(inputFile);
      var processedData1 = yield Promise.promisify(process1)(inputData);
      var processedData2 = yield Promise.promisify(process2)(processedData1);
      var result = yield Promise.promisify(process3)(processedData2);

      yield fs.writeFile(outputFile, result);
      res.status(200).send('success');

    } catch (err) {
      res.status(500).send(err);
    }
  }).catch(onError);
});

See part 1 of this series here.

comments powered by Disqus