This post is the continuation of a previous post about asynchronous flow in JavaScript/node.js
This time we'll look at
- fibers (fibrous.js)
- generators (ES6)
- generators + co + mz
We'll still use my (poor) example of an express route:
- reading from a file
- doing some processing (in 3 steps)
- process* is just some arbitrary async operation that calls back with extended data
- writing the result to a file
- responding to the request with either a success or error message
Approach 1 - Using fibers
var fs = require('fs');
var express = require('express');
var fibrous = require('fibrous');
var app = express();
app.get('/', function(req, res) {
fibrous.run(function() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = fs.sync.readFile(inputFile);
var processedData1 = process1.sync(inputData);
var processedData2 = process2.sync(processedData1);
var result = process3.sync(processedData2);
fs.sync.writeFile(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
}, function(err) {...});
});
Using fibers out flow looks a lot like synchronous code, since our async functions now return a value. This is some of the "magic" of fibers. All functions/object-methods now have a .sync version that enables this.
Approach 2 - Generators (ES6)
Generators is a new construct introduced in ES6, giving you the ability to create generator functions using the function *() keyword. These generator functions returns an iterator when called (var iter = genFun();) which has the special ability of letting you step through your code until you hit the newly introduced yield keyword, which will pause the execution while waiting for a return value, without blocking the execution of your application.
Read more about them here
example using partial application
var fs = require('fs');
var express = require('express');
var run = require('../../lib/runner');
var app = express();
app.get('/', function(req, res) {
run(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = yield fs.readFile.bind(fs, inputFile);
var processedData1 = yield process1.bind(null, inputData);
var processedData2 = yield process2.bind(null, processedData1);
var result = yield process3.bind(null, processedData2);
yield fs.writeFile.bind(fs, outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
});
});
where the iterator runner function run is:
function run(fn) {
var gen = fn();
function next(err, res) {
if (err) return gen.throw(err);
var ret = gen.next(res);
if (ret.done) return;
ret.value(next);
}
next();
};
example using currying (from lodash)
var fs = require('fs');
var express = require('express');
var run = require('../../lib/runner');
var _ = require('lodash');
var app = express();
app.get('/', function(req, res) {
run(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = yield _.curry(fs.readFile.bind(fs))(inputFile);
var processedData1 = yield _.curry(process1)(inputData);
var processedData2 = yield _.curry(process2)(processedData1);
var result = yield _.curry(process3)(processedData2);
yield _.curry(fs.writeFile.bind(fs))(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
});
});
example using thunks
var fs = require('fs');
var express = require('express');
var run = require('../../lib/runner');
var thunkify = require('../../lib/thunkify');
var app = express();
app.get('/', function(req, res) {
run(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = yield thunkify(fs.readFile.bind(fs))(inputFile);
var processedData1 = yield thunkify(process1)(inputData);
var processedData2 = yield thunkify(process2)(processedData1);
var result = yield thunkify(process3)(processedData2);
yield thunkify(fs.writeFile.bind(fs))(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
});
});
Where thunkify is a function that basicly does a 2-step curry - meaning you pass input data as the first call, and a callback as the second:
function thunkify(fn) {
return function() {
var args = Array.prototype.slice.call(arguments, 0, fn.length - 1);
return function(done) {
fn.apply(null, args.concat(done));
};
};
};
example using promises
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var express = require('express');
var run = require('../../lib/runner');
var app = express();
app.get('/', function(req, res) {
run(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = fs.readFileAsync(inputFile);
var processedData1 = yield Promise.promisify(process1)(inputData);
var processedData2 = yield Promise.promisify(process2)(processedData1);
var result = yield Promise.promisify(process3)(processedData2);
yield fs.writeFileAsync(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
});
});
To be able to use Promises when yielding in our generator function we'll need to modify out iterator runner function run to support them:
function run(fn) {
var gen = fn();
function next(err, res) {
if (err) return gen.throw(err);
var ret = gen.next(res);
if (ret.done) return;
if (typeof ret.value.then === 'function') {
try {
ret.value.then(function(value) {
next(null, value);
}, next);
} catch (e) {
gen.throw(e);
}
} else {
try {
ret.value(next);
} catch (e) {
gen.throw(e);
}
}
}
next();
};
Approach 3 - Generators + co + mz
Instead of building out own iterator runner function we can use the module co by T.J Holowaychuck, which support yielding to a number of options (promises, thunks, arrays, objects, etc..) while wrapping the entire execution in a Promise.
example using co
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var express = require('express');
var co = require('co');
var app = express();
app.get('/', function(req, res) {
co(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = fs.readFileAsync(inputFile);
var processedData1 = yield Promise.promisify(process1)(inputData);
var processedData2 = yield Promise.promisify(process2)(processedData1);
var result = yield Promise.promisify(process3)(processedData2);
yield fs.writeFileAsync(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
}).catch(funtion(err) {...});
});
To get example even cleaner we can use a helper library called mz to shim all native async API's to return a Promise.
example using co + mz
var Promise = require('bluebird');
var fs = require('mz/fs');
var express = require('express');
var co = require('co');
var app = express();
app.get('/', function(req, res) {
co(function *() {
var inputFile = 'input.txt';
var outputFile = 'output.txt';
try {
var inputData = yield fs.readFile(inputFile);
var processedData1 = yield Promise.promisify(process1)(inputData);
var processedData2 = yield Promise.promisify(process2)(processedData1);
var result = yield Promise.promisify(process3)(processedData2);
yield fs.writeFile(outputFile, result);
res.status(200).send('success');
} catch (err) {
res.status(500).send(err);
}
}).catch(onError);
});
See part 1 of this series here.