master
Sven Slootweg 9 years ago
parent 8b557cb9b0
commit ae332d53d2

3
.gitignore vendored

@ -1,3 +1,4 @@
# https://git-scm.com/docs/gitignore
# https://help.github.com/articles/ignoring-files
# Example .gitignore files: https://github.com/github/gitignore
# Example .gitignore files: https://github.com/github/gitignore
/node_modules/

@ -0,0 +1,117 @@
# combined-stream2
A drop-in Streams2-compatible replacement for the `combined-stream` module.
Supports most of the `combined-stream` API. Automatically wraps Streams1 streams, so that they work as well.
Both Promises and nodebacks are supported.
## License
[WTFPL](http://www.wtfpl.net/txt/copying/) or [CC0](https://creativecommons.org/publicdomain/zero/1.0/), whichever you prefer. A donation and/or attribution are appreciated, but not required.
## Donate
My income consists entirely of donations for my projects. If this module is useful to you, consider [making a donation](http://cryto.net/~joepie91/donate.html)!
You can donate using Bitcoin, PayPal, Gratipay, Flattr, cash-in-mail, SEPA transfers, and pretty much anything else.
## Contributing
Pull requests welcome. Please make sure your modifications are in line with the overall code style, and ensure that you're editing the `.coffee` files, not the `.js` files.
Build tool of choice is `gulp`; simply run `gulp` while developing, and it will watch for changes.
Be aware that by making a pull request, you agree to release your modifications under the licenses stated above.
## Migrating from `combined-stream`
Note that there are a few important differences between `combined-stream` and `combined-stream`:
* You cannot supply strings, only Buffers and streams. This is because `combined-stream2` doesn't know what the encoding of your string would be, and can't guess safely. You will need to manually encode strings to a Buffer before passing them on to `combined-stream2`.
* The `pauseStreams` option does not exist. All streams are read lazily in non-flowing mode; that is, no data is read until something explicitly tries to read the combined stream.
* The `maxDataSize` option does not exist.
* The `.write()`, `.end()` and `.destroy()` methods are not (yet) implemented.
* There is a `.getCombinedStreamLength()` method that asynchronously returns the total length of all streams (or an error if it cannot be determined). __This method will 'resolve' all callback-supplied streams, as if the stream were being read.__
Most usecases will not be affected by these differences, but your mileage may vary.
## Usage
```javascript
var CombinedStream = require('combined-stream2');
var fs = require('fs');
var combinedStream = CombinedStream.create();
combinedStream.append(fs.createReadStream('file1.txt'));
combinedStream.append(fs.createReadStream('file2.txt'));
combinedStream.pipe(fs.createWriteStream('combined.txt'));
```
### Appending a stream asynchronously (lazily)
The function will only be called when you try to either read/pipe the combined stream, or retrieve the total stream length.
```javascript
combinedStream.append(function(next){
next(fs.createReadStream('file3.txt'));
});
```
### Getting the combined length of all streams
See the API documentation below for more details.
```javascript
Promise = require("bluebird");
Promise.try(function(){
combinedStream.getCombinedStreamLength()
}).then(function(length){
console.log("Total stream length is " + length + " bytes.");
}).catch(function(err){
console.log("Could not determine the total stream length!");
});
```
... or using nodebacks:
```javascript
combinedStream.getCombinedStreamLength(function(err, length){
if(err) {
console.log("Could not determine the total stream length!");
} else {
console.log("Total stream length is " + length + " bytes.");
}
});
```
## API
Since `combined-stream2` is a `stream.Readable`, it inherits the [regular Readable stream properties](http://nodejs.org/api/stream.html#stream_class_stream_readable). Aside from that, the following methods exist:
### CombinedStream.create()
Creates and returns a new `combinedStream`. Contrary to the `.create()` method for the original `combined-stream` module, this method does not accept options.
### combinedStream.append(source, [options])
Adds a source to the combined stream. Valid sources are streams, Buffers, and callbacks that return either of the two (asynchronously).
* __source__: The source to add.
* __options__: *Optional.* Additional stream options.
* __contentLength__: The length of the stream. Useful if your stream type is not supported by `stream-length`, but you know the length of the stream in advance. Also available as `knownLength` for backwards compatibility reasons.
### combinedStream.getCombinedStreamLength()
__This method will 'resolve' all callback-supplied streams, as if the stream were being read.__
Asynchronously returns the total length of all streams (and Buffers) together. If the total length cannot be determined (ie. at least one of the streams is of an unsupported type), an error is thrown asynchronously.
This functionality uses the [`stream-length`](https://www.npmjs.com/package/stream-length) module.
### combinedStream.pipe(target)
Like for other Streams2 Readable streams, this will start piping the combined stream contents into the `target` stream.
After calling this, you can no longer append new streams.

@ -0,0 +1,28 @@
var gulp = require('gulp');
/* CoffeeScript compile deps */
var path = require('path');
var gutil = require('gulp-util');
var concat = require('gulp-concat');
var rename = require('gulp-rename');
var coffee = require('gulp-coffee');
var cache = require('gulp-cached');
var remember = require('gulp-remember');
var plumber = require('gulp-plumber');
var source = ["lib/**/*.coffee", "index.coffee"]
gulp.task('coffee', function() {
return gulp.src(source, {base: "."})
.pipe(plumber())
.pipe(cache("coffee"))
.pipe(coffee({bare: true}).on('error', gutil.log)).on('data', gutil.log)
.pipe(remember("coffee"))
.pipe(gulp.dest("."));
});
gulp.task('watch', function () {
gulp.watch(source, ['coffee']);
});
gulp.task('default', ['coffee', 'watch']);

@ -0,0 +1 @@
module.exports = require("./lib/combined-stream2");

@ -0,0 +1 @@
module.exports = require("./lib/combined-stream2");

@ -0,0 +1,180 @@
stream = require "stream"
Promise = require "bluebird"
streamLength = require "stream-length"
debug = require("debug")("combined-stream2")
# FIXME: .error handler on streams?
# Utility functions
ofTypes = (obj, types) ->
match = false
for type in types
match = match or obj instanceof type
return match
isStream = (obj) ->
return ofTypes obj, [stream.Readable, stream.Duplex, stream.Transform]
makeStreams2 = (stream) ->
# Adapted from https://github.com/feross/multistream/blob/master/index.js
if not stream or typeof stream == "function" or stream instanceof Buffer or stream._readableState?
return stream
wrapper = new stream.Readable().wrap(stream)
if stream.destroy?
wrapper.destroy = stream.destroy.bind(stream)
return wrapper
# The actual stream class definition
class CombinedStream extends stream.Readable
constructor: ->
super
@_reading = false
@_sources = []
@_currentSource = null
@_sourceDataAvailable = false
@_wantData = false
append: (source, options = {}) ->
# Only readable binary data sources are allowed.
if not ofTypes source, [stream.Readable, stream.Duplex, stream.Transform, Buffer, Function]
throw new Error "The provided source must be either a readable stream or a Buffer, or a callback providing either of those. If it is currently a string, you need to convert it to a Buffer yourself and ensure that the encoding is correct."
debug "appending source: %s", source.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r")
@_sources.push [makeStreams2(source), options]
getStreamLengths: ->
debug "getting stream lengths"
if @_reading
Promise.reject new Error("You can't obtain the stream lengths anymore once you've started reading!")
else
Promise.try =>
@_resolveAllSources()
.then (actualSources) =>
@_sources = actualSources
Promise.resolve actualSources
.map (source) ->
if source[1]?.knownLength? or source[1]?.contentLength?
Promise.resolve source[1]?.knownLength ? source[1]?.contentLength
else
streamLength source[0]
getCombinedStreamLength: ->
debug "getting combined stream length"
Promise.try =>
@getStreamLengths()
.reduce ((total, current) -> total + current), 0
_resolveAllSources: ->
debug "resolving all sources"
Promise.all (@_resolveSource(source) for source in @_sources)
_resolveSource: (source) ->
# If the 'source' is a function, then it's actually a callback that will *return* the source. We call the callback, and supply it with a `next` function that will post-process the source, and eventually trigger the actual read.
new Promise (resolve, reject) => # WARN?
if source[0] instanceof Function
debug "resolving %s", source[0].toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r")
source[0] (realSource) =>
resolve [realSource, source[1]]
else
# It's a regular source, so we immediately continue.
debug "source %s is already resolved", source[0].toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r")
resolve source
_initiateRead: ->
Promise.try =>
@_reading = true
@_resolveAllSources()
.then (actualSources) =>
@_sources = actualSources
Promise.resolve()
_read: (size) ->
Promise.try =>
if @_reading == false
@_initiateRead()
else
Promise.resolve()
.then =>
@_doRead size
_doRead: (size) ->
# FIXME: We should probably try to do something with `size` ourselves. Just passing it on for now, but it'd be nice to implement it properly in the future - this might help efficiency in some cases.
Promise.try =>
if @_currentSource == null
# We're not currently actively reading from any sources. Set a new source to be the current source.
@_nextSource size
else
# We haven't changed our source - immediately continue with the actual read.
Promise.resolve()
.then =>
@_doActualRead size
_nextSource: (readSize) ->
if @_sources.length == 0
# We've run out of sources - signal EOF and bail.
@push null
return
@_currentSource = @_sources.shift()[0]
@_currentIsStream = isStream @_currentSource
if @_currentIsStream
@_currentSource.once "end", =>
# We've depleted the stream (ie. we've read 'null') The current source should be set to `null`, so that on the next read a new source will be picked. We'll also immediately trigger the next read - the stream will be expecting to receive *some* kind of data before calling the next read itself.
@_currentSource = null
@_doRead readSize # FIXME: This should probably use the last-requested read size, not the one that was requested when *setting up* the `end` event.
@_currentSource.on "readable", =>
debug "received readable event, setting sourceDataAvailable to true"
@_sourceDataAvailable = true
if @_wantData
debug "wantData queued, reading"
@_doStreamRead()
Promise.resolve()
# We're wrapping the actual reading code in a separate function, so as to facilitate source-returning callbacks in the sources list.
_doActualRead: (size) =>
# FIXME: Apparently, it may be possible to push more than one chunk in a single _read call. The implementation specifics of this should probably be looked into - that could perhaps make our stream a bit more efficient. On the other hand, shouldn't we leave this for the Writable to decide?
new Promise (resolve, reject) =>
if @_currentIsStream
# This is a readable stream of some sort - we'll do a read, and pass on the result. We'll pass on the `size` parameter, but there's no guarantee that anything will actually be done with it.
if @_sourceDataAvailable
@_doStreamRead()
return resolve()
else
debug "want data, but no readable event fired yet, setting wantData to true"
@_wantData = true
return resolve() # We haven't actually read anything yet, but whatever.
else
# This is a Buffer - we'll push it as is, and immediately mark it as completed.
chunk = @_currentSource
# We need to unset it *before* pushing the chunk, because otherwise V8 will sometimes not give control back to this function, and a second read may occur before the source can be unset.
@_currentSource = null
if chunk != null # FIXME: ???
debug "pushing buffer %s", chunk.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r")
@push chunk
else
debug "WARN: current source was null, pushing empty buffer"
@push new Buffer("")
resolve()
_doStreamRead: =>
Promise.try =>
@_sourceDataAvailable = false
@_wantData = false
@push @_currentSource.read()
Promise.resolve()
# Public module API
module.exports =
create: (options) ->
# We implement the same API as the original `combined-stream`, for drop-in compatibility reasons.
return new CombinedStream(options)

@ -0,0 +1,254 @@
var CombinedStream, Promise, debug, isStream, makeStreams2, ofTypes, stream, streamLength,
__bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
__hasProp = {}.hasOwnProperty,
__extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
stream = require("stream");
Promise = require("bluebird");
streamLength = require("stream-length");
debug = require("debug")("combined-stream2");
ofTypes = function(obj, types) {
var match, type, _i, _len;
match = false;
for (_i = 0, _len = types.length; _i < _len; _i++) {
type = types[_i];
match = match || obj instanceof type;
}
return match;
};
isStream = function(obj) {
return ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]);
};
makeStreams2 = function(stream) {
var wrapper;
if (!stream || typeof stream === "function" || stream instanceof Buffer || (stream._readableState != null)) {
return stream;
}
wrapper = new stream.Readable().wrap(stream);
if (stream.destroy != null) {
wrapper.destroy = stream.destroy.bind(stream);
}
return wrapper;
};
CombinedStream = (function(_super) {
__extends(CombinedStream, _super);
function CombinedStream() {
this._doStreamRead = __bind(this._doStreamRead, this);
this._doActualRead = __bind(this._doActualRead, this);
CombinedStream.__super__.constructor.apply(this, arguments);
this._reading = false;
this._sources = [];
this._currentSource = null;
this._sourceDataAvailable = false;
this._wantData = false;
}
CombinedStream.prototype.append = function(source, options) {
if (options == null) {
options = {};
}
if (!ofTypes(source, [stream.Readable, stream.Duplex, stream.Transform, Buffer, Function])) {
throw new Error("The provided source must be either a readable stream or a Buffer, or a callback providing either of those. If it is currently a string, you need to convert it to a Buffer yourself and ensure that the encoding is correct.");
}
debug("appending source: %s", source.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r"));
return this._sources.push([makeStreams2(source), options]);
};
CombinedStream.prototype.getStreamLengths = function() {
debug("getting stream lengths");
if (this._reading) {
return Promise.reject(new Error("You can't obtain the stream lengths anymore once you've started reading!"));
} else {
return Promise["try"]((function(_this) {
return function() {
return _this._resolveAllSources();
};
})(this)).then((function(_this) {
return function(actualSources) {
_this._sources = actualSources;
return Promise.resolve(actualSources);
};
})(this)).map(function(source) {
var _ref, _ref1, _ref2, _ref3, _ref4;
if ((((_ref = source[1]) != null ? _ref.knownLength : void 0) != null) || (((_ref1 = source[1]) != null ? _ref1.contentLength : void 0) != null)) {
return Promise.resolve((_ref2 = (_ref3 = source[1]) != null ? _ref3.knownLength : void 0) != null ? _ref2 : (_ref4 = source[1]) != null ? _ref4.contentLength : void 0);
} else {
return streamLength(source[0]);
}
});
}
};
CombinedStream.prototype.getCombinedStreamLength = function() {
debug("getting combined stream length");
return Promise["try"]((function(_this) {
return function() {
return _this.getStreamLengths();
};
})(this)).reduce((function(total, current) {
return total + current;
}), 0);
};
CombinedStream.prototype._resolveAllSources = function() {
var source;
debug("resolving all sources");
return Promise.all((function() {
var _i, _len, _ref, _results;
_ref = this._sources;
_results = [];
for (_i = 0, _len = _ref.length; _i < _len; _i++) {
source = _ref[_i];
_results.push(this._resolveSource(source));
}
return _results;
}).call(this));
};
CombinedStream.prototype._resolveSource = function(source) {
return new Promise((function(_this) {
return function(resolve, reject) {
if (source[0] instanceof Function) {
debug("resolving %s", source[0].toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r"));
return source[0](function(realSource) {
return resolve([realSource, source[1]]);
});
} else {
debug("source %s is already resolved", source[0].toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r"));
return resolve(source);
}
};
})(this));
};
CombinedStream.prototype._initiateRead = function() {
return Promise["try"]((function(_this) {
return function() {
_this._reading = true;
return _this._resolveAllSources();
};
})(this)).then((function(_this) {
return function(actualSources) {
_this._sources = actualSources;
return Promise.resolve();
};
})(this));
};
CombinedStream.prototype._read = function(size) {
return Promise["try"]((function(_this) {
return function() {
if (_this._reading === false) {
return _this._initiateRead();
} else {
return Promise.resolve();
}
};
})(this)).then((function(_this) {
return function() {
return _this._doRead(size);
};
})(this));
};
CombinedStream.prototype._doRead = function(size) {
return Promise["try"]((function(_this) {
return function() {
if (_this._currentSource === null) {
return _this._nextSource(size);
} else {
return Promise.resolve();
}
};
})(this)).then((function(_this) {
return function() {
return _this._doActualRead(size);
};
})(this));
};
CombinedStream.prototype._nextSource = function(readSize) {
if (this._sources.length === 0) {
this.push(null);
return;
}
this._currentSource = this._sources.shift()[0];
this._currentIsStream = isStream(this._currentSource);
if (this._currentIsStream) {
this._currentSource.once("end", (function(_this) {
return function() {
_this._currentSource = null;
return _this._doRead(readSize);
};
})(this));
this._currentSource.on("readable", (function(_this) {
return function() {
debug("received readable event, setting sourceDataAvailable to true");
_this._sourceDataAvailable = true;
if (_this._wantData) {
debug("wantData queued, reading");
return _this._doStreamRead();
}
};
})(this));
}
return Promise.resolve();
};
CombinedStream.prototype._doActualRead = function(size) {
return new Promise((function(_this) {
return function(resolve, reject) {
var chunk;
if (_this._currentIsStream) {
if (_this._sourceDataAvailable) {
_this._doStreamRead();
return resolve();
} else {
debug("want data, but no readable event fired yet, setting wantData to true");
_this._wantData = true;
return resolve();
}
} else {
chunk = _this._currentSource;
_this._currentSource = null;
if (chunk !== null) {
debug("pushing buffer %s", chunk.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r"));
_this.push(chunk);
} else {
debug("WARN: current source was null, pushing empty buffer");
_this.push(new Buffer(""));
}
return resolve();
}
};
})(this));
};
CombinedStream.prototype._doStreamRead = function() {
return Promise["try"]((function(_this) {
return function() {
_this._sourceDataAvailable = false;
_this._wantData = false;
_this.push(_this._currentSource.read());
return Promise.resolve();
};
})(this));
};
return CombinedStream;
})(stream.Readable);
module.exports = {
create: function(options) {
return new CombinedStream(options);
}
};

@ -0,0 +1,38 @@
{
"name": "node-combined-stream2",
"version": "1.0.0",
"description": "A drop-in Streams2-compatible replacement for combined-stream.",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git@git.cryto.net:projects/joepie91/node-combined-stream2"
},
"keywords": [
"stream",
"combined",
"multistream",
"streams2"
],
"author": "Sven Slootweg",
"license": "WTFPL",
"devDependencies": {
"debug": "^2.1.1",
"gulp": "~3.8.0",
"gulp-cached": "~0.0.3",
"gulp-coffee": "~2.0.1",
"gulp-concat": "~2.2.0",
"gulp-livereload": "~2.1.0",
"gulp-nodemon": "~1.0.4",
"gulp-plumber": "~0.6.3",
"gulp-remember": "~0.2.0",
"gulp-rename": "~1.2.0",
"gulp-util": "~2.2.17"
},
"dependencies": {
"bluebird": "^2.8.1",
"stream-length": "^1.0.1"
}
}
Loading…
Cancel
Save