From fd13f88463d2dcafacd38cfcd31c4ebbd1118b86 Mon Sep 17 00:00:00 2001 From: Sven Slootweg Date: Thu, 22 Jan 2015 17:15:54 +0100 Subject: [PATCH] v1.0.0 --- .gitignore | 3 +- README.md | 120 ++++++++++++++++++++++++++++++ gulpfile.js | 28 +++++++ index.coffee | 1 + index.js | 1 + lib/form-data2.coffee | 131 +++++++++++++++++++++++++++++++++ lib/form-data2.js | 167 ++++++++++++++++++++++++++++++++++++++++++ package.json | 41 +++++++++++ 8 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 README.md create mode 100644 gulpfile.js create mode 100644 index.coffee create mode 100644 index.js create mode 100644 lib/form-data2.coffee create mode 100644 lib/form-data2.js create mode 100644 package.json diff --git a/.gitignore b/.gitignore index f86fa8e..23981d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # https://git-scm.com/docs/gitignore # https://help.github.com/articles/ignoring-files -# Example .gitignore files: https://github.com/github/gitignore \ No newline at end of file +# Example .gitignore files: https://github.com/github/gitignore +/node_modules/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..78ba4d9 --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +# form-data2 + +A Streams2-compatible drop-in replacement for the `form-data` module. Through the wrapping done by the underlying `combined-stream2` module, old-style streams are also supported. + +Takes a number of streams or Buffers, and turns them into a valid `multipart/form-data` stream. + +## License + +[WTFPL](http://www.wtfpl.net/txt/copying/) or [CC0](https://creativecommons.org/publicdomain/zero/1.0/), whichever you prefer. A donation and/or attribution are appreciated, but not required. + +## Donate + +My income consists entirely of donations for my projects. If this module is useful to you, consider [making a donation](http://cryto.net/~joepie91/donate.html)! + +You can donate using Bitcoin, PayPal, Gratipay, Flattr, cash-in-mail, SEPA transfers, and pretty much anything else. + +## Contributing + +Pull requests welcome. Please make sure your modifications are in line with the overall code style, and ensure that you're editing the `.coffee` files, not the `.js` files. + +Build tool of choice is `gulp`; simply run `gulp` while developing, and it will watch for changes. + +Be aware that by making a pull request, you agree to release your modifications under the licenses stated above. + +## Migrating from `form-data` + +While `form-data2` was designed to be roughly API-compatible with the `form-data` module, there are a few notable differences: + +* `form-data2` does __not__ do HTTP requests. It is purely a multipart/form-data encoder. This means you should use a different module for your HTTP requests, such as [`bhttp`](https://www.npmjs.com/package/bhttp), [`request`](https://www.npmjs.com/package/request), or the core `http` module. `bhttp` uses `form-data2` internally - that means you won't have to manually use `form-data2`. +* The `header` option for the `form.append()` options object is not (yet) implemented. This means that you cannot currently set custom field headers. +* There is no `form.getLengthSync()` method. Length retrieval is always asynchronous. +* A `form-data2` stream is not a real stream. Only the `form.pipe()` method is implemented, other stream methods are unavailable. This may change in the future. + +## Usage + +```javascript +var FormData = require('form-data2'); +var fs = require('fs'); + +var form = new FormData(); +form.append('my_field', 'my value'); +form.append('my_buffer', new Buffer(10)); +form.append('my_file', fs.createReadStream('/foo/bar.jpg')); +``` + +### Getting the HTTP headers + +```javascript +Promise = require("bluebird"); +util = require("util"); + +Promise.try(function(){ + form.getHeaders(); +}).then(function(headers){ + console.log("Stream headers:", util.inspect(headers)); +}).catch(function(err){ + console.log("Something went wrong!"); +}); +``` + +... or using nodebacks: + +```javascript +form.getHeaders(function(err, headers){ + if(err) { + console.log("Something went wrong!"); + } else { + console.log("Stream headers:", util.inspect(headers)); + } +}); +``` + +## API + +Note that this is __not__ a real stream, and does therefore not expose the regular `stream.Readable` properties. The `pipe` method will simply pipe the underlying `combined-stream2` stream into the specified target. If you want to access the stream directly for some reason, use `form._stream`. + +### append(name, source, [options]) + +Adds a new data source to the multipart/form-data stream. + +This module will *not* automatically handle arrays for you - if you need to send an array, you will need to call this method for each element individually, using the same `[]`-suffixed field name for each element. + +* __name__: The name of the form field. +* __source__: The data source to add. This can be a stream, a Buffer, or a UTF-8 string. +* __options__: *Optional.* Additional options for the stream. + * __contentLength__: The total length of the stream. Useful if your stream is of a type that [`stream-length`](https://www.npmjs.com/package/stream-length) can't automatically determine the length of. Also available under the alias `knownLength`, for backwards compatibility reasons. + * __contentType__: The MIME type of the source. It's recommended to *always* specify this manually; however, if it's not supplied, `form-data2` will attempt to determine it for you where possible. + * __filename__: The filename of the source, if it is a file or should be represented as such. It's recommended to *always* specify this manually (if the source is a file of some sort); however, if it's not supplied, `form-data2` will attempt to determine it for you where possible. + +### getHeaders([callback]) + +Asynchronously returns the HTTP request headers that you will need to successfully transmit this stream, as an object. + +All object keys are lower-case. To use the headers, simply merge them into the headers object for your request. + +If you specify a `callback`, it will be treated as a nodeback. If you do *not* specify a `callback`, a Promise will be returned. + +### pipe(target) + +Pipes the `multipart/form-data` stream into the `target` stream. + +### done() + +__You probably don't need to call this.__ + +Calling this method will append the multipart/form-data footer to the stream. It is called automatically when you `pipe` the stream into something else. + +### getBoundary() + +__You probably don't need to call this.__ + +Returns the form boundary used in the `multipart/form-data` stream. + +### getLength([callback]) + +__You probably don't need to call this.__ + +Asynchronously returns the total length of the stream in bytes. + +If you specify a `callback`, it will be treated as a nodeback. If you do *not* specify a `callback`, a Promise will be returned. diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..bb7f05f --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,28 @@ +var gulp = require('gulp'); + +/* CoffeeScript compile deps */ +var path = require('path'); +var gutil = require('gulp-util'); +var concat = require('gulp-concat'); +var rename = require('gulp-rename'); +var coffee = require('gulp-coffee'); +var cache = require('gulp-cached'); +var remember = require('gulp-remember'); +var plumber = require('gulp-plumber'); + +var source = ["lib/**/*.coffee", "index.coffee"] + +gulp.task('coffee', function() { + return gulp.src(source, {base: "."}) + .pipe(plumber()) + .pipe(cache("coffee")) + .pipe(coffee({bare: true}).on('error', gutil.log)).on('data', gutil.log) + .pipe(remember("coffee")) + .pipe(gulp.dest(".")); +}); + +gulp.task('watch', function () { + gulp.watch(source, ['coffee']); +}); + +gulp.task('default', ['coffee', 'watch']); \ No newline at end of file diff --git a/index.coffee b/index.coffee new file mode 100644 index 0000000..97e64bb --- /dev/null +++ b/index.coffee @@ -0,0 +1 @@ +module.exports = require "./lib/form-data2.coffee" diff --git a/index.js b/index.js new file mode 100644 index 0000000..82b270f --- /dev/null +++ b/index.js @@ -0,0 +1 @@ +module.exports = require("./lib/form-data2.coffee"); diff --git a/lib/form-data2.coffee b/lib/form-data2.coffee new file mode 100644 index 0000000..950e75b --- /dev/null +++ b/lib/form-data2.coffee @@ -0,0 +1,131 @@ +# Standard library +path = require "path" +stream = require "stream" + +# Third-party dependencies +uuid = require "uuid" +mime = require "mime" +combinedStream2 = require "combined-stream2" +Promise = require "bluebird" +_ = require "lodash" +debug = require("debug")("form-data2") + +CRLF = "\r\n" + +# Utility functions +ofTypes = (obj, types) -> + match = false + for type in types + match = match or obj instanceof type + return match + + +module.exports = class FormData + constructor: -> + @_firstHeader = false + @_closingHeaderAppended = false + @_boundary = "----" + uuid.v4() + @_headers = { "content-type": "multipart/form-data; boundary=#{@_boundary}" } + @_stream = combinedStream2.create() + + _getStreamMetadata: (source, options) -> # FIXME: Make work with deferred sources (ie. callback-provided) + debug "obtaining metadata for source: %s", source.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r") + fullPath = options.filename ? source.client?._httpMessage?.path ? source.path + + if fullPath? # This is a file... + filename = path.basename(fullPath) + contentType = options.contentType ? source.headers?["content-type"] ? mime.lookup(filename) + contentLength = options.knownLength ? options.contentLength ? source.headers?["content-length"] # FIXME: Is this even used anywhere? + else # Probably just a plaintext form value, or an unidentified stream + contentType = options.contentType ? source.headers?["content-type"] + contentLength = options.knownLength ? options.contentLength ? source.headers?["content-length"] + + return {filename: filename, contentType: contentType, contentLength: contentLength} + + _generateHeaderFields: (name, metadata) -> + debug "generating headers for: %s", metadata + headerFields = [] + + if metadata.filename? + escapedFilename = metadata.filename.replace '"', '\\"' + headerFields.push "Content-Disposition: form-data; name=\"#{name}\"; filename=\"#{escapedFilename}\"" + else + headerFields.push "Content-Disposition: form-data; name=\"#{name}\"" + + if metadata.contentType? + headerFields.push "Content-Type: #{metadata.contentType}" + + debug "generated headers: %s", headerFields + return headerFields.join CRLF + + _appendHeader: (name, metadata) -> + if @_firstHeader == false + debug "appending header" + leadingCRLF = "" + @_firstHeader = true + else + debug "appending first header" + leadingCRLF = CRLF + + headerFields = @_generateHeaderFields name, metadata + + @_stream.append new Buffer(leadingCRLF + "--#{@_boundary}" + CRLF + headerFields + CRLF + CRLF) + + _appendClosingHeader: -> + debug "appending closing header" + @_stream.append new Buffer(CRLF + "--#{@_boundary}--") + + append: (name, source, options = {}) -> + debug "appending source" + if @_closingHeaderAppended + throw new Error "The stream has already been prepared for usage; you either piped it or generated the HTTP headers. No new sources can be appended anymore." + + if not ofTypes(source, [stream.Readable, stream.Duplex, stream.Transform, Buffer, Function]) and typeof source != "string" + throw new Error "The provided value must be either a readable stream, a Buffer, a callback providing either of those, or a string." + + if typeof source == "string" + source = new Buffer(source) # If the string isn't UTF-8, this won't end well! + options.contentType ?= "text/plain" + + metadata = @_getStreamMetadata source, options + @_appendHeader name, metadata + + @_stream.append source, options + + done: -> + # This method should be called when the user is finished adding streams. It adds the termination header at the end of the combined stream. When piping, this method is automatically called! + debug "called 'done'" + + if not @_closingHeaderAppended + @_closingHeaderAppended = true + @_appendClosingHeader() + + getBoundary: -> + return @_boundary + + getHeaders: (callback) -> + # Returns the headers needed to correctly transmit the generated multipart/form-data blob. We will first need to call @done() to make sure that the multipart footer is there - from this point on, no new sources can be appended anymore. + @done() + + Promise.try => + @_stream.getCombinedStreamLength() + .then (length) -> + debug "total combined stream length: %s", length + Promise.resolve { "content-length": length } + .catch (err) -> + # We couldn't get the stream length, most likely there was a stream involved that `stream-length` does not support. + debug "WARN: could not get total combined stream length" + Promise.resolve { "transfer-encoding": "chunked" } + .then (sizeHeaders) => + Promise.resolve _.extend(sizeHeaders, @_headers) + .nodeify(callback) + + getLength: (callback) -> + @_stream.getCombinedStreamLength(callback) + + pipe: (target) -> + @done() + + # Pass through to the underlying `combined-stream`. + debug "piping underlying combined-stream2 to target writable" + @_stream.pipe target diff --git a/lib/form-data2.js b/lib/form-data2.js new file mode 100644 index 0000000..56c38af --- /dev/null +++ b/lib/form-data2.js @@ -0,0 +1,167 @@ +var CRLF, FormData, Promise, combinedStream2, debug, mime, ofTypes, path, stream, uuid, _; + +path = require("path"); + +stream = require("stream"); + +uuid = require("uuid"); + +mime = require("mime"); + +combinedStream2 = require("combined-stream2"); + +Promise = require("bluebird"); + +_ = require("lodash"); + +debug = require("debug")("form-data2"); + +CRLF = "\r\n"; + +ofTypes = function(obj, types) { + var match, type, _i, _len; + match = false; + for (_i = 0, _len = types.length; _i < _len; _i++) { + type = types[_i]; + match = match || obj instanceof type; + } + return match; +}; + +module.exports = FormData = (function() { + function FormData() { + this._firstHeader = false; + this._closingHeaderAppended = false; + this._boundary = "----" + uuid.v4(); + this._headers = { + "content-type": "multipart/form-data; boundary=" + this._boundary + }; + this._stream = combinedStream2.create(); + } + + FormData.prototype._getStreamMetadata = function(source, options) { + var contentLength, contentType, filename, fullPath, _ref, _ref1, _ref10, _ref11, _ref12, _ref13, _ref14, _ref2, _ref3, _ref4, _ref5, _ref6, _ref7, _ref8, _ref9; + debug("obtaining metadata for source: %s", source.toString().replace(/\n/g, "\\n").replace(/\r/g, "\\r")); + fullPath = (_ref = (_ref1 = options.filename) != null ? _ref1 : (_ref2 = source.client) != null ? (_ref3 = _ref2._httpMessage) != null ? _ref3.path : void 0 : void 0) != null ? _ref : source.path; + if (fullPath != null) { + filename = path.basename(fullPath); + contentType = (_ref4 = (_ref5 = options.contentType) != null ? _ref5 : (_ref6 = source.headers) != null ? _ref6["content-type"] : void 0) != null ? _ref4 : mime.lookup(filename); + contentLength = (_ref7 = (_ref8 = options.knownLength) != null ? _ref8 : options.contentLength) != null ? _ref7 : (_ref9 = source.headers) != null ? _ref9["content-length"] : void 0; + } else { + contentType = (_ref10 = options.contentType) != null ? _ref10 : (_ref11 = source.headers) != null ? _ref11["content-type"] : void 0; + contentLength = (_ref12 = (_ref13 = options.knownLength) != null ? _ref13 : options.contentLength) != null ? _ref12 : (_ref14 = source.headers) != null ? _ref14["content-length"] : void 0; + } + return { + filename: filename, + contentType: contentType, + contentLength: contentLength + }; + }; + + FormData.prototype._generateHeaderFields = function(name, metadata) { + var escapedFilename, headerFields; + debug("generating headers for: %s", metadata); + headerFields = []; + if (metadata.filename != null) { + escapedFilename = metadata.filename.replace('"', '\\"'); + headerFields.push("Content-Disposition: form-data; name=\"" + name + "\"; filename=\"" + escapedFilename + "\""); + } else { + headerFields.push("Content-Disposition: form-data; name=\"" + name + "\""); + } + if (metadata.contentType != null) { + headerFields.push("Content-Type: " + metadata.contentType); + } + debug("generated headers: %s", headerFields); + return headerFields.join(CRLF); + }; + + FormData.prototype._appendHeader = function(name, metadata) { + var headerFields, leadingCRLF; + if (this._firstHeader === false) { + debug("appending header"); + leadingCRLF = ""; + this._firstHeader = true; + } else { + debug("appending first header"); + leadingCRLF = CRLF; + } + headerFields = this._generateHeaderFields(name, metadata); + return this._stream.append(new Buffer(leadingCRLF + ("--" + this._boundary) + CRLF + headerFields + CRLF + CRLF)); + }; + + FormData.prototype._appendClosingHeader = function() { + debug("appending closing header"); + return this._stream.append(new Buffer(CRLF + ("--" + this._boundary + "--"))); + }; + + FormData.prototype.append = function(name, source, options) { + var metadata; + if (options == null) { + options = {}; + } + debug("appending source"); + if (this._closingHeaderAppended) { + throw new Error("The stream has already been prepared for usage; you either piped it or generated the HTTP headers. No new sources can be appended anymore."); + } + if (!ofTypes(source, [stream.Readable, stream.Duplex, stream.Transform, Buffer, Function]) && typeof source !== "string") { + throw new Error("The provided value must be either a readable stream, a Buffer, a callback providing either of those, or a string."); + } + if (typeof source === "string") { + source = new Buffer(source); + if (options.contentType == null) { + options.contentType = "text/plain"; + } + } + metadata = this._getStreamMetadata(source, options); + this._appendHeader(name, metadata); + return this._stream.append(source, options); + }; + + FormData.prototype.done = function() { + debug("called 'done'"); + if (!this._closingHeaderAppended) { + this._closingHeaderAppended = true; + return this._appendClosingHeader(); + } + }; + + FormData.prototype.getBoundary = function() { + return this._boundary; + }; + + FormData.prototype.getHeaders = function(callback) { + this.done(); + return Promise["try"]((function(_this) { + return function() { + return _this._stream.getCombinedStreamLength(); + }; + })(this)).then(function(length) { + debug("total combined stream length: %s", length); + return Promise.resolve({ + "content-length": length + }); + })["catch"](function(err) { + debug("WARN: could not get total combined stream length"); + return Promise.resolve({ + "transfer-encoding": "chunked" + }); + }).then((function(_this) { + return function(sizeHeaders) { + return Promise.resolve(_.extend(sizeHeaders, _this._headers)); + }; + })(this)).nodeify(callback); + }; + + FormData.prototype.getLength = function(callback) { + return this._stream.getCombinedStreamLength(callback); + }; + + FormData.prototype.pipe = function(target) { + this.done(); + debug("piping underlying combined-stream2 to target writable"); + return this._stream.pipe(target); + }; + + return FormData; + +})(); diff --git a/package.json b/package.json new file mode 100644 index 0000000..fa4b825 --- /dev/null +++ b/package.json @@ -0,0 +1,41 @@ +{ + "name": "form-data2", + "version": "1.0.0", + "description": "A Streams2-compatible drop-in replacement for the `form-data` module.", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git://github.com/joepie91/node-form-data2" + }, + "keywords": [ + "form-data", + "multipart", + "http", + "stream" + ], + "author": "Sven Slootweg", + "license": "WTFPL", + "devDependencies": { + "gulp": "~3.8.0", + "gulp-cached": "~0.0.3", + "gulp-coffee": "~2.0.1", + "gulp-concat": "~2.2.0", + "gulp-livereload": "~2.1.0", + "gulp-nodemon": "~1.0.4", + "gulp-plumber": "~0.6.3", + "gulp-remember": "~0.2.0", + "gulp-rename": "~1.2.0", + "gulp-util": "~2.2.17" + }, + "dependencies": { + "bluebird": "^2.8.2", + "combined-stream2": "^1.0.2", + "debug": "^2.1.1", + "lodash": "^2.4.1", + "mime": "^1.2.11", + "uuid": "^2.0.1" + } +}