v1.0.0
Sven Slootweg
10 years ago
parent
c2ea38c968
commit
59f7ebabce
@ -1,3 +1,4 @@
|
||||
# https://git-scm.com/docs/gitignore
|
||||
# https://help.github.com/articles/ignoring-files
|
||||
# Example .gitignore files: https://github.com/github/gitignore
|
||||
# Example .gitignore files: https://github.com/github/gitignore
|
||||
/node_modules/
|
||||
|
||||
@ -0,0 +1,182 @@
|
||||
# bhttp
|
||||
|
||||
A sane HTTP client library for Node.js with Streams2 support.
|
||||
|
||||
## Why bhttp?
|
||||
|
||||
There are already a few commonly used HTTP client libraries for Node.js, but all of them have issues:
|
||||
|
||||
* The core `http` module is rather low-level, and even relatively simple requests take a lot of work to make correctly. It also automatically uses an agent for HTTP requests, which slows down concurrent HTTP requests when you're streaming the responses somewhere.
|
||||
* `request` is buggy, only supports old-style streams, has the same 'agent' problem as `http`, the documentation is poor, and the API is not very intuitive.
|
||||
* `needle` is a lot simpler, but suffers from the same 'agent' problem, and the API can be a bit annoying in some ways. It also doesn't have a proper session API.
|
||||
* `hyperquest` (mostly) solves the 'agent' problem correctly, but has a very spartan API. Making non-GET requests is more complex than it should be.
|
||||
|
||||
All these issues (and more) are solved in `bhttp`. It offers the following:
|
||||
|
||||
* A simple, well-documented API.
|
||||
* Sane default behaviour.
|
||||
* Minimal behind-the-scenes 'magic', meaning less opportunities for bugs to be introduced. No 'gotchas' in dealing with response streams either.
|
||||
* Support for `multipart/form-data` (eg. file uploads), __with support for Streams2__, and support for legacy streams.
|
||||
* Fully automatic detection of desired payload type - URL-encoded, multipart/form-data, or even a stream or Buffer directly. Just give it the data you want to send, and it will make sure it arrives correctly. Optionally, you can also specify JSON encoding (for JSON APIs).
|
||||
* Easy-to-use session mechanics - a new session will automatically give you a new cookie jar, cookies are kept track of automatically, and 'default options' are deep-merged.
|
||||
* Streaming requests are kept out of the agent pool - ie. no blocking of other requests.
|
||||
* Optionally, a Promises API (you can also use nodebacks).
|
||||
|
||||
## License
|
||||
|
||||
[WTFPL](http://www.wtfpl.net/txt/copying/) or [CC0](https://creativecommons.org/publicdomain/zero/1.0/), whichever you prefer. A donation and/or attribution are appreciated, but not required.
|
||||
|
||||
## Donate
|
||||
|
||||
My income consists entirely of donations for my projects. If this module is useful to you, consider [making a donation](http://cryto.net/~joepie91/donate.html)!
|
||||
|
||||
You can donate using Bitcoin, PayPal, Gratipay, Flattr, cash-in-mail, SEPA transfers, and pretty much anything else.
|
||||
|
||||
## Contributing
|
||||
|
||||
Pull requests welcome. Please make sure your modifications are in line with the overall code style, and ensure that you're editing the `.coffee` files, not the `.js` files.
|
||||
|
||||
Build tool of choice is `gulp`; simply run `gulp` while developing, and it will watch for changes.
|
||||
|
||||
Be aware that by making a pull request, you agree to release your modifications under the licenses stated above.
|
||||
|
||||
## Usage
|
||||
|
||||
```javascript
|
||||
var bhttp = require("bhttp");
|
||||
|
||||
Promise.try(function() {
|
||||
return bhttp.get("http://somesite.com/bigfile.mp4", {stream: true});
|
||||
}).then(function(response) {
|
||||
return bhttp.post("http://somehostingservice.com/upload", {
|
||||
fileOne: response,
|
||||
fileTwo: fs.createReadStream("./otherbigfile.mkv")
|
||||
});
|
||||
}).then(function(response) {
|
||||
console.log("Response from hosting service:", response.body.toString());
|
||||
});
|
||||
```
|
||||
|
||||
... or, using nodebacks:
|
||||
|
||||
```javascript
|
||||
var bhttp = require("bhttp");
|
||||
|
||||
bhttp.get("http://somesite.com/bigfile.mp4", {stream: true}, function(err, responseOne) {
|
||||
var payload = {
|
||||
fileOne: responseOne,
|
||||
fileTwo: fs.createReadStream("./otherbigfile.mkv")
|
||||
};
|
||||
|
||||
bhttp.post("http://somehostingservice.com/upload", payload, {}, function(err, responseTwo) {
|
||||
console.log("Response from hosting service:", responseTwo.body.toString());
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
### Sessions
|
||||
|
||||
```javascript
|
||||
var bhttp = require("bhttp");
|
||||
|
||||
var session = bhttp.session({ headers: {"user-agent": "MyCustomUserAgent/2.0"} });
|
||||
|
||||
// Our new session now automatically has a cookie jar, and also uses our preset option(s).
|
||||
|
||||
Promise.try(function(){
|
||||
session.get("http://hypotheticalsite.com/cookietest"); // Assume that this site now sets a cookie
|
||||
}).then(function(response){
|
||||
session.get("http://hypotheticalsite.com/other-endpoint"); // This now sends along the cookie!
|
||||
});
|
||||
```
|
||||
|
||||
## API
|
||||
|
||||
### bhttp.head(url, [options, [callback]])
|
||||
### bhttp.get(url, [options, [callback]])
|
||||
### bhttp.delete(url, [options, [callback]])
|
||||
### bhttp.post(url, [data, [options, [callback]]])
|
||||
### bhttp.put(url, [data, [options, [callback]]])
|
||||
### bhttp.patch(url, [data, [options, [callback]]])
|
||||
|
||||
Convenience methods that pre-set the request method, and automatically send along the payload using the correct options.
|
||||
|
||||
* __url__: The URL to request, with protocol. When using HTTPS, please be sure to read the 'Caveats' section.
|
||||
* __data__: *Optional, only for POST/PUT/PATCH.* The payload to send along.
|
||||
* __options__: *Optional.* Extra options for the request. More details under the documentation forthe `bhttp.request` method below.
|
||||
* __callback__: *Optional.* When using the nodeback API, the callback to use. If not specified, a Promise will be returned.
|
||||
|
||||
The `data` payload can be one of the following things:
|
||||
|
||||
* __String / Buffer__: The contents will be written to the request as-is.
|
||||
* __A stream__: The entire stream will be written to the request as-is.
|
||||
* __An object__: Will be encoded as form data, and can contain any combination of Strings, Buffers, streams, and arrays of any of those. When only strings are used, the form data is querystring-encoded - if Buffers or streams are used, it will be encoded as multipart/form-data.
|
||||
|
||||
### bhttp.request(url, [options, [callback]])
|
||||
|
||||
Makes a request, and returns the response object asynchronously. The response object is a standard `http.IncomingMessages` with a few additional properties (documented below the argument list).
|
||||
|
||||
* __url__: The URL to request, with protocol. When using HTTPS, please be sure to read the 'Caveats' section.
|
||||
* __options__: *Optional.* Extra options for the request. Any other options not listed here will be passed on directly to the `http` or `https` module.
|
||||
* __Basic options__
|
||||
* __stream__: *Defaults to `false`.* Whether the response is meant to be streamed. If `true`, the response body won't be parsed, an unread response stream is returned, and the request is kept out of the 'agent' pool.
|
||||
* __headers__: Any extra request headers to set. (Non-custom) header names must be lowercase.
|
||||
* __followRedirects__: *Defaults to `true`.* Whether to automatically follow redirects or not (the redirect history is available as the `redirectHistory` property on the response).
|
||||
* __redirectLimit__: *Defaults to `10`.* The maximum amount of redirects to follow before erroring out, to prevent redirect loops.
|
||||
* __Encoding and decoding__
|
||||
* __forceMultipart__: *Defaults to `false`.* Ensures that `mulipart/form-data` encoding is used, no matter what the payload contents are.
|
||||
* __encodeJSON__: *Defaults to `false`.* When set to `true`, the request payload will be encoded as JSON. This cannot be used if you are using any streams in your payload.
|
||||
* __decodeJSON__: *Defaults to `false`.* When set to `true`, the response will always be decoded as JSON, no matter what the `Content-Type` says. You'll probably want to keep this set to `false` - most APIs send the correct `Content-Type` headers, and in those cases `bhttp` will automatically decode the response as JSON.
|
||||
* __noDecode__: *Defaults to `false`.* Never decode the response, even if the `Content-Type` says that it's JSON.
|
||||
* __Request payloads__ (you won't need these when using the shorthand methods)
|
||||
* __inputBuffer__: A Buffer or String to send as the entire payload.
|
||||
* __inputStream__: A stream to send as the entire payload.
|
||||
* __formFields__: Form data to encode. This can also include files to upload.
|
||||
* __files__: Form data to send explicitly as a file. This will automatically enable `multipart/form-data` encoding.
|
||||
* __Advanced options__
|
||||
* __method__: The request method to use. You don't need this when using the shorthand methods.
|
||||
* __cookieJar__: A custom cookie jar to use. You'll probably want to use `bhttp.session()` instead.
|
||||
* __allowChunkedMultipart__: *Defaults to `false`.* Many servers don't support `multipart/form-data` when it is transmitted with chunked transfer encoding (eg. when the stream length is unknown), and silently fail with an empty request payload - this is why `bhttp` disallows it by default. If you are *absolutely certain* that the endpoint supports this functionality, you can override the behaviour by setting this to `true`.
|
||||
* __discardResponse__: *Defaults to `false`.* Whether to throw away the response without reading it. Only really useful for fire-and-forget calls. This is almost never what you want.
|
||||
* __keepRedirectResponses__: *Defaults to `false`.* Whether to keep the response streams of redirects. You probably don't need this. __When enabling this, you must *explicitly* read out every single redirect response, or you will experience memory leaks!__
|
||||
* __justPrepare__: *Defaults to `false`.* When set to `true`, bhttp just prepares the request, and doesn't actually carry it out; useful if you want to make some manual modifications. Instead of a response, the method will asynchronously return an array with the signature `[request, response, requestState]` that you will need to pass into the `bhttp.makeRequest()` method.
|
||||
|
||||
* __callback__: *Optional.* When using the nodeback API, the callback to use. If not specified, a Promise will be returned.
|
||||
|
||||
A few extra properties are set on the response object (which is a `http.IncomingMessage`):
|
||||
|
||||
* __body__: When `stream` is set to `false` (the default), this will contain the response body. This can be either a string or, in the case of a JSON response, a decoded JSON object.
|
||||
* __redirectHistory__: An array containing the redirect responses, if any, in chronological order. Response bodies are discarded by default; if you do not want this, use the `keepRedirectResponses` option.
|
||||
* __request__: The request configuration that was generated by `bhttp`. You probably don't need this.
|
||||
* __requestState__: The request state that was accumulated by `bhttp`. You probably don't need this.
|
||||
|
||||
`bhttp` can automatically parse the metadata for the following types of streams:
|
||||
|
||||
* `fs` streams
|
||||
* `http` and `bhttp` responses
|
||||
* `request` requests
|
||||
* `combined-stream` streams (assuming all the underlying streams are of one of the types listed here)
|
||||
|
||||
If you are using a different type of streams, you can wrap the stream using `bhttp.wrapStream` to manually specify the needed metadata.
|
||||
|
||||
### bhttp.session([defaultOptions])
|
||||
|
||||
This will create a new session. The `defaultOptions` will be deep-merged with the options specified for each request (where the request-specific options have priority).
|
||||
|
||||
A new cookie jar is automatically created, unless you either specify a custom `cookieJar` option or set the `cookieJar` option to `false` (in which case no cookie jar is used).
|
||||
|
||||
### bhttp.wrapStream(stream, options)
|
||||
|
||||
This will return a 'stream wrapper' containing explicit metadata for a stream. You'll need to use it when passing an unsupported type of stream to a `data` parameter or `formFields`/`files` option.
|
||||
|
||||
* __stream__: The stream to wrap.
|
||||
* __options__: The options for this stream. All options are optional, but recommended to specify.
|
||||
* __contentLength__: The length of the stream in bytes.
|
||||
* __contentType__: The MIME type of the stream.
|
||||
* __filename__: The filename of the stream.
|
||||
|
||||
The resulting wrapper can be passed on to the `bhttp` methods as if it were a regular stream.
|
||||
|
||||
### bhttp.makeRequest(request, response, requestState)
|
||||
|
||||
When using the `justPrepare` option, you can use this method to proceed with the request after manual modifications. The function signature is identical to the signature of the array returned when using `justPrepare`. `response` will usually be `null`, but must be passed on as is, to account for future API changes.
|
||||
@ -0,0 +1,28 @@
|
||||
var gulp = require('gulp');
|
||||
|
||||
/* CoffeeScript compile deps */
|
||||
var path = require('path');
|
||||
var gutil = require('gulp-util');
|
||||
var concat = require('gulp-concat');
|
||||
var rename = require('gulp-rename');
|
||||
var coffee = require('gulp-coffee');
|
||||
var cache = require('gulp-cached');
|
||||
var remember = require('gulp-remember');
|
||||
var plumber = require('gulp-plumber');
|
||||
|
||||
var source = ["lib/**/*.coffee", "index.coffee"]
|
||||
|
||||
gulp.task('coffee', function() {
|
||||
return gulp.src(source, {base: "."})
|
||||
.pipe(plumber())
|
||||
.pipe(cache("coffee"))
|
||||
.pipe(coffee({bare: true}).on('error', gutil.log)).on('data', gutil.log)
|
||||
.pipe(remember("coffee"))
|
||||
.pipe(gulp.dest("."));
|
||||
});
|
||||
|
||||
gulp.task('watch', function () {
|
||||
gulp.watch(source, ['coffee']);
|
||||
});
|
||||
|
||||
gulp.task('default', ['coffee', 'watch']);
|
||||
@ -0,0 +1 @@
|
||||
module.exports = require "./lib/bhttp"
|
||||
@ -0,0 +1,545 @@
|
||||
# FIXME: Force-lowercase user-supplied headers before merging them into the request?
|
||||
# FIXME: Deep-merge query-string arguments between URL and argument?
|
||||
# FIXME: Named arrays for multipart/form-data?
|
||||
|
||||
# Core modules
|
||||
urlUtil = require "url"
|
||||
querystring = require "querystring"
|
||||
stream = require "stream"
|
||||
http = require "http"
|
||||
https = require "https"
|
||||
util = require "util"
|
||||
|
||||
# Utility modules
|
||||
Promise = require "bluebird"
|
||||
_ = require "lodash"
|
||||
S = require "string"
|
||||
formFixArray = require "form-fix-array"
|
||||
errors = require "errors"
|
||||
debug = require("debug")("bhttp")
|
||||
|
||||
# Other third-party modules
|
||||
formData = require "form-data2"
|
||||
concatStream = require "concat-stream"
|
||||
toughCookie = require "tough-cookie"
|
||||
streamLength = require "stream-length"
|
||||
|
||||
# For the version in the user agent, etc.
|
||||
packageConfig = require "../package.json"
|
||||
|
||||
# Error types
|
||||
|
||||
errors.create
|
||||
name: "bhttpError"
|
||||
|
||||
errors.create
|
||||
name: "ConflictingOptionsError"
|
||||
parents: errors.bhttpError
|
||||
|
||||
errors.create
|
||||
name: "UnsupportedProtocolError"
|
||||
parents: errors.bhttpError
|
||||
|
||||
errors.create
|
||||
name: "RedirectError"
|
||||
parents: errors.bhttpError
|
||||
|
||||
errors.create
|
||||
name: "MultipartError"
|
||||
parents: errors.bhttpError
|
||||
|
||||
# Utility functions
|
||||
|
||||
ofTypes = (obj, types) ->
|
||||
match = false
|
||||
for type in types
|
||||
match = match or obj instanceof type
|
||||
return match
|
||||
|
||||
addErrorData = (err, request, response, requestState) ->
|
||||
err.request = request
|
||||
err.response = response
|
||||
err.requestState = requestState
|
||||
return err
|
||||
|
||||
isStream = (obj) ->
|
||||
obj? and (ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]) or obj.hasOwnProperty("_bhttpStreamWrapper"))
|
||||
|
||||
# Middleware
|
||||
# NOTE: requestState is an object that signifies the current state of the overall request; eg. for a response involving one or more redirects, it will hold a 'redirect history'.
|
||||
prepareSession = (request, response, requestState) ->
|
||||
debug "preparing session"
|
||||
Promise.try ->
|
||||
if requestState.sessionOptions?
|
||||
# Request options take priority over session options
|
||||
request.options = _.merge _.clone(requestState.sessionOptions), request.options
|
||||
|
||||
# Create a headers parameter if it doesn't exist yet - we'll need to add some stuff to this later on
|
||||
# FIXME: We may need to do a deep-clone of other mutable options later on as well; otherwise, when getting a redirect in a session with pre-defined options, the contents may not be correctly cleared after following the redirect.
|
||||
if request.options.headers?
|
||||
request.options.headers = _.clone(request.options.headers, true)
|
||||
else
|
||||
request.options.headers = {}
|
||||
|
||||
# If we have a cookie jar, start out by setting the cookie string.
|
||||
if request.options.cookieJar?
|
||||
Promise.try ->
|
||||
# Move the cookieJar to the request object, the http/https module doesn't need it.
|
||||
request.cookieJar = request.options.cookieJar
|
||||
delete request.options.cookieJar
|
||||
|
||||
# Get the current cookie string for the URL
|
||||
request.cookieJar.get request.url
|
||||
.then (cookieString) ->
|
||||
debug "sending cookie string: %s", cookieString
|
||||
request.options.headers["cookie"] = cookieString
|
||||
Promise.resolve [request, response, requestState]
|
||||
else
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
prepareDefaults = (request, response, requestState) ->
|
||||
debug "preparing defaults"
|
||||
Promise.try ->
|
||||
# These are the options that we need for response processing, but don't need to be passed on to the http/https module.
|
||||
request.responseOptions =
|
||||
discardResponse: request.options.discardResponse ? false
|
||||
keepRedirectResponses: request.options.keepRedirectResponses ? false
|
||||
followRedirects: request.options.followRedirects ? true
|
||||
noDecode: request.options.noDecode ? false
|
||||
decodeJSON: request.options.decodeJSON ? false
|
||||
stream: request.options.stream ? false
|
||||
justPrepare: request.options.justPrepare ? false
|
||||
redirectLimit: request.options.redirectLimit ? 10
|
||||
|
||||
# Whether chunked transfer encoding for multipart/form-data payloads is acceptable. This is likely to break quietly on a lot of servers.
|
||||
request.options.allowChunkedMultipart ?= false
|
||||
|
||||
# Whether we should always use multipart/form-data for payloads, even if querystring-encoding would be a possibility.
|
||||
request.options.forceMultipart ?= false
|
||||
|
||||
# If no custom user-agent is defined, set our own
|
||||
request.options.headers["user-agent"] ?= "bhttp/#{packageConfig.version}"
|
||||
|
||||
# Normalize the request method to lowercase.
|
||||
request.options.method = request.options.method.toLowerCase()
|
||||
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
prepareUrl = (request, response, requestState) ->
|
||||
debug "preparing URL"
|
||||
Promise.try ->
|
||||
# Parse the specified URL, and use the resulting information to build a complete `options` object
|
||||
urlOptions = urlUtil.parse request.url, true
|
||||
|
||||
_.extend request.options, {hostname: urlOptions.hostname, port: urlOptions.port}
|
||||
request.options.path = urlUtil.format {pathname: urlOptions.pathname, query: request.options.query ? urlOptions.query}
|
||||
request.protocol = S(urlOptions.protocol).chompRight(":").toString()
|
||||
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
prepareProtocol = (request, response, requestState) ->
|
||||
debug "preparing protocol"
|
||||
Promise.try ->
|
||||
request.protocolModule = switch request.protocol
|
||||
when "http" then http
|
||||
when "https" then https # CAUTION / FIXME: Node will silently ignore SSL settings without a custom agent!
|
||||
else null
|
||||
|
||||
if not request.protocolModule?
|
||||
return Promise.reject() new errors.UnsupportedProtocolError "The protocol specified (#{protocol}) is not currently supported by this module."
|
||||
|
||||
request.options.port ?= switch request.protocol
|
||||
when "http" then 80
|
||||
when "https" then 443
|
||||
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
prepareOptions = (request, response, requestState) ->
|
||||
debug "preparing options"
|
||||
Promise.try ->
|
||||
# Do some sanity checks - there are a number of options that cannot be used together
|
||||
if (request.options.formFields? or request.options.files?) and (request.options.inputStream? or request.options.inputBuffer?)
|
||||
return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot define both formFields/files and a raw inputStream or inputBuffer."), request, response, requestState)
|
||||
|
||||
if request.options.encodeJSON and (request.options.inputStream? or request.options.inputBuffer?)
|
||||
return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot use both encodeJSON and a raw inputStream or inputBuffer.", undefined, "If you meant to JSON-encode the stream, you will currently have to do so manually."), request, response, requestState)
|
||||
|
||||
# If the user plans on streaming the response, we need to disable the agent entirely - otherwise the streams will block the pool.
|
||||
if request.responseOptions.stream
|
||||
request.options.agent ?= false
|
||||
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
preparePayload = (request, response, requestState) ->
|
||||
debug "preparing payload"
|
||||
Promise.try ->
|
||||
# If a 'files' parameter is present, then we will send the form data as multipart data - it's most likely binary data.
|
||||
multipart = request.options.forceMultipart or request.options.files?
|
||||
|
||||
# Similarly, if any of the formFields values are either a Stream or a Buffer, we will assume that the form should be sent as multipart.
|
||||
multipart = multipart or _.any request.options.formFields, (item) ->
|
||||
item instanceof Buffer or isStream(item)
|
||||
|
||||
# Really, 'files' and 'formFields' are the same thing - they mostly have different names for 1) clarity and 2) multipart detection. We combine them here.
|
||||
_.extend request.options.formFields, request.options.files
|
||||
|
||||
# For a last sanity check, we want to know whether there are any Stream objects in our form data *at all* - these can't be used when encodeJSON is enabled.
|
||||
containsStreams = _.any request.options.formFields, (item) -> isStream(item)
|
||||
|
||||
if request.options.encodeJSON and containsStreams
|
||||
return Promise.reject() new errors.ConflictingOptionsError "Sending a JSON-encoded payload containing data from a stream is not currently supported.", undefined, "Either don't use encodeJSON, or read your stream into a string or Buffer."
|
||||
|
||||
if request.options.method in ["post", "put", "patch"]
|
||||
# Prepare the payload, and set the appropriate headers.
|
||||
if (request.options.encodeJSON or request.options.formFields?) and not multipart
|
||||
# We know the payload and its size in advance.
|
||||
debug "got url-encodable form-data"
|
||||
request.options.headers["content-type"] = "application/x-www-form-urlencoded"
|
||||
|
||||
if request.options.encodeJSON
|
||||
request.payload = JSON.stringify request.options.formFields ? null
|
||||
else if not _.isEmpty request.options.formFields
|
||||
# The `querystring` module copies the key name verbatim, even if the value is actually an array. Things like PHP don't understand this, and expect every array-containing key to be suffixed with []. We'll just append that ourselves, then.
|
||||
request.payload = querystring.stringify formFixArray(request.options.formFields)
|
||||
else
|
||||
request.payload = ""
|
||||
|
||||
request.options.headers["content-length"] = request.payload.length
|
||||
|
||||
return Promise.resolve()
|
||||
else if request.options.formFields? and multipart
|
||||
# This is going to be multipart data, and we'll let `form-data` set the headers for us.
|
||||
debug "got multipart form-data"
|
||||
formDataObject = new formData()
|
||||
|
||||
for fieldName, fieldValue of formFixArray(request.options.formFields)
|
||||
if not _.isArray fieldValue
|
||||
fieldValue = [fieldValue]
|
||||
|
||||
for valueElement in fieldValue
|
||||
if valueElement._bhttpStreamWrapper?
|
||||
streamOptions = valueElement.options
|
||||
valueElement = valueElement.stream
|
||||
else
|
||||
streamOptions = {}
|
||||
|
||||
formDataObject.append fieldName, valueElement, streamOptions
|
||||
|
||||
request.payloadStream = formDataObject
|
||||
|
||||
Promise.try ->
|
||||
formDataObject.getHeaders()
|
||||
.then (headers) ->
|
||||
if headers["content-transfer-encoding"] == "chunked" and not request.options.allowChunkedMultipart
|
||||
Promise.reject addErrorData(new MultipartError("Most servers do not support chunked transfer encoding for multipart/form-data payloads, and we could not determine the length of all the input streams. See the documentation for more information."), request, response, requestState)
|
||||
else
|
||||
_.extend request.options.headers, headers
|
||||
Promise.resolve()
|
||||
else if request.options.inputStream?
|
||||
# A raw inputStream was provided, just leave it be.
|
||||
debug "got inputStream"
|
||||
Promise.try ->
|
||||
request.payloadStream = request.options.inputStream
|
||||
|
||||
if request.payloadStream._bhttpStreamWrapper? and (request.payloadStream.options.contentLength? or request.payloadStream.options.knownLength?)
|
||||
Promise.resolve(request.payloadStream.options.contentLength ? request.payloadStream.options.knownLength)
|
||||
else
|
||||
streamLength request.options.inputStream
|
||||
.then (length) ->
|
||||
debug "length for inputStream is %s", length
|
||||
request.options.headers["content-length"] = length
|
||||
.catch (err) ->
|
||||
debug "unable to determine inputStream length, switching to chunked transfer encoding"
|
||||
request.options.headers["content-transfer-encoding"] = "chunked"
|
||||
else if request.options.inputBuffer?
|
||||
# A raw inputBuffer was provided, just leave it be (but make sure it's an actual Buffer).
|
||||
debug "got inputBuffer"
|
||||
if typeof request.options.inputBuffer == "string"
|
||||
request.payload = new Buffer(request.options.inputBuffer) # Input string should be utf-8!
|
||||
else
|
||||
request.payload = request.options.inputBuffer
|
||||
|
||||
debug "length for inputBuffer is %s", request.payload.length
|
||||
request.options.headers["content-length"] = request.payload.length
|
||||
|
||||
return Promise.resolve()
|
||||
else
|
||||
# GET, HEAD and DELETE should not have a payload. While technically not prohibited by the spec, it's also not specified, and we'd rather not upset poorly-compliant webservers.
|
||||
# FIXME: Should this throw an Error?
|
||||
return Promise.resolve()
|
||||
.then ->
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
prepareCleanup = (request, response, requestState) ->
|
||||
debug "preparing cleanup"
|
||||
Promise.try ->
|
||||
# Remove the options that we're not going to pass on to the actual http/https library.
|
||||
delete request.options[key] for key in ["query", "formFields", "files", "encodeJSON", "inputStream", "inputBuffer", "discardResponse", "keepRedirectResponses", "followRedirects", "noDecode", "decodeJSON", "allowChunkedMultipart", "forceMultipart"]
|
||||
|
||||
# Lo-Dash apparently has no `map` equivalent for object keys...?
|
||||
fixedHeaders = {}
|
||||
for key, value of request.options.headers
|
||||
fixedHeaders[key.toLowerCase()] = value
|
||||
request.options.headers = fixedHeaders
|
||||
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
# The guts of the module
|
||||
|
||||
prepareRequest = (request, response, requestState) ->
|
||||
debug "preparing request"
|
||||
# FIXME: Mock httpd for testing functionality.
|
||||
Promise.try ->
|
||||
middlewareFunctions = [
|
||||
prepareSession
|
||||
prepareDefaults
|
||||
prepareUrl
|
||||
prepareProtocol
|
||||
prepareOptions
|
||||
preparePayload
|
||||
prepareCleanup
|
||||
]
|
||||
|
||||
promiseChain = Promise.resolve [request, response, requestState]
|
||||
|
||||
middlewareFunctions.forEach (middleware) -> # We must use the functional construct here, to avoid losing references
|
||||
promiseChain = promiseChain.spread (_request, _response, _requestState) ->
|
||||
middleware(_request, _response, _requestState)
|
||||
|
||||
return promiseChain
|
||||
|
||||
makeRequest = (request, response, requestState) ->
|
||||
debug "making %s request to %s", request.options.method.toUpperCase(), request.url
|
||||
Promise.try ->
|
||||
# Instantiate a regular HTTP/HTTPS request
|
||||
req = request.protocolModule.request request.options
|
||||
|
||||
# This is where we write our payload or stream to the request, and the actual request is made.
|
||||
if request.payload?
|
||||
# The entire payload is a single Buffer.
|
||||
debug "sending payload"
|
||||
req.write request.payload
|
||||
req.end()
|
||||
else if request.payloadStream?
|
||||
# The payload is a stream.
|
||||
debug "piping payloadStream"
|
||||
if request.payloadStream._bhttpStreamWrapper?
|
||||
request.payloadStream.stream.pipe req
|
||||
else
|
||||
request.payloadStream.pipe req
|
||||
else
|
||||
# For GET, HEAD, DELETE, etc. there is no payload, but we still need to call end() to complete the request.
|
||||
debug "closing request without payload"
|
||||
req.end()
|
||||
|
||||
new Promise (resolve, reject) ->
|
||||
# In case something goes wrong during this process, somehow...
|
||||
req.on "error", (err) ->
|
||||
reject err
|
||||
|
||||
req.on "response", (res) ->
|
||||
resolve res
|
||||
.then (response) ->
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
processResponse = (request, response, requestState) ->
|
||||
debug "processing response, got status code %s", response.statusCode
|
||||
|
||||
# When we receive the response, we'll buffer it up and/or decode it, depending on what the user specified, and resolve the returned Promise. If the user just wants the raw stream, we resolve immediately after receiving a response.
|
||||
|
||||
Promise.try ->
|
||||
# First, if a cookie jar is set and we received one or more cookies from the server, we should store them in our cookieJar.
|
||||
if request.cookieJar? and response.headers["set-cookie"]?
|
||||
promises = for cookieHeader in response.headers["set-cookie"]
|
||||
debug "storing cookie: %s", cookieHeader
|
||||
request.cookieJar.set cookieHeader, request.url
|
||||
Promise.all promises
|
||||
else
|
||||
Promise.resolve()
|
||||
.then ->
|
||||
# Now the actual response processing.
|
||||
response.request = request
|
||||
response.requestState = requestState
|
||||
response.redirectHistory = requestState.redirectHistory
|
||||
|
||||
if response.statusCode in [301, 302, 303, 307] and request.responseOptions.followRedirects
|
||||
if requestState.redirectHistory.length >= (request.responseOptions.redirectLimit - 1)
|
||||
return Promise.reject addErrorData(new errors.RedirectError("The maximum amount of redirects ({request.responseOptions.redirectLimit}) was reached."))
|
||||
|
||||
# 301: For GET and HEAD, redirect unchanged. For POST, PUT, PATCH, DELETE, "ask user" (in our case: throw an error.)
|
||||
# 302: Redirect, change method to GET.
|
||||
# 303: Redirect, change method to GET.
|
||||
# 307: Redirect, retain method. Make same request again.
|
||||
switch response.statusCode
|
||||
when 301
|
||||
switch request.options.method
|
||||
when "get", "head"
|
||||
return redirectUnchanged request, response, requestState
|
||||
when "post", "put", "patch", "delete"
|
||||
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect for POST, PUT, PATCH or DELETE. RFC says we can't automatically continue."), request, response, requestState)
|
||||
else
|
||||
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect, but not sure how to proceed for the #{request.options.method.toUpperCase()} method."))
|
||||
when 302, 303
|
||||
return redirectGet request, response, requestState
|
||||
when 307
|
||||
if request.containsStreams and request.options.method not in ["get", "head"]
|
||||
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 307 redirect for POST, PUT or DELETE, but your payload contained (single-use) streams. We therefore can't automatically follow the redirect."), request, response, requestState)
|
||||
else
|
||||
return redirectUnchanged request, response, requestState
|
||||
else if request.responseOptions.discardResponse
|
||||
response.resume() # Drain the response stream
|
||||
Promise.resolve response
|
||||
else
|
||||
new Promise (resolve, reject) ->
|
||||
if request.responseOptions.stream
|
||||
resolve response
|
||||
else
|
||||
response.on "error", (err) ->
|
||||
reject err
|
||||
|
||||
response.pipe concatStream (body) ->
|
||||
# FIXME: Separate module for header parsing?
|
||||
if request.responseOptions.decodeJSON or ((response.headers["content-type"] ? "").split(";")[0] == "application/json" and not request.responseOptions.noDecode)
|
||||
try
|
||||
response.body = JSON.parse body
|
||||
catch err
|
||||
reject err
|
||||
else
|
||||
response.body = body
|
||||
|
||||
resolve response
|
||||
|
||||
.then (response) ->
|
||||
Promise.resolve [request, response, requestState]
|
||||
|
||||
# Some wrappers
|
||||
|
||||
doPayloadRequest = (url, data, options) ->
|
||||
# A wrapper that processes the second argument to .post, .put, .patch shorthand API methods.
|
||||
# FIXME: Treat a {} for data as a null? Otherwise {} combined with inputBuffer/inputStream will error.
|
||||
if isStream(data)
|
||||
options.inputStream = data
|
||||
else if ofTypes(data, [Buffer]) or typeof data == "string"
|
||||
options.inputBuffer = data
|
||||
else
|
||||
options.formFields = data
|
||||
|
||||
@request url, options
|
||||
|
||||
redirectGet = (request, response, requestState) ->
|
||||
debug "following forced-GET redirect to %s", response.headers["location"]
|
||||
Promise.try ->
|
||||
options = _.clone(requestState.originalOptions)
|
||||
options.method = "get"
|
||||
|
||||
delete options[key] for key in ["inputBuffer", "inputStream", "files", "formFields"]
|
||||
|
||||
doRedirect request, response, requestState, options
|
||||
|
||||
redirectUnchanged = (request, response, requestState) ->
|
||||
debug "following same-method redirect to %s", response.headers["location"]
|
||||
Promise.try ->
|
||||
options = _.clone(requestState.originalOptions)
|
||||
doRedirect request, response, requestState, options
|
||||
|
||||
doRedirect = (request, response, requestState, newOptions) ->
|
||||
Promise.try ->
|
||||
if not request.responseOptions.keepRedirectResponses
|
||||
response.resume() # Let the response stream drain out...
|
||||
|
||||
requestState.redirectHistory.push response
|
||||
bhttpAPI._doRequest response.headers["location"], newOptions, requestState
|
||||
|
||||
createCookieJar = (jar) ->
|
||||
# Creates a cookie jar wrapper with a simplified API.
|
||||
return {
|
||||
set: (cookie, url) ->
|
||||
new Promise (resolve, reject) =>
|
||||
@jar.setCookie cookie, url, (err, cookie) ->
|
||||
if err then reject(err) else resolve(cookie)
|
||||
get: (url) ->
|
||||
new Promise (resolve, reject) =>
|
||||
@jar.getCookieString url, (err, cookies) ->
|
||||
if err then reject(err) else resolve(cookies)
|
||||
jar: jar
|
||||
}
|
||||
|
||||
# The exposed API
|
||||
|
||||
bhttpAPI =
|
||||
head: (url, options = {}, callback) ->
|
||||
options.method = "head"
|
||||
@request url, options
|
||||
get: (url, options = {}, callback) ->
|
||||
options.method = "get"
|
||||
@request url, options
|
||||
post: (url, data, options = {}, callback) ->
|
||||
options.method = "post"
|
||||
doPayloadRequest.bind(this) url, data, options
|
||||
put: (url, data, options = {}, callback) ->
|
||||
options.method = "put"
|
||||
doPayloadRequest.bind(this) url, data, options
|
||||
patch: (url, data, options = {}, callback) ->
|
||||
options.method = "patch"
|
||||
doPayloadRequest.bind(this) url, data, options
|
||||
delete: (url, data, options = {}, callback) ->
|
||||
options.method = "delete"
|
||||
@request url, options
|
||||
request: (url, options = {}, callback) ->
|
||||
@_doRequest(url, options).nodeify(callback)
|
||||
_doRequest: (url, options, requestState) ->
|
||||
# This is split from the `request` method, so that the user doesn't have to pass in `undefined` for the `requestState` when they want to specify a `callback`.
|
||||
Promise.try =>
|
||||
request = {url: url, options: _.clone(options)}
|
||||
response = null
|
||||
requestState ?= {originalOptions: _.clone(options), redirectHistory: []}
|
||||
requestState.sessionOptions ?= @_sessionOptions ? {}
|
||||
|
||||
prepareRequest request, response, requestState
|
||||
.spread (request, response, requestState) =>
|
||||
if request.responseOptions.justPrepare
|
||||
Promise.resolve [request, response, requestState]
|
||||
else
|
||||
Promise.try ->
|
||||
bhttpAPI.executeRequest request, response, requestState
|
||||
.spread (request, response, requestState) ->
|
||||
# The user likely only wants the response.
|
||||
Promise.resolve response
|
||||
executeRequest: (request, response, requestState) ->
|
||||
# Executes a pre-configured request.
|
||||
Promise.try ->
|
||||
makeRequest request, response, requestState
|
||||
.spread (request, response, requestState) ->
|
||||
processResponse request, response, requestState
|
||||
session: (options) ->
|
||||
options ?= {}
|
||||
options = _.clone options
|
||||
session = {}
|
||||
|
||||
for key, value of this
|
||||
if value instanceof Function
|
||||
value = value.bind(session)
|
||||
session[key] = value
|
||||
|
||||
if not options.cookieJar?
|
||||
options.cookieJar = createCookieJar(new toughCookie.CookieJar())
|
||||
else if options.cookieJar == false
|
||||
delete options.cookieJar
|
||||
else
|
||||
# Assume we've gotten a cookie jar.
|
||||
options.cookieJar = createCookieJar(options.cookieJar)
|
||||
|
||||
session._sessionOptions = options
|
||||
|
||||
return session
|
||||
wrapStream: (stream, options) ->
|
||||
# This is a method for wrapping a stream in an object that also contains metadata.
|
||||
return {
|
||||
_bhttpStreamWrapper: true
|
||||
stream: stream
|
||||
options: options
|
||||
}
|
||||
|
||||
module.exports = bhttpAPI
|
||||
|
||||
# That's all, folks!
|
||||
@ -0,0 +1,655 @@
|
||||
var Promise, S, addErrorData, bhttpAPI, concatStream, createCookieJar, debug, doPayloadRequest, doRedirect, errors, formData, formFixArray, http, https, isStream, makeRequest, ofTypes, packageConfig, prepareCleanup, prepareDefaults, prepareOptions, preparePayload, prepareProtocol, prepareRequest, prepareSession, prepareUrl, processResponse, querystring, redirectGet, redirectUnchanged, stream, streamLength, toughCookie, urlUtil, util, _;
|
||||
|
||||
urlUtil = require("url");
|
||||
|
||||
querystring = require("querystring");
|
||||
|
||||
stream = require("stream");
|
||||
|
||||
http = require("http");
|
||||
|
||||
https = require("https");
|
||||
|
||||
util = require("util");
|
||||
|
||||
Promise = require("bluebird");
|
||||
|
||||
_ = require("lodash");
|
||||
|
||||
S = require("string");
|
||||
|
||||
formFixArray = require("form-fix-array");
|
||||
|
||||
errors = require("errors");
|
||||
|
||||
debug = require("debug")("bhttp");
|
||||
|
||||
formData = require("form-data2");
|
||||
|
||||
concatStream = require("concat-stream");
|
||||
|
||||
toughCookie = require("tough-cookie");
|
||||
|
||||
streamLength = require("stream-length");
|
||||
|
||||
packageConfig = require("../package.json");
|
||||
|
||||
errors.create({
|
||||
name: "bhttpError"
|
||||
});
|
||||
|
||||
errors.create({
|
||||
name: "ConflictingOptionsError",
|
||||
parents: errors.bhttpError
|
||||
});
|
||||
|
||||
errors.create({
|
||||
name: "UnsupportedProtocolError",
|
||||
parents: errors.bhttpError
|
||||
});
|
||||
|
||||
errors.create({
|
||||
name: "RedirectError",
|
||||
parents: errors.bhttpError
|
||||
});
|
||||
|
||||
errors.create({
|
||||
name: "MultipartError",
|
||||
parents: errors.bhttpError
|
||||
});
|
||||
|
||||
ofTypes = function(obj, types) {
|
||||
var match, type, _i, _len;
|
||||
match = false;
|
||||
for (_i = 0, _len = types.length; _i < _len; _i++) {
|
||||
type = types[_i];
|
||||
match = match || obj instanceof type;
|
||||
}
|
||||
return match;
|
||||
};
|
||||
|
||||
addErrorData = function(err, request, response, requestState) {
|
||||
err.request = request;
|
||||
err.response = response;
|
||||
err.requestState = requestState;
|
||||
return err;
|
||||
};
|
||||
|
||||
isStream = function(obj) {
|
||||
return (obj != null) && (ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]) || obj.hasOwnProperty("_bhttpStreamWrapper"));
|
||||
};
|
||||
|
||||
prepareSession = function(request, response, requestState) {
|
||||
debug("preparing session");
|
||||
return Promise["try"](function() {
|
||||
if (requestState.sessionOptions != null) {
|
||||
request.options = _.merge(_.clone(requestState.sessionOptions), request.options);
|
||||
}
|
||||
if (request.options.headers != null) {
|
||||
request.options.headers = _.clone(request.options.headers, true);
|
||||
} else {
|
||||
request.options.headers = {};
|
||||
}
|
||||
if (request.options.cookieJar != null) {
|
||||
return Promise["try"](function() {
|
||||
request.cookieJar = request.options.cookieJar;
|
||||
delete request.options.cookieJar;
|
||||
return request.cookieJar.get(request.url);
|
||||
}).then(function(cookieString) {
|
||||
debug("sending cookie string: %s", cookieString);
|
||||
request.options.headers["cookie"] = cookieString;
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
} else {
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
prepareDefaults = function(request, response, requestState) {
|
||||
debug("preparing defaults");
|
||||
return Promise["try"](function() {
|
||||
var _base, _base1, _base2, _ref, _ref1, _ref2, _ref3, _ref4, _ref5, _ref6, _ref7;
|
||||
request.responseOptions = {
|
||||
discardResponse: (_ref = request.options.discardResponse) != null ? _ref : false,
|
||||
keepRedirectResponses: (_ref1 = request.options.keepRedirectResponses) != null ? _ref1 : false,
|
||||
followRedirects: (_ref2 = request.options.followRedirects) != null ? _ref2 : true,
|
||||
noDecode: (_ref3 = request.options.noDecode) != null ? _ref3 : false,
|
||||
decodeJSON: (_ref4 = request.options.decodeJSON) != null ? _ref4 : false,
|
||||
stream: (_ref5 = request.options.stream) != null ? _ref5 : false,
|
||||
justPrepare: (_ref6 = request.options.justPrepare) != null ? _ref6 : false,
|
||||
redirectLimit: (_ref7 = request.options.redirectLimit) != null ? _ref7 : 10
|
||||
};
|
||||
if ((_base = request.options).allowChunkedMultipart == null) {
|
||||
_base.allowChunkedMultipart = false;
|
||||
}
|
||||
if ((_base1 = request.options).forceMultipart == null) {
|
||||
_base1.forceMultipart = false;
|
||||
}
|
||||
if ((_base2 = request.options.headers)["user-agent"] == null) {
|
||||
_base2["user-agent"] = "bhttp/" + packageConfig.version;
|
||||
}
|
||||
request.options.method = request.options.method.toLowerCase();
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
prepareUrl = function(request, response, requestState) {
|
||||
debug("preparing URL");
|
||||
return Promise["try"](function() {
|
||||
var urlOptions, _ref;
|
||||
urlOptions = urlUtil.parse(request.url, true);
|
||||
_.extend(request.options, {
|
||||
hostname: urlOptions.hostname,
|
||||
port: urlOptions.port
|
||||
});
|
||||
request.options.path = urlUtil.format({
|
||||
pathname: urlOptions.pathname,
|
||||
query: (_ref = request.options.query) != null ? _ref : urlOptions.query
|
||||
});
|
||||
request.protocol = S(urlOptions.protocol).chompRight(":").toString();
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
prepareProtocol = function(request, response, requestState) {
|
||||
debug("preparing protocol");
|
||||
return Promise["try"](function() {
|
||||
var _base;
|
||||
request.protocolModule = (function() {
|
||||
switch (request.protocol) {
|
||||
case "http":
|
||||
return http;
|
||||
case "https":
|
||||
return https;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
if (request.protocolModule == null) {
|
||||
return Promise.reject()(new errors.UnsupportedProtocolError("The protocol specified (" + protocol + ") is not currently supported by this module."));
|
||||
}
|
||||
if ((_base = request.options).port == null) {
|
||||
_base.port = (function() {
|
||||
switch (request.protocol) {
|
||||
case "http":
|
||||
return 80;
|
||||
case "https":
|
||||
return 443;
|
||||
}
|
||||
})();
|
||||
}
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
prepareOptions = function(request, response, requestState) {
|
||||
debug("preparing options");
|
||||
return Promise["try"](function() {
|
||||
var _base;
|
||||
if (((request.options.formFields != null) || (request.options.files != null)) && ((request.options.inputStream != null) || (request.options.inputBuffer != null))) {
|
||||
return Promise.reject(addErrorData(new errors.ConflictingOptionsError("You cannot define both formFields/files and a raw inputStream or inputBuffer."), request, response, requestState));
|
||||
}
|
||||
if (request.options.encodeJSON && ((request.options.inputStream != null) || (request.options.inputBuffer != null))) {
|
||||
return Promise.reject(addErrorData(new errors.ConflictingOptionsError("You cannot use both encodeJSON and a raw inputStream or inputBuffer.", void 0, "If you meant to JSON-encode the stream, you will currently have to do so manually."), request, response, requestState));
|
||||
}
|
||||
if (request.responseOptions.stream) {
|
||||
if ((_base = request.options).agent == null) {
|
||||
_base.agent = false;
|
||||
}
|
||||
}
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
preparePayload = function(request, response, requestState) {
|
||||
debug("preparing payload");
|
||||
return Promise["try"](function() {
|
||||
var containsStreams, fieldName, fieldValue, formDataObject, multipart, streamOptions, valueElement, _i, _len, _ref, _ref1, _ref2;
|
||||
multipart = request.options.forceMultipart || (request.options.files != null);
|
||||
multipart = multipart || _.any(request.options.formFields, function(item) {
|
||||
return item instanceof Buffer || isStream(item);
|
||||
});
|
||||
_.extend(request.options.formFields, request.options.files);
|
||||
containsStreams = _.any(request.options.formFields, function(item) {
|
||||
return isStream(item);
|
||||
});
|
||||
if (request.options.encodeJSON && containsStreams) {
|
||||
return Promise.reject()(new errors.ConflictingOptionsError("Sending a JSON-encoded payload containing data from a stream is not currently supported.", void 0, "Either don't use encodeJSON, or read your stream into a string or Buffer."));
|
||||
}
|
||||
if ((_ref = request.options.method) === "post" || _ref === "put" || _ref === "patch") {
|
||||
if ((request.options.encodeJSON || (request.options.formFields != null)) && !multipart) {
|
||||
debug("got url-encodable form-data");
|
||||
request.options.headers["content-type"] = "application/x-www-form-urlencoded";
|
||||
if (request.options.encodeJSON) {
|
||||
request.payload = JSON.stringify((_ref1 = request.options.formFields) != null ? _ref1 : null);
|
||||
} else if (!_.isEmpty(request.options.formFields)) {
|
||||
request.payload = querystring.stringify(formFixArray(request.options.formFields));
|
||||
} else {
|
||||
request.payload = "";
|
||||
}
|
||||
request.options.headers["content-length"] = request.payload.length;
|
||||
return Promise.resolve();
|
||||
} else if ((request.options.formFields != null) && multipart) {
|
||||
debug("got multipart form-data");
|
||||
formDataObject = new formData();
|
||||
_ref2 = formFixArray(request.options.formFields);
|
||||
for (fieldName in _ref2) {
|
||||
fieldValue = _ref2[fieldName];
|
||||
if (!_.isArray(fieldValue)) {
|
||||
fieldValue = [fieldValue];
|
||||
}
|
||||
for (_i = 0, _len = fieldValue.length; _i < _len; _i++) {
|
||||
valueElement = fieldValue[_i];
|
||||
if (valueElement._bhttpStreamWrapper != null) {
|
||||
streamOptions = valueElement.options;
|
||||
valueElement = valueElement.stream;
|
||||
} else {
|
||||
streamOptions = {};
|
||||
}
|
||||
formDataObject.append(fieldName, valueElement, streamOptions);
|
||||
}
|
||||
}
|
||||
request.payloadStream = formDataObject;
|
||||
return Promise["try"](function() {
|
||||
return formDataObject.getHeaders();
|
||||
}).then(function(headers) {
|
||||
if (headers["content-transfer-encoding"] === "chunked" && !request.options.allowChunkedMultipart) {
|
||||
return Promise.reject(addErrorData(new MultipartError("Most servers do not support chunked transfer encoding for multipart/form-data payloads, and we could not determine the length of all the input streams. See the documentation for more information."), request, response, requestState));
|
||||
} else {
|
||||
_.extend(request.options.headers, headers);
|
||||
return Promise.resolve();
|
||||
}
|
||||
});
|
||||
} else if (request.options.inputStream != null) {
|
||||
debug("got inputStream");
|
||||
return Promise["try"](function() {
|
||||
var _ref3;
|
||||
request.payloadStream = request.options.inputStream;
|
||||
if ((request.payloadStream._bhttpStreamWrapper != null) && ((request.payloadStream.options.contentLength != null) || (request.payloadStream.options.knownLength != null))) {
|
||||
return Promise.resolve((_ref3 = request.payloadStream.options.contentLength) != null ? _ref3 : request.payloadStream.options.knownLength);
|
||||
} else {
|
||||
return streamLength(request.options.inputStream);
|
||||
}
|
||||
}).then(function(length) {
|
||||
debug("length for inputStream is %s", length);
|
||||
return request.options.headers["content-length"] = length;
|
||||
})["catch"](function(err) {
|
||||
debug("unable to determine inputStream length, switching to chunked transfer encoding");
|
||||
return request.options.headers["content-transfer-encoding"] = "chunked";
|
||||
});
|
||||
} else if (request.options.inputBuffer != null) {
|
||||
debug("got inputBuffer");
|
||||
if (typeof request.options.inputBuffer === "string") {
|
||||
request.payload = new Buffer(request.options.inputBuffer);
|
||||
} else {
|
||||
request.payload = request.options.inputBuffer;
|
||||
}
|
||||
debug("length for inputBuffer is %s", request.payload.length);
|
||||
request.options.headers["content-length"] = request.payload.length;
|
||||
return Promise.resolve();
|
||||
}
|
||||
} else {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}).then(function() {
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
prepareCleanup = function(request, response, requestState) {
|
||||
debug("preparing cleanup");
|
||||
return Promise["try"](function() {
|
||||
var fixedHeaders, key, value, _i, _len, _ref, _ref1;
|
||||
_ref = ["query", "formFields", "files", "encodeJSON", "inputStream", "inputBuffer", "discardResponse", "keepRedirectResponses", "followRedirects", "noDecode", "decodeJSON", "allowChunkedMultipart", "forceMultipart"];
|
||||
for (_i = 0, _len = _ref.length; _i < _len; _i++) {
|
||||
key = _ref[_i];
|
||||
delete request.options[key];
|
||||
}
|
||||
fixedHeaders = {};
|
||||
_ref1 = request.options.headers;
|
||||
for (key in _ref1) {
|
||||
value = _ref1[key];
|
||||
fixedHeaders[key.toLowerCase()] = value;
|
||||
}
|
||||
request.options.headers = fixedHeaders;
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
prepareRequest = function(request, response, requestState) {
|
||||
debug("preparing request");
|
||||
return Promise["try"](function() {
|
||||
var middlewareFunctions, promiseChain;
|
||||
middlewareFunctions = [prepareSession, prepareDefaults, prepareUrl, prepareProtocol, prepareOptions, preparePayload, prepareCleanup];
|
||||
promiseChain = Promise.resolve([request, response, requestState]);
|
||||
middlewareFunctions.forEach(function(middleware) {
|
||||
return promiseChain = promiseChain.spread(function(_request, _response, _requestState) {
|
||||
return middleware(_request, _response, _requestState);
|
||||
});
|
||||
});
|
||||
return promiseChain;
|
||||
});
|
||||
};
|
||||
|
||||
makeRequest = function(request, response, requestState) {
|
||||
debug("making %s request to %s", request.options.method.toUpperCase(), request.url);
|
||||
return Promise["try"](function() {
|
||||
var req;
|
||||
req = request.protocolModule.request(request.options);
|
||||
if (request.payload != null) {
|
||||
debug("sending payload");
|
||||
req.write(request.payload);
|
||||
req.end();
|
||||
} else if (request.payloadStream != null) {
|
||||
debug("piping payloadStream");
|
||||
if (request.payloadStream._bhttpStreamWrapper != null) {
|
||||
request.payloadStream.stream.pipe(req);
|
||||
} else {
|
||||
request.payloadStream.pipe(req);
|
||||
}
|
||||
} else {
|
||||
debug("closing request without payload");
|
||||
req.end();
|
||||
}
|
||||
return new Promise(function(resolve, reject) {
|
||||
req.on("error", function(err) {
|
||||
return reject(err);
|
||||
});
|
||||
return req.on("response", function(res) {
|
||||
return resolve(res);
|
||||
});
|
||||
});
|
||||
}).then(function(response) {
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
processResponse = function(request, response, requestState) {
|
||||
debug("processing response, got status code %s", response.statusCode);
|
||||
return Promise["try"](function() {
|
||||
var cookieHeader, promises;
|
||||
if ((request.cookieJar != null) && (response.headers["set-cookie"] != null)) {
|
||||
promises = (function() {
|
||||
var _i, _len, _ref, _results;
|
||||
_ref = response.headers["set-cookie"];
|
||||
_results = [];
|
||||
for (_i = 0, _len = _ref.length; _i < _len; _i++) {
|
||||
cookieHeader = _ref[_i];
|
||||
debug("storing cookie: %s", cookieHeader);
|
||||
_results.push(request.cookieJar.set(cookieHeader, request.url));
|
||||
}
|
||||
return _results;
|
||||
})();
|
||||
return Promise.all(promises);
|
||||
} else {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}).then(function() {
|
||||
var _ref, _ref1;
|
||||
response.request = request;
|
||||
response.requestState = requestState;
|
||||
response.redirectHistory = requestState.redirectHistory;
|
||||
if (((_ref = response.statusCode) === 301 || _ref === 302 || _ref === 303 || _ref === 307) && request.responseOptions.followRedirects) {
|
||||
if (requestState.redirectHistory.length >= (request.responseOptions.redirectLimit - 1)) {
|
||||
return Promise.reject(addErrorData(new errors.RedirectError("The maximum amount of redirects ({request.responseOptions.redirectLimit}) was reached.")));
|
||||
}
|
||||
switch (response.statusCode) {
|
||||
case 301:
|
||||
switch (request.options.method) {
|
||||
case "get":
|
||||
case "head":
|
||||
return redirectUnchanged(request, response, requestState);
|
||||
case "post":
|
||||
case "put":
|
||||
case "patch":
|
||||
case "delete":
|
||||
return Promise.reject(addErrorData(new errors.RedirectError("Encountered a 301 redirect for POST, PUT, PATCH or DELETE. RFC says we can't automatically continue."), request, response, requestState));
|
||||
default:
|
||||
return Promise.reject(addErrorData(new errors.RedirectError("Encountered a 301 redirect, but not sure how to proceed for the " + (request.options.method.toUpperCase()) + " method.")));
|
||||
}
|
||||
break;
|
||||
case 302:
|
||||
case 303:
|
||||
return redirectGet(request, response, requestState);
|
||||
case 307:
|
||||
if (request.containsStreams && ((_ref1 = request.options.method) !== "get" && _ref1 !== "head")) {
|
||||
return Promise.reject(addErrorData(new errors.RedirectError("Encountered a 307 redirect for POST, PUT or DELETE, but your payload contained (single-use) streams. We therefore can't automatically follow the redirect."), request, response, requestState));
|
||||
} else {
|
||||
return redirectUnchanged(request, response, requestState);
|
||||
}
|
||||
}
|
||||
} else if (request.responseOptions.discardResponse) {
|
||||
response.resume();
|
||||
return Promise.resolve(response);
|
||||
} else {
|
||||
return new Promise(function(resolve, reject) {
|
||||
if (request.responseOptions.stream) {
|
||||
return resolve(response);
|
||||
} else {
|
||||
response.on("error", function(err) {
|
||||
return reject(err);
|
||||
});
|
||||
return response.pipe(concatStream(function(body) {
|
||||
var err, _ref2;
|
||||
if (request.responseOptions.decodeJSON || (((_ref2 = response.headers["content-type"]) != null ? _ref2 : "").split(";")[0] === "application/json" && !request.responseOptions.noDecode)) {
|
||||
try {
|
||||
response.body = JSON.parse(body);
|
||||
} catch (_error) {
|
||||
err = _error;
|
||||
reject(err);
|
||||
}
|
||||
} else {
|
||||
response.body = body;
|
||||
}
|
||||
return resolve(response);
|
||||
}));
|
||||
}
|
||||
});
|
||||
}
|
||||
}).then(function(response) {
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
});
|
||||
};
|
||||
|
||||
doPayloadRequest = function(url, data, options) {
|
||||
if (isStream(data)) {
|
||||
options.inputStream = data;
|
||||
} else if (ofTypes(data, [Buffer]) || typeof data === "string") {
|
||||
options.inputBuffer = data;
|
||||
} else {
|
||||
options.formFields = data;
|
||||
}
|
||||
return this.request(url, options);
|
||||
};
|
||||
|
||||
redirectGet = function(request, response, requestState) {
|
||||
debug("following forced-GET redirect to %s", response.headers["location"]);
|
||||
return Promise["try"](function() {
|
||||
var key, options, _i, _len, _ref;
|
||||
options = _.clone(requestState.originalOptions);
|
||||
options.method = "get";
|
||||
_ref = ["inputBuffer", "inputStream", "files", "formFields"];
|
||||
for (_i = 0, _len = _ref.length; _i < _len; _i++) {
|
||||
key = _ref[_i];
|
||||
delete options[key];
|
||||
}
|
||||
return doRedirect(request, response, requestState, options);
|
||||
});
|
||||
};
|
||||
|
||||
redirectUnchanged = function(request, response, requestState) {
|
||||
debug("following same-method redirect to %s", response.headers["location"]);
|
||||
return Promise["try"](function() {
|
||||
var options;
|
||||
options = _.clone(requestState.originalOptions);
|
||||
return doRedirect(request, response, requestState, options);
|
||||
});
|
||||
};
|
||||
|
||||
doRedirect = function(request, response, requestState, newOptions) {
|
||||
return Promise["try"](function() {
|
||||
if (!request.responseOptions.keepRedirectResponses) {
|
||||
response.resume();
|
||||
}
|
||||
requestState.redirectHistory.push(response);
|
||||
return bhttpAPI._doRequest(response.headers["location"], newOptions, requestState);
|
||||
});
|
||||
};
|
||||
|
||||
createCookieJar = function(jar) {
|
||||
return {
|
||||
set: function(cookie, url) {
|
||||
return new Promise((function(_this) {
|
||||
return function(resolve, reject) {
|
||||
return _this.jar.setCookie(cookie, url, function(err, cookie) {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
} else {
|
||||
return resolve(cookie);
|
||||
}
|
||||
});
|
||||
};
|
||||
})(this));
|
||||
},
|
||||
get: function(url) {
|
||||
return new Promise((function(_this) {
|
||||
return function(resolve, reject) {
|
||||
return _this.jar.getCookieString(url, function(err, cookies) {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
} else {
|
||||
return resolve(cookies);
|
||||
}
|
||||
});
|
||||
};
|
||||
})(this));
|
||||
},
|
||||
jar: jar
|
||||
};
|
||||
};
|
||||
|
||||
bhttpAPI = {
|
||||
head: function(url, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "head";
|
||||
return this.request(url, options);
|
||||
},
|
||||
get: function(url, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "get";
|
||||
return this.request(url, options);
|
||||
},
|
||||
post: function(url, data, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "post";
|
||||
return doPayloadRequest.bind(this)(url, data, options);
|
||||
},
|
||||
put: function(url, data, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "put";
|
||||
return doPayloadRequest.bind(this)(url, data, options);
|
||||
},
|
||||
patch: function(url, data, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "patch";
|
||||
return doPayloadRequest.bind(this)(url, data, options);
|
||||
},
|
||||
"delete": function(url, data, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options.method = "delete";
|
||||
return this.request(url, options);
|
||||
},
|
||||
request: function(url, options, callback) {
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
return this._doRequest(url, options).nodeify(callback);
|
||||
},
|
||||
_doRequest: function(url, options, requestState) {
|
||||
return Promise["try"]((function(_this) {
|
||||
return function() {
|
||||
var request, response, _ref;
|
||||
request = {
|
||||
url: url,
|
||||
options: _.clone(options)
|
||||
};
|
||||
response = null;
|
||||
if (requestState == null) {
|
||||
requestState = {
|
||||
originalOptions: _.clone(options),
|
||||
redirectHistory: []
|
||||
};
|
||||
}
|
||||
if (requestState.sessionOptions == null) {
|
||||
requestState.sessionOptions = (_ref = _this._sessionOptions) != null ? _ref : {};
|
||||
}
|
||||
return prepareRequest(request, response, requestState);
|
||||
};
|
||||
})(this)).spread((function(_this) {
|
||||
return function(request, response, requestState) {
|
||||
if (request.responseOptions.justPrepare) {
|
||||
return Promise.resolve([request, response, requestState]);
|
||||
} else {
|
||||
return Promise["try"](function() {
|
||||
return bhttpAPI.executeRequest(request, response, requestState);
|
||||
}).spread(function(request, response, requestState) {
|
||||
return Promise.resolve(response);
|
||||
});
|
||||
}
|
||||
};
|
||||
})(this));
|
||||
},
|
||||
executeRequest: function(request, response, requestState) {
|
||||
return Promise["try"](function() {
|
||||
return makeRequest(request, response, requestState);
|
||||
}).spread(function(request, response, requestState) {
|
||||
return processResponse(request, response, requestState);
|
||||
});
|
||||
},
|
||||
session: function(options) {
|
||||
var key, session, value;
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
options = _.clone(options);
|
||||
session = {};
|
||||
for (key in this) {
|
||||
value = this[key];
|
||||
if (value instanceof Function) {
|
||||
value = value.bind(session);
|
||||
}
|
||||
session[key] = value;
|
||||
}
|
||||
if (options.cookieJar == null) {
|
||||
options.cookieJar = createCookieJar(new toughCookie.CookieJar());
|
||||
} else if (options.cookieJar === false) {
|
||||
delete options.cookieJar;
|
||||
} else {
|
||||
options.cookieJar = createCookieJar(options.cookieJar);
|
||||
}
|
||||
session._sessionOptions = options;
|
||||
return session;
|
||||
},
|
||||
wrapStream: function(stream, options) {
|
||||
return {
|
||||
_bhttpStreamWrapper: true,
|
||||
stream: stream,
|
||||
options: options
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = bhttpAPI;
|
||||
@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "bhttp",
|
||||
"version": "1.0.0",
|
||||
"description": "A sane HTTP client library for Node.js with Streams2 support.",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git://github.com/joepie91/node-bhttp"
|
||||
},
|
||||
"keywords": [
|
||||
"http",
|
||||
"client",
|
||||
"multipart",
|
||||
"stream",
|
||||
"hyperquest",
|
||||
"request",
|
||||
"needle"
|
||||
],
|
||||
"author": "Sven Slootweg",
|
||||
"license": "WTFPL",
|
||||
"dependencies": {
|
||||
"bluebird": "^2.8.2",
|
||||
"concat-stream": "^1.4.7",
|
||||