Automatically migrated from Gitolite
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

546 lines
22 KiB

# FIXME: Force-lowercase user-supplied headers before merging them into the request?
# FIXME: Deep-merge query-string arguments between URL and argument?
# FIXME: Named arrays for multipart/form-data?
# FIXME: Are arrays of streams in `data` correctly recognized as being streams?
# Core modules
urlUtil = require "url"
querystring = require "querystring"
stream = require "stream"
http = require "http"
https = require "https"
util = require "util"
# Utility modules
Promise = require "bluebird"
_ = require "lodash"
S = require "string"
formFixArray = require "form-fix-array"
errors = require "errors"
debug = require("debug")("bhttp")
# Other third-party modules
formData = require "form-data2"
concatStream = require "concat-stream"
toughCookie = require "tough-cookie"
streamLength = require "stream-length"
# For the version in the user agent, etc.
packageConfig = require "../package.json"
# Error types
name: "bhttpError"
name: "ConflictingOptionsError"
parents: errors.bhttpError
name: "UnsupportedProtocolError"
parents: errors.bhttpError
name: "RedirectError"
parents: errors.bhttpError
name: "MultipartError"
parents: errors.bhttpError
# Utility functions
ofTypes = (obj, types) ->
match = false
for type in types
match = match or obj instanceof type
return match
addErrorData = (err, request, response, requestState) ->
err.request = request
err.response = response
err.requestState = requestState
return err
isStream = (obj) ->
obj? and (ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]) or obj.hasOwnProperty("_bhttpStreamWrapper"))
# Middleware
# NOTE: requestState is an object that signifies the current state of the overall request; eg. for a response involving one or more redirects, it will hold a 'redirect history'.
prepareSession = (request, response, requestState) ->
debug "preparing session"
Promise.try ->
if requestState.sessionOptions?
# Request options take priority over session options
request.options = _.merge _.clone(requestState.sessionOptions), request.options
# Create a headers parameter if it doesn't exist yet - we'll need to add some stuff to this later on
# FIXME: We may need to do a deep-clone of other mutable options later on as well; otherwise, when getting a redirect in a session with pre-defined options, the contents may not be correctly cleared after following the redirect.
if request.options.headers?
request.options.headers = _.clone(request.options.headers, true)
request.options.headers = {}
# If we have a cookie jar, start out by setting the cookie string.
if request.options.cookieJar?
Promise.try ->
# Move the cookieJar to the request object, the http/https module doesn't need it.
request.cookieJar = request.options.cookieJar
delete request.options.cookieJar
# Get the current cookie string for the URL
request.cookieJar.get request.url
.then (cookieString) ->
debug "sending cookie string: %s", cookieString
request.options.headers["cookie"] = cookieString
Promise.resolve [request, response, requestState]
Promise.resolve [request, response, requestState]
prepareDefaults = (request, response, requestState) ->
debug "preparing defaults"
Promise.try ->
# These are the options that we need for response processing, but don't need to be passed on to the http/https module.
request.responseOptions =
discardResponse: request.options.discardResponse ? false
keepRedirectResponses: request.options.keepRedirectResponses ? false
followRedirects: request.options.followRedirects ? true
noDecode: request.options.noDecode ? false
decodeJSON: request.options.decodeJSON ? false
stream: ? false
justPrepare: request.options.justPrepare ? false
redirectLimit: request.options.redirectLimit ? 10
# Whether chunked transfer encoding for multipart/form-data payloads is acceptable. This is likely to break quietly on a lot of servers.
request.options.allowChunkedMultipart ?= false
# Whether we should always use multipart/form-data for payloads, even if querystring-encoding would be a possibility.
request.options.forceMultipart ?= false
# If no custom user-agent is defined, set our own
request.options.headers["user-agent"] ?= "bhttp/#{packageConfig.version}"
# Normalize the request method to lowercase.
request.options.method = request.options.method.toLowerCase()
Promise.resolve [request, response, requestState]
prepareUrl = (request, response, requestState) ->
debug "preparing URL"
Promise.try ->
# Parse the specified URL, and use the resulting information to build a complete `options` object
urlOptions = urlUtil.parse request.url, true
_.extend request.options, {hostname: urlOptions.hostname, port: urlOptions.port}
request.options.path = urlUtil.format {pathname: urlOptions.pathname, query: request.options.query ? urlOptions.query}
request.protocol = S(urlOptions.protocol).chompRight(":").toString()
Promise.resolve [request, response, requestState]
prepareProtocol = (request, response, requestState) ->
debug "preparing protocol"
Promise.try ->
request.protocolModule = switch request.protocol
when "http" then http
when "https" then https # CAUTION / FIXME: Node will silently ignore SSL settings without a custom agent!
else null
if not request.protocolModule?
return Promise.reject() new errors.UnsupportedProtocolError "The protocol specified (#{protocol}) is not currently supported by this module."
request.options.port ?= switch request.protocol
when "http" then 80
when "https" then 443
Promise.resolve [request, response, requestState]
prepareOptions = (request, response, requestState) ->
debug "preparing options"
Promise.try ->
# Do some sanity checks - there are a number of options that cannot be used together
if (request.options.formFields? or request.options.files?) and (request.options.inputStream? or request.options.inputBuffer?)
return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot define both formFields/files and a raw inputStream or inputBuffer."), request, response, requestState)
if request.options.encodeJSON and (request.options.inputStream? or request.options.inputBuffer?)
return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot use both encodeJSON and a raw inputStream or inputBuffer.", undefined, "If you meant to JSON-encode the stream, you will currently have to do so manually."), request, response, requestState)
# If the user plans on streaming the response, we need to disable the agent entirely - otherwise the streams will block the pool.
request.options.agent ?= false
Promise.resolve [request, response, requestState]
preparePayload = (request, response, requestState) ->
debug "preparing payload"
Promise.try ->
# If a 'files' parameter is present, then we will send the form data as multipart data - it's most likely binary data.
multipart = request.options.forceMultipart or request.options.files?
# Similarly, if any of the formFields values are either a Stream or a Buffer, we will assume that the form should be sent as multipart.
multipart = multipart or _.any request.options.formFields, (item) ->
item instanceof Buffer or isStream(item)
# Really, 'files' and 'formFields' are the same thing - they mostly have different names for 1) clarity and 2) multipart detection. We combine them here.
_.extend request.options.formFields, request.options.files
# For a last sanity check, we want to know whether there are any Stream objects in our form data *at all* - these can't be used when encodeJSON is enabled.
containsStreams = _.any request.options.formFields, (item) -> isStream(item)
if request.options.encodeJSON and containsStreams
return Promise.reject() new errors.ConflictingOptionsError "Sending a JSON-encoded payload containing data from a stream is not currently supported.", undefined, "Either don't use encodeJSON, or read your stream into a string or Buffer."
if request.options.method in ["post", "put", "patch"]
# Prepare the payload, and set the appropriate headers.
if (request.options.encodeJSON or request.options.formFields?) and not multipart
# We know the payload and its size in advance.
debug "got url-encodable form-data"
request.options.headers["content-type"] = "application/x-www-form-urlencoded"
if request.options.encodeJSON
request.payload = JSON.stringify request.options.formFields ? null
else if not _.isEmpty request.options.formFields
# The `querystring` module copies the key name verbatim, even if the value is actually an array. Things like PHP don't understand this, and expect every array-containing key to be suffixed with []. We'll just append that ourselves, then.
request.payload = querystring.stringify formFixArray(request.options.formFields)
request.payload = ""
request.options.headers["content-length"] = request.payload.length
return Promise.resolve()
else if request.options.formFields? and multipart
# This is going to be multipart data, and we'll let `form-data` set the headers for us.
debug "got multipart form-data"
formDataObject = new formData()
for fieldName, fieldValue of formFixArray(request.options.formFields)
if not _.isArray fieldValue
fieldValue = [fieldValue]
for valueElement in fieldValue
if valueElement._bhttpStreamWrapper?
streamOptions = valueElement.options
valueElement =
streamOptions = {}
formDataObject.append fieldName, valueElement, streamOptions
request.payloadStream = formDataObject
Promise.try ->
.then (headers) ->
if headers["content-transfer-encoding"] == "chunked" and not request.options.allowChunkedMultipart
Promise.reject addErrorData(new MultipartError("Most servers do not support chunked transfer encoding for multipart/form-data payloads, and we could not determine the length of all the input streams. See the documentation for more information."), request, response, requestState)
_.extend request.options.headers, headers
else if request.options.inputStream?
# A raw inputStream was provided, just leave it be.
debug "got inputStream"
Promise.try ->
request.payloadStream = request.options.inputStream
if request.payloadStream._bhttpStreamWrapper? and (request.payloadStream.options.contentLength? or request.payloadStream.options.knownLength?)
Promise.resolve(request.payloadStream.options.contentLength ? request.payloadStream.options.knownLength)
streamLength request.options.inputStream
.then (length) ->
debug "length for inputStream is %s", length
request.options.headers["content-length"] = length
.catch (err) ->
debug "unable to determine inputStream length, switching to chunked transfer encoding"
request.options.headers["content-transfer-encoding"] = "chunked"
else if request.options.inputBuffer?
# A raw inputBuffer was provided, just leave it be (but make sure it's an actual Buffer).
debug "got inputBuffer"
if typeof request.options.inputBuffer == "string"
request.payload = new Buffer(request.options.inputBuffer) # Input string should be utf-8!
request.payload = request.options.inputBuffer
debug "length for inputBuffer is %s", request.payload.length
request.options.headers["content-length"] = request.payload.length
return Promise.resolve()
# GET, HEAD and DELETE should not have a payload. While technically not prohibited by the spec, it's also not specified, and we'd rather not upset poorly-compliant webservers.
# FIXME: Should this throw an Error?
return Promise.resolve()
.then ->
Promise.resolve [request, response, requestState]
prepareCleanup = (request, response, requestState) ->
debug "preparing cleanup"
Promise.try ->
# Remove the options that we're not going to pass on to the actual http/https library.
delete request.options[key] for key in ["query", "formFields", "files", "encodeJSON", "inputStream", "inputBuffer", "discardResponse", "keepRedirectResponses", "followRedirects", "noDecode", "decodeJSON", "allowChunkedMultipart", "forceMultipart"]
# Lo-Dash apparently has no `map` equivalent for object keys...?
fixedHeaders = {}
for key, value of request.options.headers
fixedHeaders[key.toLowerCase()] = value
request.options.headers = fixedHeaders
Promise.resolve [request, response, requestState]
# The guts of the module
prepareRequest = (request, response, requestState) ->
debug "preparing request"
# FIXME: Mock httpd for testing functionality.
Promise.try ->
middlewareFunctions = [
promiseChain = Promise.resolve [request, response, requestState]
middlewareFunctions.forEach (middleware) -> # We must use the functional construct here, to avoid losing references
promiseChain = promiseChain.spread (_request, _response, _requestState) ->
middleware(_request, _response, _requestState)
return promiseChain
makeRequest = (request, response, requestState) ->
debug "making %s request to %s", request.options.method.toUpperCase(), request.url
Promise.try ->
# Instantiate a regular HTTP/HTTPS request
req = request.protocolModule.request request.options
# This is where we write our payload or stream to the request, and the actual request is made.
if request.payload?
# The entire payload is a single Buffer.
debug "sending payload"
req.write request.payload
else if request.payloadStream?
# The payload is a stream.
debug "piping payloadStream"
if request.payloadStream._bhttpStreamWrapper? req
request.payloadStream.pipe req
# For GET, HEAD, DELETE, etc. there is no payload, but we still need to call end() to complete the request.
debug "closing request without payload"
new Promise (resolve, reject) ->
# In case something goes wrong during this process, somehow...
req.on "error", (err) ->
reject err
req.on "response", (res) ->
resolve res
.then (response) ->
Promise.resolve [request, response, requestState]
processResponse = (request, response, requestState) ->
debug "processing response, got status code %s", response.statusCode
# When we receive the response, we'll buffer it up and/or decode it, depending on what the user specified, and resolve the returned Promise. If the user just wants the raw stream, we resolve immediately after receiving a response.
Promise.try ->
# First, if a cookie jar is set and we received one or more cookies from the server, we should store them in our cookieJar.
if request.cookieJar? and response.headers["set-cookie"]?
promises = for cookieHeader in response.headers["set-cookie"]
debug "storing cookie: %s", cookieHeader
request.cookieJar.set cookieHeader, request.url
Promise.all promises
.then ->
# Now the actual response processing.
response.request = request
response.requestState = requestState
response.redirectHistory = requestState.redirectHistory
if response.statusCode in [301, 302, 303, 307] and request.responseOptions.followRedirects
if requestState.redirectHistory.length >= (request.responseOptions.redirectLimit - 1)
return Promise.reject addErrorData(new errors.RedirectError("The maximum amount of redirects ({request.responseOptions.redirectLimit}) was reached."))
# 301: For GET and HEAD, redirect unchanged. For POST, PUT, PATCH, DELETE, "ask user" (in our case: throw an error.)
# 302: Redirect, change method to GET.
# 303: Redirect, change method to GET.
# 307: Redirect, retain method. Make same request again.
switch response.statusCode
when 301
switch request.options.method
when "get", "head"
return redirectUnchanged request, response, requestState
when "post", "put", "patch", "delete"
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect for POST, PUT, PATCH or DELETE. RFC says we can't automatically continue."), request, response, requestState)
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect, but not sure how to proceed for the #{request.options.method.toUpperCase()} method."))
when 302, 303
return redirectGet request, response, requestState
when 307
if request.containsStreams and request.options.method not in ["get", "head"]
return Promise.reject addErrorData(new errors.RedirectError("Encountered a 307 redirect for POST, PUT or DELETE, but your payload contained (single-use) streams. We therefore can't automatically follow the redirect."), request, response, requestState)
return redirectUnchanged request, response, requestState
else if request.responseOptions.discardResponse
response.resume() # Drain the response stream
Promise.resolve response
new Promise (resolve, reject) ->
resolve response
response.on "error", (err) ->
reject err
response.pipe concatStream (body) ->
# FIXME: Separate module for header parsing?
if request.responseOptions.decodeJSON or ((response.headers["content-type"] ? "").split(";")[0] == "application/json" and not request.responseOptions.noDecode)
response.body = JSON.parse body
catch err
reject err
response.body = body
resolve response
.then (response) ->
Promise.resolve [request, response, requestState]
# Some wrappers
doPayloadRequest = (url, data, options, callback) ->
# A wrapper that processes the second argument to .post, .put, .patch shorthand API methods.
# FIXME: Treat a {} for data as a null? Otherwise {} combined with inputBuffer/inputStream will error.
if isStream(data)
options.inputStream = data
else if ofTypes(data, [Buffer]) or typeof data == "string"
options.inputBuffer = data
options.formFields = data
@request url, options, callback
redirectGet = (request, response, requestState) ->
debug "following forced-GET redirect to %s", response.headers["location"]
Promise.try ->
options = _.clone(requestState.originalOptions)
options.method = "get"
delete options[key] for key in ["inputBuffer", "inputStream", "files", "formFields"]
doRedirect request, response, requestState, options
redirectUnchanged = (request, response, requestState) ->
debug "following same-method redirect to %s", response.headers["location"]
Promise.try ->
options = _.clone(requestState.originalOptions)
doRedirect request, response, requestState, options
doRedirect = (request, response, requestState, newOptions) ->
Promise.try ->
if not request.responseOptions.keepRedirectResponses
response.resume() # Let the response stream drain out...
requestState.redirectHistory.push response
bhttpAPI._doRequest response.headers["location"], newOptions, requestState
createCookieJar = (jar) ->
# Creates a cookie jar wrapper with a simplified API.
return {
set: (cookie, url) ->
new Promise (resolve, reject) =>
@jar.setCookie cookie, url, (err, cookie) ->
if err then reject(err) else resolve(cookie)
get: (url) ->
new Promise (resolve, reject) =>
@jar.getCookieString url, (err, cookies) ->
if err then reject(err) else resolve(cookies)
jar: jar
# The exposed API
bhttpAPI =
head: (url, options = {}, callback) ->
options.method = "head"
@request url, options, callback
get: (url, options = {}, callback) ->
options.method = "get"
@request url, options, callback
post: (url, data, options = {}, callback) ->
options.method = "post"
doPayloadRequest.bind(this) url, data, options, callback
put: (url, data, options = {}, callback) ->
options.method = "put"
doPayloadRequest.bind(this) url, data, options, callback
patch: (url, data, options = {}, callback) ->
options.method = "patch"
doPayloadRequest.bind(this) url, data, options, callback
delete: (url, data, options = {}, callback) ->
options.method = "delete"
@request url, options, callback
request: (url, options = {}, callback) ->
@_doRequest(url, options).nodeify(callback)
_doRequest: (url, options, requestState) ->
# This is split from the `request` method, so that the user doesn't have to pass in `undefined` for the `requestState` when they want to specify a `callback`.
Promise.try =>
request = {url: url, options: _.clone(options)}
response = null
requestState ?= {originalOptions: _.clone(options), redirectHistory: []}
requestState.sessionOptions ?= @_sessionOptions ? {}
prepareRequest request, response, requestState
.spread (request, response, requestState) =>
if request.responseOptions.justPrepare
Promise.resolve [request, response, requestState]
Promise.try ->
bhttpAPI.executeRequest request, response, requestState
.spread (request, response, requestState) ->
# The user likely only wants the response.
Promise.resolve response
executeRequest: (request, response, requestState) ->
# Executes a pre-configured request.
Promise.try ->
makeRequest request, response, requestState
.spread (request, response, requestState) ->
processResponse request, response, requestState
session: (options) ->
options ?= {}
options = _.clone options
session = {}
for key, value of this
if value instanceof Function
value = value.bind(session)
session[key] = value
if not options.cookieJar?
options.cookieJar = createCookieJar(new toughCookie.CookieJar())
else if options.cookieJar == false
delete options.cookieJar
# Assume we've gotten a cookie jar.
options.cookieJar = createCookieJar(options.cookieJar)
session._sessionOptions = options
return session
wrapStream: (stream, options) ->
# This is a method for wrapping a stream in an object that also contains metadata.
return {
_bhttpStreamWrapper: true
stream: stream
options: options
module.exports = bhttpAPI
# That's all, folks!