# FIXME: Force-lowercase user-supplied headers before merging them into the request? # FIXME: Deep-merge query-string arguments between URL and argument? # FIXME: Named arrays for multipart/form-data? # Core modules urlUtil = require "url" querystring = require "querystring" stream = require "stream" http = require "http" https = require "https" util = require "util" # Utility modules Promise = require "bluebird" _ = require "lodash" S = require "string" formFixArray = require "form-fix-array" errors = require "errors" debug = require("debug")("bhttp") # Other third-party modules formData = require "form-data2" concatStream = require "concat-stream" toughCookie = require "tough-cookie" streamLength = require "stream-length" # For the version in the user agent, etc. packageConfig = require "../package.json" # Error types errors.create name: "bhttpError" errors.create name: "ConflictingOptionsError" parents: errors.bhttpError errors.create name: "UnsupportedProtocolError" parents: errors.bhttpError errors.create name: "RedirectError" parents: errors.bhttpError errors.create name: "MultipartError" parents: errors.bhttpError # Utility functions ofTypes = (obj, types) -> match = false for type in types match = match or obj instanceof type return match addErrorData = (err, request, response, requestState) -> err.request = request err.response = response err.requestState = requestState return err isStream = (obj) -> obj? and (ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]) or obj.hasOwnProperty("_bhttpStreamWrapper")) # Middleware # NOTE: requestState is an object that signifies the current state of the overall request; eg. for a response involving one or more redirects, it will hold a 'redirect history'. prepareSession = (request, response, requestState) -> debug "preparing session" Promise.try -> if requestState.sessionOptions? # Request options take priority over session options request.options = _.merge _.clone(requestState.sessionOptions), request.options # Create a headers parameter if it doesn't exist yet - we'll need to add some stuff to this later on # FIXME: We may need to do a deep-clone of other mutable options later on as well; otherwise, when getting a redirect in a session with pre-defined options, the contents may not be correctly cleared after following the redirect. if request.options.headers? request.options.headers = _.clone(request.options.headers, true) else request.options.headers = {} # If we have a cookie jar, start out by setting the cookie string. if request.options.cookieJar? Promise.try -> # Move the cookieJar to the request object, the http/https module doesn't need it. request.cookieJar = request.options.cookieJar delete request.options.cookieJar # Get the current cookie string for the URL request.cookieJar.get request.url .then (cookieString) -> debug "sending cookie string: %s", cookieString request.options.headers["cookie"] = cookieString Promise.resolve [request, response, requestState] else Promise.resolve [request, response, requestState] prepareDefaults = (request, response, requestState) -> debug "preparing defaults" Promise.try -> # These are the options that we need for response processing, but don't need to be passed on to the http/https module. request.responseOptions = discardResponse: request.options.discardResponse ? false keepRedirectResponses: request.options.keepRedirectResponses ? false followRedirects: request.options.followRedirects ? true noDecode: request.options.noDecode ? false decodeJSON: request.options.decodeJSON ? false stream: request.options.stream ? false justPrepare: request.options.justPrepare ? false redirectLimit: request.options.redirectLimit ? 10 # Whether chunked transfer encoding for multipart/form-data payloads is acceptable. This is likely to break quietly on a lot of servers. request.options.allowChunkedMultipart ?= false # Whether we should always use multipart/form-data for payloads, even if querystring-encoding would be a possibility. request.options.forceMultipart ?= false # If no custom user-agent is defined, set our own request.options.headers["user-agent"] ?= "bhttp/#{packageConfig.version}" # Normalize the request method to lowercase. request.options.method = request.options.method.toLowerCase() Promise.resolve [request, response, requestState] prepareUrl = (request, response, requestState) -> debug "preparing URL" Promise.try -> # Parse the specified URL, and use the resulting information to build a complete `options` object urlOptions = urlUtil.parse request.url, true _.extend request.options, {hostname: urlOptions.hostname, port: urlOptions.port} request.options.path = urlUtil.format {pathname: urlOptions.pathname, query: request.options.query ? urlOptions.query} request.protocol = S(urlOptions.protocol).chompRight(":").toString() Promise.resolve [request, response, requestState] prepareProtocol = (request, response, requestState) -> debug "preparing protocol" Promise.try -> request.protocolModule = switch request.protocol when "http" then http when "https" then https # CAUTION / FIXME: Node will silently ignore SSL settings without a custom agent! else null if not request.protocolModule? return Promise.reject() new errors.UnsupportedProtocolError "The protocol specified (#{protocol}) is not currently supported by this module." request.options.port ?= switch request.protocol when "http" then 80 when "https" then 443 Promise.resolve [request, response, requestState] prepareOptions = (request, response, requestState) -> debug "preparing options" Promise.try -> # Do some sanity checks - there are a number of options that cannot be used together if (request.options.formFields? or request.options.files?) and (request.options.inputStream? or request.options.inputBuffer?) return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot define both formFields/files and a raw inputStream or inputBuffer."), request, response, requestState) if request.options.encodeJSON and (request.options.inputStream? or request.options.inputBuffer?) return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot use both encodeJSON and a raw inputStream or inputBuffer.", undefined, "If you meant to JSON-encode the stream, you will currently have to do so manually."), request, response, requestState) # If the user plans on streaming the response, we need to disable the agent entirely - otherwise the streams will block the pool. if request.responseOptions.stream request.options.agent ?= false Promise.resolve [request, response, requestState] preparePayload = (request, response, requestState) -> debug "preparing payload" Promise.try -> # If a 'files' parameter is present, then we will send the form data as multipart data - it's most likely binary data. multipart = request.options.forceMultipart or request.options.files? # Similarly, if any of the formFields values are either a Stream or a Buffer, we will assume that the form should be sent as multipart. multipart = multipart or _.any request.options.formFields, (item) -> item instanceof Buffer or isStream(item) # Really, 'files' and 'formFields' are the same thing - they mostly have different names for 1) clarity and 2) multipart detection. We combine them here. _.extend request.options.formFields, request.options.files # For a last sanity check, we want to know whether there are any Stream objects in our form data *at all* - these can't be used when encodeJSON is enabled. containsStreams = _.any request.options.formFields, (item) -> isStream(item) if request.options.encodeJSON and containsStreams return Promise.reject() new errors.ConflictingOptionsError "Sending a JSON-encoded payload containing data from a stream is not currently supported.", undefined, "Either don't use encodeJSON, or read your stream into a string or Buffer." if request.options.method in ["post", "put", "patch"] # Prepare the payload, and set the appropriate headers. if (request.options.encodeJSON or request.options.formFields?) and not multipart # We know the payload and its size in advance. debug "got url-encodable form-data" request.options.headers["content-type"] = "application/x-www-form-urlencoded" if request.options.encodeJSON request.payload = JSON.stringify request.options.formFields ? null else if not _.isEmpty request.options.formFields # The `querystring` module copies the key name verbatim, even if the value is actually an array. Things like PHP don't understand this, and expect every array-containing key to be suffixed with []. We'll just append that ourselves, then. request.payload = querystring.stringify formFixArray(request.options.formFields) else request.payload = "" request.options.headers["content-length"] = request.payload.length return Promise.resolve() else if request.options.formFields? and multipart # This is going to be multipart data, and we'll let `form-data` set the headers for us. debug "got multipart form-data" formDataObject = new formData() for fieldName, fieldValue of formFixArray(request.options.formFields) if not _.isArray fieldValue fieldValue = [fieldValue] for valueElement in fieldValue if valueElement._bhttpStreamWrapper? streamOptions = valueElement.options valueElement = valueElement.stream else streamOptions = {} formDataObject.append fieldName, valueElement, streamOptions request.payloadStream = formDataObject Promise.try -> formDataObject.getHeaders() .then (headers) -> if headers["content-transfer-encoding"] == "chunked" and not request.options.allowChunkedMultipart Promise.reject addErrorData(new MultipartError("Most servers do not support chunked transfer encoding for multipart/form-data payloads, and we could not determine the length of all the input streams. See the documentation for more information."), request, response, requestState) else _.extend request.options.headers, headers Promise.resolve() else if request.options.inputStream? # A raw inputStream was provided, just leave it be. debug "got inputStream" Promise.try -> request.payloadStream = request.options.inputStream if request.payloadStream._bhttpStreamWrapper? and (request.payloadStream.options.contentLength? or request.payloadStream.options.knownLength?) Promise.resolve(request.payloadStream.options.contentLength ? request.payloadStream.options.knownLength) else streamLength request.options.inputStream .then (length) -> debug "length for inputStream is %s", length request.options.headers["content-length"] = length .catch (err) -> debug "unable to determine inputStream length, switching to chunked transfer encoding" request.options.headers["content-transfer-encoding"] = "chunked" else if request.options.inputBuffer? # A raw inputBuffer was provided, just leave it be (but make sure it's an actual Buffer). debug "got inputBuffer" if typeof request.options.inputBuffer == "string" request.payload = new Buffer(request.options.inputBuffer) # Input string should be utf-8! else request.payload = request.options.inputBuffer debug "length for inputBuffer is %s", request.payload.length request.options.headers["content-length"] = request.payload.length return Promise.resolve() else # GET, HEAD and DELETE should not have a payload. While technically not prohibited by the spec, it's also not specified, and we'd rather not upset poorly-compliant webservers. # FIXME: Should this throw an Error? return Promise.resolve() .then -> Promise.resolve [request, response, requestState] prepareCleanup = (request, response, requestState) -> debug "preparing cleanup" Promise.try -> # Remove the options that we're not going to pass on to the actual http/https library. delete request.options[key] for key in ["query", "formFields", "files", "encodeJSON", "inputStream", "inputBuffer", "discardResponse", "keepRedirectResponses", "followRedirects", "noDecode", "decodeJSON", "allowChunkedMultipart", "forceMultipart"] # Lo-Dash apparently has no `map` equivalent for object keys...? fixedHeaders = {} for key, value of request.options.headers fixedHeaders[key.toLowerCase()] = value request.options.headers = fixedHeaders Promise.resolve [request, response, requestState] # The guts of the module prepareRequest = (request, response, requestState) -> debug "preparing request" # FIXME: Mock httpd for testing functionality. Promise.try -> middlewareFunctions = [ prepareSession prepareDefaults prepareUrl prepareProtocol prepareOptions preparePayload prepareCleanup ] promiseChain = Promise.resolve [request, response, requestState] middlewareFunctions.forEach (middleware) -> # We must use the functional construct here, to avoid losing references promiseChain = promiseChain.spread (_request, _response, _requestState) -> middleware(_request, _response, _requestState) return promiseChain makeRequest = (request, response, requestState) -> debug "making %s request to %s", request.options.method.toUpperCase(), request.url Promise.try -> # Instantiate a regular HTTP/HTTPS request req = request.protocolModule.request request.options # This is where we write our payload or stream to the request, and the actual request is made. if request.payload? # The entire payload is a single Buffer. debug "sending payload" req.write request.payload req.end() else if request.payloadStream? # The payload is a stream. debug "piping payloadStream" if request.payloadStream._bhttpStreamWrapper? request.payloadStream.stream.pipe req else request.payloadStream.pipe req else # For GET, HEAD, DELETE, etc. there is no payload, but we still need to call end() to complete the request. debug "closing request without payload" req.end() new Promise (resolve, reject) -> # In case something goes wrong during this process, somehow... req.on "error", (err) -> reject err req.on "response", (res) -> resolve res .then (response) -> Promise.resolve [request, response, requestState] processResponse = (request, response, requestState) -> debug "processing response, got status code %s", response.statusCode # When we receive the response, we'll buffer it up and/or decode it, depending on what the user specified, and resolve the returned Promise. If the user just wants the raw stream, we resolve immediately after receiving a response. Promise.try -> # First, if a cookie jar is set and we received one or more cookies from the server, we should store them in our cookieJar. if request.cookieJar? and response.headers["set-cookie"]? promises = for cookieHeader in response.headers["set-cookie"] debug "storing cookie: %s", cookieHeader request.cookieJar.set cookieHeader, request.url Promise.all promises else Promise.resolve() .then -> # Now the actual response processing. response.request = request response.requestState = requestState response.redirectHistory = requestState.redirectHistory if response.statusCode in [301, 302, 303, 307] and request.responseOptions.followRedirects if requestState.redirectHistory.length >= (request.responseOptions.redirectLimit - 1) return Promise.reject addErrorData(new errors.RedirectError("The maximum amount of redirects ({request.responseOptions.redirectLimit}) was reached.")) # 301: For GET and HEAD, redirect unchanged. For POST, PUT, PATCH, DELETE, "ask user" (in our case: throw an error.) # 302: Redirect, change method to GET. # 303: Redirect, change method to GET. # 307: Redirect, retain method. Make same request again. switch response.statusCode when 301 switch request.options.method when "get", "head" return redirectUnchanged request, response, requestState when "post", "put", "patch", "delete" return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect for POST, PUT, PATCH or DELETE. RFC says we can't automatically continue."), request, response, requestState) else return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect, but not sure how to proceed for the #{request.options.method.toUpperCase()} method.")) when 302, 303 return redirectGet request, response, requestState when 307 if request.containsStreams and request.options.method not in ["get", "head"] return Promise.reject addErrorData(new errors.RedirectError("Encountered a 307 redirect for POST, PUT or DELETE, but your payload contained (single-use) streams. We therefore can't automatically follow the redirect."), request, response, requestState) else return redirectUnchanged request, response, requestState else if request.responseOptions.discardResponse response.resume() # Drain the response stream Promise.resolve response else new Promise (resolve, reject) -> if request.responseOptions.stream resolve response else response.on "error", (err) -> reject err response.pipe concatStream (body) -> # FIXME: Separate module for header parsing? if request.responseOptions.decodeJSON or ((response.headers["content-type"] ? "").split(";")[0] == "application/json" and not request.responseOptions.noDecode) try response.body = JSON.parse body catch err reject err else response.body = body resolve response .then (response) -> Promise.resolve [request, response, requestState] # Some wrappers doPayloadRequest = (url, data, options, callback) -> # A wrapper that processes the second argument to .post, .put, .patch shorthand API methods. # FIXME: Treat a {} for data as a null? Otherwise {} combined with inputBuffer/inputStream will error. if isStream(data) options.inputStream = data else if ofTypes(data, [Buffer]) or typeof data == "string" options.inputBuffer = data else options.formFields = data @request url, options, callback redirectGet = (request, response, requestState) -> debug "following forced-GET redirect to %s", response.headers["location"] Promise.try -> options = _.clone(requestState.originalOptions) options.method = "get" delete options[key] for key in ["inputBuffer", "inputStream", "files", "formFields"] doRedirect request, response, requestState, options redirectUnchanged = (request, response, requestState) -> debug "following same-method redirect to %s", response.headers["location"] Promise.try -> options = _.clone(requestState.originalOptions) doRedirect request, response, requestState, options doRedirect = (request, response, requestState, newOptions) -> Promise.try -> if not request.responseOptions.keepRedirectResponses response.resume() # Let the response stream drain out... requestState.redirectHistory.push response bhttpAPI._doRequest response.headers["location"], newOptions, requestState createCookieJar = (jar) -> # Creates a cookie jar wrapper with a simplified API. return { set: (cookie, url) -> new Promise (resolve, reject) => @jar.setCookie cookie, url, (err, cookie) -> if err then reject(err) else resolve(cookie) get: (url) -> new Promise (resolve, reject) => @jar.getCookieString url, (err, cookies) -> if err then reject(err) else resolve(cookies) jar: jar } # The exposed API bhttpAPI = head: (url, options = {}, callback) -> options.method = "head" @request url, options, callback get: (url, options = {}, callback) -> options.method = "get" @request url, options, callback post: (url, data, options = {}, callback) -> options.method = "post" doPayloadRequest.bind(this) url, data, options, callback put: (url, data, options = {}, callback) -> options.method = "put" doPayloadRequest.bind(this) url, data, options, callback patch: (url, data, options = {}, callback) -> options.method = "patch" doPayloadRequest.bind(this) url, data, options, callback delete: (url, data, options = {}, callback) -> options.method = "delete" @request url, options, callback request: (url, options = {}, callback) -> @_doRequest(url, options).nodeify(callback) _doRequest: (url, options, requestState) -> # This is split from the `request` method, so that the user doesn't have to pass in `undefined` for the `requestState` when they want to specify a `callback`. Promise.try => request = {url: url, options: _.clone(options)} response = null requestState ?= {originalOptions: _.clone(options), redirectHistory: []} requestState.sessionOptions ?= @_sessionOptions ? {} prepareRequest request, response, requestState .spread (request, response, requestState) => if request.responseOptions.justPrepare Promise.resolve [request, response, requestState] else Promise.try -> bhttpAPI.executeRequest request, response, requestState .spread (request, response, requestState) -> # The user likely only wants the response. Promise.resolve response executeRequest: (request, response, requestState) -> # Executes a pre-configured request. Promise.try -> makeRequest request, response, requestState .spread (request, response, requestState) -> processResponse request, response, requestState session: (options) -> options ?= {} options = _.clone options session = {} for key, value of this if value instanceof Function value = value.bind(session) session[key] = value if not options.cookieJar? options.cookieJar = createCookieJar(new toughCookie.CookieJar()) else if options.cookieJar == false delete options.cookieJar else # Assume we've gotten a cookie jar. options.cookieJar = createCookieJar(options.cookieJar) session._sessionOptions = options return session wrapStream: (stream, options) -> # This is a method for wrapping a stream in an object that also contains metadata. return { _bhttpStreamWrapper: true stream: stream options: options } module.exports = bhttpAPI # That's all, folks!