Automatically migrated from Gitolite
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

546 lines
22 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
  1. # FIXME: Force-lowercase user-supplied headers before merging them into the request?
  2. # FIXME: Deep-merge query-string arguments between URL and argument?
  3. # FIXME: Named arrays for multipart/form-data?
  4. # FIXME: Are arrays of streams in `data` correctly recognized as being streams?
  5. # Core modules
  6. urlUtil = require "url"
  7. querystring = require "querystring"
  8. stream = require "stream"
  9. http = require "http"
  10. https = require "https"
  11. util = require "util"
  12. # Utility modules
  13. Promise = require "bluebird"
  14. _ = require "lodash"
  15. S = require "string"
  16. formFixArray = require "form-fix-array"
  17. errors = require "errors"
  18. debug = require("debug")("bhttp")
  19. # Other third-party modules
  20. formData = require "form-data2"
  21. concatStream = require "concat-stream"
  22. toughCookie = require "tough-cookie"
  23. streamLength = require "stream-length"
  24. # For the version in the user agent, etc.
  25. packageConfig = require "../package.json"
  26. # Error types
  27. errors.create
  28. name: "bhttpError"
  29. errors.create
  30. name: "ConflictingOptionsError"
  31. parents: errors.bhttpError
  32. errors.create
  33. name: "UnsupportedProtocolError"
  34. parents: errors.bhttpError
  35. errors.create
  36. name: "RedirectError"
  37. parents: errors.bhttpError
  38. errors.create
  39. name: "MultipartError"
  40. parents: errors.bhttpError
  41. # Utility functions
  42. ofTypes = (obj, types) ->
  43. match = false
  44. for type in types
  45. match = match or obj instanceof type
  46. return match
  47. addErrorData = (err, request, response, requestState) ->
  48. err.request = request
  49. err.response = response
  50. err.requestState = requestState
  51. return err
  52. isStream = (obj) ->
  53. obj? and (ofTypes(obj, [stream.Readable, stream.Duplex, stream.Transform]) or obj.hasOwnProperty("_bhttpStreamWrapper"))
  54. # Middleware
  55. # NOTE: requestState is an object that signifies the current state of the overall request; eg. for a response involving one or more redirects, it will hold a 'redirect history'.
  56. prepareSession = (request, response, requestState) ->
  57. debug "preparing session"
  58. Promise.try ->
  59. if requestState.sessionOptions?
  60. # Request options take priority over session options
  61. request.options = _.merge _.clone(requestState.sessionOptions), request.options
  62. # Create a headers parameter if it doesn't exist yet - we'll need to add some stuff to this later on
  63. # FIXME: We may need to do a deep-clone of other mutable options later on as well; otherwise, when getting a redirect in a session with pre-defined options, the contents may not be correctly cleared after following the redirect.
  64. if request.options.headers?
  65. request.options.headers = _.clone(request.options.headers, true)
  66. else
  67. request.options.headers = {}
  68. # If we have a cookie jar, start out by setting the cookie string.
  69. if request.options.cookieJar?
  70. Promise.try ->
  71. # Move the cookieJar to the request object, the http/https module doesn't need it.
  72. request.cookieJar = request.options.cookieJar
  73. delete request.options.cookieJar
  74. # Get the current cookie string for the URL
  75. request.cookieJar.get request.url
  76. .then (cookieString) ->
  77. debug "sending cookie string: %s", cookieString
  78. request.options.headers["cookie"] = cookieString
  79. Promise.resolve [request, response, requestState]
  80. else
  81. Promise.resolve [request, response, requestState]
  82. prepareDefaults = (request, response, requestState) ->
  83. debug "preparing defaults"
  84. Promise.try ->
  85. # These are the options that we need for response processing, but don't need to be passed on to the http/https module.
  86. request.responseOptions =
  87. discardResponse: request.options.discardResponse ? false
  88. keepRedirectResponses: request.options.keepRedirectResponses ? false
  89. followRedirects: request.options.followRedirects ? true
  90. noDecode: request.options.noDecode ? false
  91. decodeJSON: request.options.decodeJSON ? false
  92. stream: request.options.stream ? false
  93. justPrepare: request.options.justPrepare ? false
  94. redirectLimit: request.options.redirectLimit ? 10
  95. # Whether chunked transfer encoding for multipart/form-data payloads is acceptable. This is likely to break quietly on a lot of servers.
  96. request.options.allowChunkedMultipart ?= false
  97. # Whether we should always use multipart/form-data for payloads, even if querystring-encoding would be a possibility.
  98. request.options.forceMultipart ?= false
  99. # If no custom user-agent is defined, set our own
  100. request.options.headers["user-agent"] ?= "bhttp/#{packageConfig.version}"
  101. # Normalize the request method to lowercase.
  102. request.options.method = request.options.method.toLowerCase()
  103. Promise.resolve [request, response, requestState]
  104. prepareUrl = (request, response, requestState) ->
  105. debug "preparing URL"
  106. Promise.try ->
  107. # Parse the specified URL, and use the resulting information to build a complete `options` object
  108. urlOptions = urlUtil.parse request.url, true
  109. _.extend request.options, {hostname: urlOptions.hostname, port: urlOptions.port}
  110. request.options.path = urlUtil.format {pathname: urlOptions.pathname, query: request.options.query ? urlOptions.query}
  111. request.protocol = S(urlOptions.protocol).chompRight(":").toString()
  112. Promise.resolve [request, response, requestState]
  113. prepareProtocol = (request, response, requestState) ->
  114. debug "preparing protocol"
  115. Promise.try ->
  116. request.protocolModule = switch request.protocol
  117. when "http" then http
  118. when "https" then https # CAUTION / FIXME: Node will silently ignore SSL settings without a custom agent!
  119. else null
  120. if not request.protocolModule?
  121. return Promise.reject() new errors.UnsupportedProtocolError "The protocol specified (#{protocol}) is not currently supported by this module."
  122. request.options.port ?= switch request.protocol
  123. when "http" then 80
  124. when "https" then 443
  125. Promise.resolve [request, response, requestState]
  126. prepareOptions = (request, response, requestState) ->
  127. debug "preparing options"
  128. Promise.try ->
  129. # Do some sanity checks - there are a number of options that cannot be used together
  130. if (request.options.formFields? or request.options.files?) and (request.options.inputStream? or request.options.inputBuffer?)
  131. return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot define both formFields/files and a raw inputStream or inputBuffer."), request, response, requestState)
  132. if request.options.encodeJSON and (request.options.inputStream? or request.options.inputBuffer?)
  133. return Promise.reject addErrorData(new errors.ConflictingOptionsError("You cannot use both encodeJSON and a raw inputStream or inputBuffer.", undefined, "If you meant to JSON-encode the stream, you will currently have to do so manually."), request, response, requestState)
  134. # If the user plans on streaming the response, we need to disable the agent entirely - otherwise the streams will block the pool.
  135. if request.responseOptions.stream
  136. request.options.agent ?= false
  137. Promise.resolve [request, response, requestState]
  138. preparePayload = (request, response, requestState) ->
  139. debug "preparing payload"
  140. Promise.try ->
  141. # If a 'files' parameter is present, then we will send the form data as multipart data - it's most likely binary data.
  142. multipart = request.options.forceMultipart or request.options.files?
  143. # Similarly, if any of the formFields values are either a Stream or a Buffer, we will assume that the form should be sent as multipart.
  144. multipart = multipart or _.any request.options.formFields, (item) ->
  145. item instanceof Buffer or isStream(item)
  146. # Really, 'files' and 'formFields' are the same thing - they mostly have different names for 1) clarity and 2) multipart detection. We combine them here.
  147. _.extend request.options.formFields, request.options.files
  148. # For a last sanity check, we want to know whether there are any Stream objects in our form data *at all* - these can't be used when encodeJSON is enabled.
  149. containsStreams = _.any request.options.formFields, (item) -> isStream(item)
  150. if request.options.encodeJSON and containsStreams
  151. return Promise.reject() new errors.ConflictingOptionsError "Sending a JSON-encoded payload containing data from a stream is not currently supported.", undefined, "Either don't use encodeJSON, or read your stream into a string or Buffer."
  152. if request.options.method in ["post", "put", "patch"]
  153. # Prepare the payload, and set the appropriate headers.
  154. if (request.options.encodeJSON or request.options.formFields?) and not multipart
  155. # We know the payload and its size in advance.
  156. debug "got url-encodable form-data"
  157. request.options.headers["content-type"] = "application/x-www-form-urlencoded"
  158. if request.options.encodeJSON
  159. request.payload = JSON.stringify request.options.formFields ? null
  160. else if not _.isEmpty request.options.formFields
  161. # The `querystring` module copies the key name verbatim, even if the value is actually an array. Things like PHP don't understand this, and expect every array-containing key to be suffixed with []. We'll just append that ourselves, then.
  162. request.payload = querystring.stringify formFixArray(request.options.formFields)
  163. else
  164. request.payload = ""
  165. request.options.headers["content-length"] = request.payload.length
  166. return Promise.resolve()
  167. else if request.options.formFields? and multipart
  168. # This is going to be multipart data, and we'll let `form-data` set the headers for us.
  169. debug "got multipart form-data"
  170. formDataObject = new formData()
  171. for fieldName, fieldValue of formFixArray(request.options.formFields)
  172. if not _.isArray fieldValue
  173. fieldValue = [fieldValue]
  174. for valueElement in fieldValue
  175. if valueElement._bhttpStreamWrapper?
  176. streamOptions = valueElement.options
  177. valueElement = valueElement.stream
  178. else
  179. streamOptions = {}
  180. formDataObject.append fieldName, valueElement, streamOptions
  181. request.payloadStream = formDataObject
  182. Promise.try ->
  183. formDataObject.getHeaders()
  184. .then (headers) ->
  185. if headers["content-transfer-encoding"] == "chunked" and not request.options.allowChunkedMultipart
  186. Promise.reject addErrorData(new MultipartError("Most servers do not support chunked transfer encoding for multipart/form-data payloads, and we could not determine the length of all the input streams. See the documentation for more information."), request, response, requestState)
  187. else
  188. _.extend request.options.headers, headers
  189. Promise.resolve()
  190. else if request.options.inputStream?
  191. # A raw inputStream was provided, just leave it be.
  192. debug "got inputStream"
  193. Promise.try ->
  194. request.payloadStream = request.options.inputStream
  195. if request.payloadStream._bhttpStreamWrapper? and (request.payloadStream.options.contentLength? or request.payloadStream.options.knownLength?)
  196. Promise.resolve(request.payloadStream.options.contentLength ? request.payloadStream.options.knownLength)
  197. else
  198. streamLength request.options.inputStream
  199. .then (length) ->
  200. debug "length for inputStream is %s", length
  201. request.options.headers["content-length"] = length
  202. .catch (err) ->
  203. debug "unable to determine inputStream length, switching to chunked transfer encoding"
  204. request.options.headers["content-transfer-encoding"] = "chunked"
  205. else if request.options.inputBuffer?
  206. # A raw inputBuffer was provided, just leave it be (but make sure it's an actual Buffer).
  207. debug "got inputBuffer"
  208. if typeof request.options.inputBuffer == "string"
  209. request.payload = new Buffer(request.options.inputBuffer) # Input string should be utf-8!
  210. else
  211. request.payload = request.options.inputBuffer
  212. debug "length for inputBuffer is %s", request.payload.length
  213. request.options.headers["content-length"] = request.payload.length
  214. return Promise.resolve()
  215. else
  216. # GET, HEAD and DELETE should not have a payload. While technically not prohibited by the spec, it's also not specified, and we'd rather not upset poorly-compliant webservers.
  217. # FIXME: Should this throw an Error?
  218. return Promise.resolve()
  219. .then ->
  220. Promise.resolve [request, response, requestState]
  221. prepareCleanup = (request, response, requestState) ->
  222. debug "preparing cleanup"
  223. Promise.try ->
  224. # Remove the options that we're not going to pass on to the actual http/https library.
  225. delete request.options[key] for key in ["query", "formFields", "files", "encodeJSON", "inputStream", "inputBuffer", "discardResponse", "keepRedirectResponses", "followRedirects", "noDecode", "decodeJSON", "allowChunkedMultipart", "forceMultipart"]
  226. # Lo-Dash apparently has no `map` equivalent for object keys...?
  227. fixedHeaders = {}
  228. for key, value of request.options.headers
  229. fixedHeaders[key.toLowerCase()] = value
  230. request.options.headers = fixedHeaders
  231. Promise.resolve [request, response, requestState]
  232. # The guts of the module
  233. prepareRequest = (request, response, requestState) ->
  234. debug "preparing request"
  235. # FIXME: Mock httpd for testing functionality.
  236. Promise.try ->
  237. middlewareFunctions = [
  238. prepareSession
  239. prepareDefaults
  240. prepareUrl
  241. prepareProtocol
  242. prepareOptions
  243. preparePayload
  244. prepareCleanup
  245. ]
  246. promiseChain = Promise.resolve [request, response, requestState]
  247. middlewareFunctions.forEach (middleware) -> # We must use the functional construct here, to avoid losing references
  248. promiseChain = promiseChain.spread (_request, _response, _requestState) ->
  249. middleware(_request, _response, _requestState)
  250. return promiseChain
  251. makeRequest = (request, response, requestState) ->
  252. debug "making %s request to %s", request.options.method.toUpperCase(), request.url
  253. Promise.try ->
  254. # Instantiate a regular HTTP/HTTPS request
  255. req = request.protocolModule.request request.options
  256. # This is where we write our payload or stream to the request, and the actual request is made.
  257. if request.payload?
  258. # The entire payload is a single Buffer.
  259. debug "sending payload"
  260. req.write request.payload
  261. req.end()
  262. else if request.payloadStream?
  263. # The payload is a stream.
  264. debug "piping payloadStream"
  265. if request.payloadStream._bhttpStreamWrapper?
  266. request.payloadStream.stream.pipe req
  267. else
  268. request.payloadStream.pipe req
  269. else
  270. # For GET, HEAD, DELETE, etc. there is no payload, but we still need to call end() to complete the request.
  271. debug "closing request without payload"
  272. req.end()
  273. new Promise (resolve, reject) ->
  274. # In case something goes wrong during this process, somehow...
  275. req.on "error", (err) ->
  276. reject err
  277. req.on "response", (res) ->
  278. resolve res
  279. .then (response) ->
  280. Promise.resolve [request, response, requestState]
  281. processResponse = (request, response, requestState) ->
  282. debug "processing response, got status code %s", response.statusCode
  283. # When we receive the response, we'll buffer it up and/or decode it, depending on what the user specified, and resolve the returned Promise. If the user just wants the raw stream, we resolve immediately after receiving a response.
  284. Promise.try ->
  285. # First, if a cookie jar is set and we received one or more cookies from the server, we should store them in our cookieJar.
  286. if request.cookieJar? and response.headers["set-cookie"]?
  287. promises = for cookieHeader in response.headers["set-cookie"]
  288. debug "storing cookie: %s", cookieHeader
  289. request.cookieJar.set cookieHeader, request.url
  290. Promise.all promises
  291. else
  292. Promise.resolve()
  293. .then ->
  294. # Now the actual response processing.
  295. response.request = request
  296. response.requestState = requestState
  297. response.redirectHistory = requestState.redirectHistory
  298. if response.statusCode in [301, 302, 303, 307] and request.responseOptions.followRedirects
  299. if requestState.redirectHistory.length >= (request.responseOptions.redirectLimit - 1)
  300. return Promise.reject addErrorData(new errors.RedirectError("The maximum amount of redirects ({request.responseOptions.redirectLimit}) was reached."))
  301. # 301: For GET and HEAD, redirect unchanged. For POST, PUT, PATCH, DELETE, "ask user" (in our case: throw an error.)
  302. # 302: Redirect, change method to GET.
  303. # 303: Redirect, change method to GET.
  304. # 307: Redirect, retain method. Make same request again.
  305. switch response.statusCode
  306. when 301
  307. switch request.options.method
  308. when "get", "head"
  309. return redirectUnchanged request, response, requestState
  310. when "post", "put", "patch", "delete"
  311. return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect for POST, PUT, PATCH or DELETE. RFC says we can't automatically continue."), request, response, requestState)
  312. else
  313. return Promise.reject addErrorData(new errors.RedirectError("Encountered a 301 redirect, but not sure how to proceed for the #{request.options.method.toUpperCase()} method."))
  314. when 302, 303
  315. return redirectGet request, response, requestState
  316. when 307
  317. if request.containsStreams and request.options.method not in ["get", "head"]
  318. return Promise.reject addErrorData(new errors.RedirectError("Encountered a 307 redirect for POST, PUT or DELETE, but your payload contained (single-use) streams. We therefore can't automatically follow the redirect."), request, response, requestState)
  319. else
  320. return redirectUnchanged request, response, requestState
  321. else if request.responseOptions.discardResponse
  322. response.resume() # Drain the response stream
  323. Promise.resolve response
  324. else
  325. new Promise (resolve, reject) ->
  326. if request.responseOptions.stream
  327. resolve response
  328. else
  329. response.on "error", (err) ->
  330. reject err
  331. response.pipe concatStream (body) ->
  332. # FIXME: Separate module for header parsing?
  333. if request.responseOptions.decodeJSON or ((response.headers["content-type"] ? "").split(";")[0] == "application/json" and not request.responseOptions.noDecode)
  334. try
  335. response.body = JSON.parse body
  336. catch err
  337. reject err
  338. else
  339. response.body = body
  340. resolve response
  341. .then (response) ->
  342. Promise.resolve [request, response, requestState]
  343. # Some wrappers
  344. doPayloadRequest = (url, data, options, callback) ->
  345. # A wrapper that processes the second argument to .post, .put, .patch shorthand API methods.
  346. # FIXME: Treat a {} for data as a null? Otherwise {} combined with inputBuffer/inputStream will error.
  347. if isStream(data)
  348. options.inputStream = data
  349. else if ofTypes(data, [Buffer]) or typeof data == "string"
  350. options.inputBuffer = data
  351. else
  352. options.formFields = data
  353. @request url, options, callback
  354. redirectGet = (request, response, requestState) ->
  355. debug "following forced-GET redirect to %s", response.headers["location"]
  356. Promise.try ->
  357. options = _.clone(requestState.originalOptions)
  358. options.method = "get"
  359. delete options[key] for key in ["inputBuffer", "inputStream", "files", "formFields"]
  360. doRedirect request, response, requestState, options
  361. redirectUnchanged = (request, response, requestState) ->
  362. debug "following same-method redirect to %s", response.headers["location"]
  363. Promise.try ->
  364. options = _.clone(requestState.originalOptions)
  365. doRedirect request, response, requestState, options
  366. doRedirect = (request, response, requestState, newOptions) ->
  367. Promise.try ->
  368. if not request.responseOptions.keepRedirectResponses
  369. response.resume() # Let the response stream drain out...
  370. requestState.redirectHistory.push response
  371. bhttpAPI._doRequest response.headers["location"], newOptions, requestState
  372. createCookieJar = (jar) ->
  373. # Creates a cookie jar wrapper with a simplified API.
  374. return {
  375. set: (cookie, url) ->
  376. new Promise (resolve, reject) =>
  377. @jar.setCookie cookie, url, (err, cookie) ->
  378. if err then reject(err) else resolve(cookie)
  379. get: (url) ->
  380. new Promise (resolve, reject) =>
  381. @jar.getCookieString url, (err, cookies) ->
  382. if err then reject(err) else resolve(cookies)
  383. jar: jar
  384. }
  385. # The exposed API
  386. bhttpAPI =
  387. head: (url, options = {}, callback) ->
  388. options.method = "head"
  389. @request url, options, callback
  390. get: (url, options = {}, callback) ->
  391. options.method = "get"
  392. @request url, options, callback
  393. post: (url, data, options = {}, callback) ->
  394. options.method = "post"
  395. doPayloadRequest.bind(this) url, data, options, callback
  396. put: (url, data, options = {}, callback) ->
  397. options.method = "put"
  398. doPayloadRequest.bind(this) url, data, options, callback
  399. patch: (url, data, options = {}, callback) ->
  400. options.method = "patch"
  401. doPayloadRequest.bind(this) url, data, options, callback
  402. delete: (url, data, options = {}, callback) ->
  403. options.method = "delete"
  404. @request url, options, callback
  405. request: (url, options = {}, callback) ->
  406. @_doRequest(url, options).nodeify(callback)
  407. _doRequest: (url, options, requestState) ->
  408. # This is split from the `request` method, so that the user doesn't have to pass in `undefined` for the `requestState` when they want to specify a `callback`.
  409. Promise.try =>
  410. request = {url: url, options: _.clone(options)}
  411. response = null
  412. requestState ?= {originalOptions: _.clone(options), redirectHistory: []}
  413. requestState.sessionOptions ?= @_sessionOptions ? {}
  414. prepareRequest request, response, requestState
  415. .spread (request, response, requestState) =>
  416. if request.responseOptions.justPrepare
  417. Promise.resolve [request, response, requestState]
  418. else
  419. Promise.try ->
  420. bhttpAPI.executeRequest request, response, requestState
  421. .spread (request, response, requestState) ->
  422. # The user likely only wants the response.
  423. Promise.resolve response
  424. executeRequest: (request, response, requestState) ->
  425. # Executes a pre-configured request.
  426. Promise.try ->
  427. makeRequest request, response, requestState
  428. .spread (request, response, requestState) ->
  429. processResponse request, response, requestState
  430. session: (options) ->
  431. options ?= {}
  432. options = _.clone options
  433. session = {}
  434. for key, value of this
  435. if value instanceof Function
  436. value = value.bind(session)
  437. session[key] = value
  438. if not options.cookieJar?
  439. options.cookieJar = createCookieJar(new toughCookie.CookieJar())
  440. else if options.cookieJar == false
  441. delete options.cookieJar
  442. else
  443. # Assume we've gotten a cookie jar.
  444. options.cookieJar = createCookieJar(options.cookieJar)
  445. session._sessionOptions = options
  446. return session
  447. wrapStream: (stream, options) ->
  448. # This is a method for wrapping a stream in an object that also contains metadata.
  449. return {
  450. _bhttpStreamWrapper: true
  451. stream: stream
  452. options: options
  453. }
  454. module.exports = bhttpAPI
  455. # That's all, folks!