From d9e705dea88b404409c33863f6d72508eeabc966 Mon Sep 17 00:00:00 2001 From: Brian White Date: Mon, 24 Jan 2011 02:04:11 -0500 Subject: [PATCH] Modify fetch() to be async and to no longer buffer message bodies. Fix NOOP handling. --- README.md | 42 +++++++++++---- imap.js | 150 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 126 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index c6eb7ed..21df832 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,22 @@ This example fetches the 'date', 'from', 'to', 'subject' message headers and the function() { imap.connect(cb); }, function() { imap.openBox('INBOX', false, cb); }, function(result) { box = result; imap.search([ 'UNSEEN', ['SINCE', 'May 20, 2010'] ], cb); }, - function(results) { imap.fetch(results, { request: { headers: ['from', 'to', 'subject', 'date'] } }, cb); }, - function(results) { console.log(sys.inspect(results, false, 6)); imap.logout(cb); } + function(results) { + var fetch = imap.fetch(results, { request: { headers: ['from', 'to', 'subject', 'date'] } }); + fetch.on('message', function(msg) { + console.log('Got message: ' + sys.inspect(msg, false, 5)); + msg.on('data', function(chunk) { + console.log('Got message chunk of size ' + chunk.length); + }); + msg.on('end', function() { + console.log('Finished message: ' + sys.inspect(msg, false, 5)); + }); + }); + fetch.on('end', function() { + console.log('Done fetching all messages!'); + imap.logout(cb); + }); + } ]; cb(); @@ -64,15 +78,22 @@ node-imap exposes one object: **ImapConnection**. * **messages** - An Object containing properties about message counts for this mailbox. * **total** - An Integer representing total number of messages in this mailbox. * **new** - An Integer representing the number of new (unread) messages in this mailbox. -* _FetchResult_ is an Object representing the result of a message fetch, and has the following properties: - * **id** - An Integer that uniquely identifies this message (within its mailbox). - * **flags** - An Array containing the flags currently set on this message. - * **date** - A String containing the internal server date for the message (always represented in GMT?) - * **headers** - An Object containing the headers of the message, **if headers were requested when calling fetch().** Note: The value of each property in the object is an Array containing the value(s) for that particular header name (in case of duplicate headers). - * **body** - A String containing the text of the entire or a portion of the message, **if a body was requested when calling fetch().** - * **structure** - An Array containing the structure of the message, **if the structure was requested when calling fetch().** See below for an explanation of the format of this property. +* _ImapMessage_ is an Object representing an email message. It consists of: + * Properties: + * **id** - An Integer that uniquely identifies this message (within its mailbox). + * **flags** - An Array containing the flags currently set on this message. + * **date** - A String containing the internal server date for the message (always represented in GMT?) + * **headers** - An Object containing the headers of the message, **if headers were requested when calling fetch().** Note: The value of each property in the object is an Array containing the value(s) for that particular header name (just in case there are duplicate headers). + * **structure** - An Array containing the structure of the message, **if the structure was requested when calling fetch().** See below for an explanation of the format of this property. + * Events: + * **data**(String) - Emitted for each message body chunk if a message body is being fetched + * **end** - Emitted when the fetch is complete for this message and its properties +* _ImapFetch_ is an Object that emits these events: + * **message**(ImapMessage) - Emitted for each message resulting from a fetch request + * **end** - Emitted when the fetch request is complete A message structure with multiple parts might look something like the following: + [ { type: 'mixed' , params: { boundary: '000e0cd294e80dc84c0475bf339d' } , disposition: null @@ -142,6 +163,7 @@ The above structure describes a message having both an attachment and two forms Each message part is identified by a partID which is used when you want to fetch the content of that part (**see fetch()**). The structure of a message with only one part will simply look something like this: + [ { partID: '1' , type: { name: 'text/plain' @@ -330,7 +352,7 @@ ImapConnection Functions * 'UID' - Messages with message IDs corresponding to the specified message ID set. Ranges are permitted (e.g. '2504:2507' or '*' or '2504:*'). * **Note:** By default, all criterion are ANDed together. You can use the special 'OR' on **two** criterion to find messages matching either search criteria (see example above). -* **fetch**(Integer/String/Array, Object, Function) - _(void)_ - Fetches the message(s) identified by the first parameter, in the currently open mailbox. The first parameter can either be an Integer for a single message ID, a String for a message ID range (e.g. '2504:2507' or '*' or '2504:*'), or an Array containing any number of the aforementioned Integers and/or Strings. The Function parameter is the callback with two parameters: the error (null if none) and an Array of _FetchResult_ Objects containing the results of the fetch request. An Object parameter is a set of options used to determine how and what exactly to fetch. The valid options are: +* **fetch**(Integer/String/Array, Object) - _ImapFetch_ - Fetches the message(s) identified by the first parameter, in the currently open mailbox. The first parameter can either be an Integer for a single message ID, a String for a message ID range (e.g. '2504:2507' or '*' or '2504:*'), or an Array containing any number of the aforementioned Integers and/or Strings. The second (Object) parameter is a set of options used to determine how and what exactly to fetch. The valid options are: * **markSeen** - A Boolean indicating whether to mark the message(s) as read when fetching it. **Default:** false * **request** - An Object indicating what to fetch (at least **headers** OR **body** must be set to false -- in other words, you can only fetch one aspect of the message at a time): * **struct** - A Boolean indicating whether to fetch the structure of the message. **Default:** true diff --git a/imap.js b/imap.js index ed644c9..6b4fa5c 100644 --- a/imap.js +++ b/imap.js @@ -28,6 +28,7 @@ function ImapConnection (options) { tmrConn: null, curData: '', curExpected: 0, + curXferred: 0, capabilities: [], box: { _uidnext: 0, _flags: [], _newKeywords: false, validity: 0, keywords: [], permFlags: [], name: null, messages: { total: 0, new: 0 }} }; @@ -91,57 +92,96 @@ ImapConnection.prototype.connect = function(loginCb) { fnInit(); }); this._state.conn.on('data', function(data) { - var literalData = '', trailingCRLF = false; - debug('RECEIVED: ' + data); + var trailingCRLF = false, literalInfo, bypass = false; + debug('<>: ' + sys.inspect(data)); - if (data.indexOf(CRLF) === -1) { - if (self._state.curData) + if (self._state.curExpected === 0) { + if (data.indexOf(CRLF) === -1) { self._state.curData += data; - else - self._state.curData = data; - - if (self._state.curData.indexOf(CRLF) === -1) return; + } + if (self._state.curData.length) { + data = self._state.curData + data; + self._state.curData = ''; + } } - if (self._state.curData) - data = self._state.curData + data; - self._state.curData = undefined; - // Don't mess with incoming data if it's part of a literal - var literalInfo; - if (self._state.curExpected === 0 && (literalInfo = /\{(\d+)\}$/.exec(data.substr(0, data.indexOf(CRLF))))) - self._state.curExpected = parseInt(literalInfo[1]); if (self._state.curExpected > 0) { - if (data.length - (data.indexOf(CRLF)+2) <= self._state.curExpected) { - self._state.curData = data; + var extra = '', curReq = self._state.requests[0]; + if (!curReq._done) { + self._state.curXferred += data.length; + if (self._state.curXferred <= self._state.curExpected) { + if (curReq._msgtype === 'headers') + // buffer headers since they're generally not large and need to be + // processed anyway + self._state.curData += data; + else + curReq._msg.emit('data', data); + return; + } + var pos = data.length-(self._state.curXferred-self._state.curExpected); + extra = data.substr(pos); + if (pos > 0) { + if (curReq._msgtype === 'headers') { + self._state.curData += data.substr(0, pos); + curReq._msgheaders = self._state.curData; + } else + curReq._msg.emit('data', data.substr(0, pos)); + } + self._state.curData = ''; + data = extra; + curReq._done = true; + } + // make sure we have at least ")\r\n" in the post-literal data + if (data.indexOf(CRLF) === -1) { + self._state.curData += data; return; } - literalData = data.substr(data.indexOf(CRLF) + 2, self._state.curExpected); - data = data.substr(0, data.indexOf(CRLF)) + data.substr(data.indexOf(CRLF) + 2 + self._state.curExpected); + if (self._state.curData.length) + data = self._state.curData + data; + // add any additional k/v pairs that appear after the literal data + var fetchdesc = curReq._fetchdesc + data.substring(0, data.indexOf(CRLF)-1).trim(); + parseFetch(fetchdesc, curReq._msgheaders, curReq._msg); + data = data.substr(data.indexOf(CRLF)+2); self._state.curExpected = 0; - if (data.substr(data.indexOf(CRLF)+2, 1) === '*') { - // found additional responses, so don't try splitting the proceeding response(s) for better performance in case they have literals too - var extra = data.substr(data.indexOf(CRLF)+2); - process.nextTick(function() { self._state.conn.emit('data', extra); }); - data = data.substring(0, data.indexOf(CRLF)); + self._state.curXferred = 0; + self._state.curData = ''; + curReq._done = false; + curReq._msg.emit('end'); + if (data[0] === '*') { + // found additional responses, so don't try splitting the proceeding + // response(s) for better performance in case they have literals too + process.nextTick(function() { self._state.conn.emit('data', data); }); + return; } + } else if (self._state.curExpected === 0 + && (literalInfo = /\{(\d+)\}$/.exec(data.substr(0, data.indexOf(CRLF))))) { + self._state.curExpected = parseInt(literalInfo[1]); + var curReq = self._state.requests[0]; + //if (/^UID FETCH/.test(curReq.command)) { + var type = /BODY\[(.*)\](?:\<[\d]+\>)?/.exec(data.substr(0, data.indexOf(CRLF))), + msg = new ImapMessage(); + type = type[1]; + parseFetch(data.substring(data.indexOf("(")+1, data.indexOf(CRLF)), "", msg); + curReq._fetchdesc = data.substring(data.indexOf("(")+1, data.indexOf(CRLF)); + curReq._msg = msg; + curReq._fetcher.emit('message', msg); + curReq._msgtype = (type.indexOf('HEADER') === 0 ? 'headers' : 'body'); + self._state.conn.emit('data', data.substr(data.indexOf(CRLF)+2)); + //} + return; } - if (data.test(/\r\n$/)) - trailingCRLF = true; - + if (data.length === 0) + return; data = data.split(CRLF).filter(isNotEmpty); // Defer any extra server responses found in the incoming data if (data.length > 1) { - data.slice(1).forEach(function(line) { process.nextTick(function() { - if (trailingCRLF) - self._state.conn.emit('data', line + CRLF); - else - self._state.conn.emit('data', line); + self._state.conn.emit('data', line + CRLF); }); }); } @@ -254,17 +294,6 @@ ImapConnection.prototype.connect = function(loginCb) { if (self._state.box.messages.total > 0) self._state.box.messages.total--; break; - default: - // Check for FETCH result - if (/^FETCH /i.test(data[2]) && self._state.requests[0].command.indexOf('UID FETCH') === 0) { - var idxResult; - if (self._state.requests[0].args.length === 0) - self._state.requests[0].args.push([]); - self._state.requests[0].args[0].push({ id: null, flags: [], date: null, headers: null, body: null, structure: null }); - idxResult = self._state.requests[0].args[0].length-1; - parseFetch(data[2].substring(7, data[2].length-1), literalData, self._state.requests[0].args[0][idxResult]); - } - break; } } } @@ -273,9 +302,7 @@ ImapConnection.prototype.connect = function(loginCb) { clearTimeout(self._state.tmrKeepalive); self._state.tmrKeepalive = setTimeout(self._idleCheck.bind(self), self._state.tmoKeepalive); - if (data[2] === 'NOOP completed.') - return; - else if (self._state.status === STATES.BOXSELECTING) { + if (self._state.status === STATES.BOXSELECTING) { if (data[1] === 'OK') { sendBox = true; self._state.status = STATES.BOXSELECTED; @@ -308,7 +335,8 @@ ImapConnection.prototype.connect = function(loginCb) { } args.unshift(err); self._state.requests[0].callback.apply({}, args); - } + } else if (self._state.requests[0].command.indexOf("UID FETCH") === 0) + self._state.requests[0]._fetcher.emit('end'); self._state.requests.shift(); process.nextTick(function() { self._send(); }); @@ -423,7 +451,7 @@ ImapConnection.prototype.search = function(options, cb) { this._send('UID SEARCH' + buildSearchQuery(options), cb); }; -ImapConnection.prototype.fetch = function(uids, options, cb) { +ImapConnection.prototype.fetch = function(uids, options) { if (this._state.status !== STATES.BOXSELECTED) throw new Error('No mailbox is currently selected'); if (!Array.isArray(uids)) @@ -441,7 +469,6 @@ ImapConnection.prototype.fetch = function(uids, options, cb) { body: false // / } }, toFetch, bodyRange = ''; - cb = arguments[arguments.length-1]; if (typeof options !== 'object') options = {}; options = extend(true, defaults, options); @@ -471,8 +498,12 @@ ImapConnection.prototype.fetch = function(uids, options, cb) { toFetch = 'HEADER.FIELDS (' + options.request.headers.join(' ').toUpperCase() + ')'; // fetch specific headers only this._send('UID FETCH ' + uids.join(',') + ' (FLAGS INTERNALDATE' - + (options.request.struct ? ' BODYSTRUCTURE' : '') - + (toFetch ? ' BODY' + (!options.markSeen ? '.PEEK' : '') + '[' + toFetch + ']' + bodyRange : '') + ')', cb); + + (options.request.struct ? ' BODYSTRUCTURE' : '') + + (toFetch ? ' BODY' + (!options.markSeen ? '.PEEK' : '') + + '[' + toFetch + ']' + bodyRange : '') + ')'); + var imapFetcher = new ImapFetch(); + this._state.requests[this._state.requests.length-1]._fetcher = imapFetcher; + return imapFetcher; }; ImapConnection.prototype.addFlags = function(uids, flags, cb) { @@ -657,7 +688,7 @@ ImapConnection.prototype._idleCheck = function() { }; ImapConnection.prototype._noop = function() { if (this._state.status >= STATES.AUTH) - this._send('NOOP', undefined, true); + this._send('NOOP', undefined); }; ImapConnection.prototype._send = function(cmdstr, cb, bypass) { if (arguments.length > 0 && !bypass) @@ -667,10 +698,15 @@ ImapConnection.prototype._send = function(cmdstr, cb, bypass) { this._state.isIdle = false; var cmd = (bypass ? cmdstr : this._state.requests[0].command); this._state.conn.write('A' + ++this._state.curId + ' ' + cmd + CRLF); - debug('SENT: A' + this._state.curId + ' ' + cmd); + debug('<>: A' + this._state.curId + ' ' + cmd); } }; +function ImapMessage() {} +sys.inherits(ImapMessage, EventEmitter); +function ImapFetch() {} +sys.inherits(ImapFetch, EventEmitter); + /****** Utility Functions ******/ function buildSearchQuery(options, isOrChild) { @@ -878,7 +914,10 @@ function parseFetch(str, literalData, fetchData) { // and {xxxx} is the byte count for the literalData describing the preceding item (almost always "BODY") var key, idxNext; while (str.length > 0) { - key = (str.substr(0, 5) === 'BODY[' ? str.substring(0, (str.indexOf('>') > -1 ? str.indexOf('>') : str.indexOf(']'))+1) : str.substring(0, str.indexOf(' '))); + key = (str.substr(0, 5) === 'BODY[' ? + str.substring(0, + (str.indexOf('>') > -1 ? str.indexOf('>') : str.indexOf(']'))+1) + : str.substring(0, str.indexOf(' '))); str = str.substring(str.indexOf(' ')+1); if (str.substr(0, 3) === 'NIL') idxNext = 3; @@ -912,8 +951,7 @@ function parseFetch(str, literalData, fetchData) { fetchData.headers[header] = []; fetchData.headers[header].push(headers[i].substr(headers[i].indexOf(': ')+2).replace(/\r\n/g, '').trim()); } - } else // full message or part body - fetchData.body = literalData; + } } } str = str.substr(idxNext).trim();