Connection: allow UTF-8 strings in search requests

lib/Connection.js

@@ -168,9 +168,16 @@ Connection.prototype.connect = function() {
       self._idle.started = Date.now();
     } else if (/^AUTHENTICATE XOAUTH/.test(self._curReq.fullcmd)) {
       self._curReq.oauthError = new Buffer(info.text, 'base64').toString('utf8');
+      self.debug && self.debug('=> ' + inspect(CRLF));
       self._sock.write(CRLF);
-    } else if (type === 'APPEND')
+    } else if (type === 'APPEND') {
+      self.debug && self.debug('=> ' + inspect(self._curReq.appendData));
       self._sock.write(self._curReq.appendData);
+    } else if (self._curReq.lines && self._curReq.lines.length) {
+      var line = self._curReq.lines.shift() + '\r\n';
+      self.debug && self.debug('=> ' + inspect(line));
+      self._sock.write(line, 'binary');
+    }
   });
   parser.on('other', function(line) {
     var m;
@@ -392,17 +399,31 @@ Connection.prototype.expunge = function(uids, cb) {
     this._enqueue('EXPUNGE', cb);
 };
 
-Connection.prototype.search = function(options, cb) {
+Connection.prototype.search = function(criteria, cb) {
-  this._search('UID ', options, cb);
+  this._search('UID ', criteria, cb);
 };
 
-Connection.prototype._search = function(which, options, cb) {
+Connection.prototype._search = function(which, criteria, cb) {
   if (this._box === undefined)
     throw new Error('No mailbox is currently selected');
-  else if (!Array.isArray(options))
+  else if (!Array.isArray(criteria))
-    throw new Error('Expected array for search options');
+    throw new Error('Expected array for search criteria');
-
+
-  this._enqueue(which + 'SEARCH' + buildSearchQuery(options, this._caps), cb);
+  var cmd = which + 'SEARCH',
+      info = { hasUTF8: false /*output*/ },
+      query = buildSearchQuery(criteria, this._caps, info),
+      lines;
+  if (info.hasUTF8) {
+    cmd += ' CHARSET UTF-8';
+    lines = query.split(CRLF);
+    query = lines.shift();
+  }
+  cmd += query;
+  this._enqueue(cmd, cb);
+  if (info.hasUTF8) {
+    var req = this._queue[this._queue.length - 1];
+    req.lines = lines;
+  }
 };
 
 Connection.prototype.addFlags = function(uids, flags, cb) {
@@ -668,21 +689,21 @@ Connection.prototype._storeLabels = function(which, uids, labels, mode, cb) {
                 + 'X-GM-LABELS.SILENT (' + labels + ')', cb);
 };
 
-Connection.prototype.sort = function(sorts, options, cb) {
+Connection.prototype.sort = function(sorts, criteria, cb) {
-  this._sort('UID ', sorts, options, cb);
+  this._sort('UID ', sorts, criteria, cb);
 };
 
-Connection.prototype._sort = function(which, sorts, options, cb) {
+Connection.prototype._sort = function(which, sorts, criteria, cb) {
   if (this._box === undefined)
     throw new Error('No mailbox is currently selected');
   else if (!Array.isArray(sorts) || !sorts.length)
     throw new Error('Expected array with at least one sort criteria');
-  else if (!Array.isArray(options))
+  else if (!Array.isArray(criteria))
-    throw new Error('Expected array for search options');
+    throw new Error('Expected array for search criteria');
   else if (!this.serverSupports('SORT'))
     throw new Error('Sort is not supported on the server');
 
-  var criteria = sorts.map(function(c) {
+  sorts = sorts.map(function(c) {
     if (typeof c !== 'string')
       throw new Error('Unexpected sort criteria data type. '
                       + 'Expected string. Got: ' + typeof criteria);
@@ -708,8 +729,23 @@ Connection.prototype._sort = function(which, sorts, options, cb) {
     return modifier + c;
   });
 
-  this._enqueue(which + 'SORT (' + criteria.join(' ') + ') UTF-8'
+  sorts = sorts.join(' ');
-                + buildSearchQuery(options, this._caps), cb);
+
+  var info = { hasUTF8: false /*output*/ },
+      query = buildSearchQuery(criteria, this._caps, info),
+      charset = 'US-ASCII',
+      lines;
+  if (info.hasUTF8) {
+    charset = 'UTF-8';
+    lines = query.split(CRLF);
+    query = lines.shift();
+  }
+
+  this._enqueue(which + 'SORT (' + sorts + ') ' + charset + query, cb);
+  if (info.hasUTF8) {
+    var req = this._queue[this._queue.length - 1];
+    req.lines = lines;
+  }
 };
 
 Connection.prototype.esearch = function(criteria, options, cb) {
@@ -722,7 +758,15 @@ Connection.prototype._esearch = function(which, criteria, options, cb) {
   else if (!Array.isArray(criteria))
     throw new Error('Expected array for search options');
 
-  var searchQuery = buildSearchQuery(criteria, this._caps);
+  var info = { hasUTF8: false /*output*/ },
+      query = buildSearchQuery(criteria, this._caps, info),
+      charset = '',
+      lines;
+  if (info.hasUTF8) {
+    charset = 'CHARSET UTF-8';
+    lines = query.split(CRLF);
+    query = lines.shift();
+  }
 
   if (typeof options === 'function') {
     cb = options;
@@ -733,7 +777,11 @@ Connection.prototype._esearch = function(which, criteria, options, cb) {
   if (Array.isArray(options))
     options = options.join(' ');
 
-  this._enqueue(which + 'SEARCH RETURN (' + options + ') ' + searchQuery, cb);
+  this._enqueue(which + 'SEARCH RETURN (' + options + ') ' + charset + query, cb);
+  if (info.hasUTF8) {
+    var req = this._queue[this._queue.length - 1];
+    req.lines = lines;
+  }
 };
 
 Connection.prototype.setQuota = function(quotaRoot, limits, cb) {
@@ -797,10 +845,21 @@ Connection.prototype._thread = function(which, algorithm, criteria, cb) {
   if (!this.serverSupports('THREAD=' + algorithm))
     throw new Error('Server does not support that threading algorithm');
 
-  var cmd = which + 'THREAD ' + algorithm + ' UTF-8 '
+  var info = { hasUTF8: false /*output*/ },
-            + buildSearchQuery(criteria, this._caps);
+      query = buildSearchQuery(criteria, this._caps, info),
+      charset = 'US-ASCII',
+      lines;
+  if (info.hasUTF8) {
+    charset = 'UTF-8';
+    lines = query.split(CRLF);
+    query = lines.shift();
+  }
 
-  this._enqueue(cmd, cb);
+  this._enqueue(which + 'THREAD ' + algorithm + ' ' + charset + query, cb);
+  if (info.hasUTF8) {
+    var req = this._queue[this._queue.length - 1];
+    req.lines = lines;
+  }
 };
 // END Extension methods =======================================================
 
@@ -1288,7 +1347,7 @@ Connection.prototype._processQueue = function() {
 
   var out = prefix + ' ' + this._curReq.fullcmd;
   this.debug && this.debug('=> ' + inspect(out));
-  this._sock.write(out + CRLF);
+  this._sock.write(out + CRLF, 'utf8');
 };
 
 Connection.prototype._enqueue = function(fullcmd, promote, cb) {
@@ -1331,7 +1390,7 @@ module.exports = Connection;
 function escape(str) {
   return str.replace(RE_BACKSLASH, '\\\\').replace(RE_DBLQUOTE, '\\"');
 }
-function validateUIDList(uids) {
+function validateUIDList(uids, noThrow) {
   for (var i = 0, len = uids.length, intval; i < len; ++i) {
     if (typeof uids[i] === 'string') {
       if (uids[i] === '*' || uids[i] === '*:*') {
@@ -1343,14 +1402,38 @@ function validateUIDList(uids) {
     }
     intval = parseInt(''+uids[i], 10);
     if (isNaN(intval)) {
-      throw new Error('Message ID/number must be an integer, "*", or a range: '
+      var err = new Error('UID/seqno must be an integer, "*", or a range: '
-                      + uids[i]);
+                          + uids[i]);
+      if (noThrow)
+        return err;
+      else
+        throw err;
     } else if (typeof uids[i] !== 'number')
       uids[i] = intval;
   }
 }
-function buildSearchQuery(options, extensions, isOrChild) {
+function hasNonASCII(str) {
-  var searchargs = '';
+  var ret = false;
+  for (var i = 0, len = str.length; i < len; ++i) {
+    if (str.charCodeAt(i) > 0x7F) {
+      ret = true;
+      break;
+    }
+  }
+  return ret;
+}
+function buildString(str) {
+  if (typeof str !== 'string')
+    str = ''+str;
+
+  if (hasNonASCII(str)) {
+    var buf = new Buffer(str, 'utf8');
+    return '{' + buf.length + '}\r\n' + buf.toString('binary');
+  } else
+    return '"' + escape(str) + '"';
+}
+function buildSearchQuery(options, extensions, info, isOrChild) {
+  var searchargs = '', err, val;
   for (var i = 0, len = options.length; i < len; ++i) {
     var criteria = (isOrChild ? options : options[i]),
         args = null,
@@ -1369,9 +1452,9 @@ function buildSearchQuery(options, extensions, isOrChild) {
       if (args.length !== 2)
         throw new Error('OR must have exactly two arguments');
       searchargs += ' OR (';
-      searchargs += buildSearchQuery(args[0], extensions, true);
+      searchargs += buildSearchQuery(args[0], extensions, info, true);
       searchargs += ') (';
-      searchargs += buildSearchQuery(args[1], extensions, true);
+      searchargs += buildSearchQuery(args[1], extensions, info, true);
       searchargs += ')';
     } else {
       if (criteria[0] === '!') {
@@ -1406,8 +1489,10 @@ function buildSearchQuery(options, extensions, isOrChild) {
           if (!args || args.length !== 1)
             throw new Error('Incorrect number of arguments for search option: '
                             + criteria);
-          searchargs += modifier + criteria + ' "' + escape(''+args[0])
+          val = buildString(args[0]);
-                     + '"';
+          if (info && val[0] === '{')
+            info.hasUTF8 = true;
+          searchargs += modifier + criteria + ' ' + val;
         break;
         case 'BEFORE':
         case 'ON':
@@ -1448,8 +1533,11 @@ function buildSearchQuery(options, extensions, isOrChild) {
           if (!args || args.length !== 2)
             throw new Error('Incorrect number of arguments for search option: '
                             + criteria);
+          val = buildString(args[1]);
+          if (info && val[0] === '{')
+            info.hasUTF8 = true;
           searchargs += modifier + criteria + ' "' + escape(''+args[0])
-                     + '" "' + escape(''+args[1]) + '"';
+                     + '" ' + val;
         break;
         case 'UID':
           if (!args)
@@ -1463,7 +1551,6 @@ function buildSearchQuery(options, extensions, isOrChild) {
         case 'X-GM-THRID': // Gmail thread ID
           if (extensions.indexOf('X-GM-EXT-1') === -1)
             throw new Error('IMAP extension not available: ' + criteria);
-          var val;
           if (!args || args.length !== 1)
             throw new Error('Incorrect number of arguments for search option: '
                             + criteria);
@@ -1480,8 +1567,10 @@ function buildSearchQuery(options, extensions, isOrChild) {
           if (!args || args.length !== 1)
             throw new Error('Incorrect number of arguments for search option: '
                             + criteria);
-          searchargs += modifier + criteria + ' "' + escape(''+args[0])
+          val = buildString(args[0]);
-                     + '"';
+          if (info && val[0] === '{')
+            info.hasUTF8 = true;
+          searchargs += modifier + criteria + ' ' + val;
         break;
         case 'X-GM-LABELS': // Gmail labels
           if (extensions.indexOf('X-GM-EXT-1') === -1)
@@ -1492,15 +1581,13 @@ function buildSearchQuery(options, extensions, isOrChild) {
           searchargs += modifier + criteria + ' ' + args[0];
         break;
         default:
-          try {
+          // last hope it's a seqno set
-            // last hope it's a seqno set
+          // http://tools.ietf.org/html/rfc3501#section-6.4.4
-            // http://tools.ietf.org/html/rfc3501#section-6.4.4
+          var seqnos = (args ? [criteria].concat(args) : [criteria]);
-            var seqnos = (args ? [criteria].concat(args) : [criteria]);
+          if (!validateUIDList(seqnos, true))
-            validateUIDList(seqnos);
             searchargs += modifier + seqnos.join(',');
-          } catch(e) {
+          else
             throw new Error('Unexpected search option: ' + criteria);
-          }
       }
     }
     if (isOrChild)