Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

mailnews/base/util/ABQueryUtils.jsm

@@ -0,0 +1,130 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains helper methods for dealing with addressbook search URIs.
+ */
+
+this.EXPORTED_SYMBOLS = ["getSearchTokens", "getModelQuery",
+                         "modelQueryHasUserValue", "generateQueryURI",
+                         "encodeABTermValue"];
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+/**
+ * Parse the multiword search string to extract individual search terms
+ * (separated on the basis of spaces) or quoted exact phrases to search
+ * against multiple fields of the addressbook cards.
+ *
+ * @param aSearchString The full search string entered by the user.
+ *
+ * @return an array of separated search terms from the full search string.
+ */
+function getSearchTokens(aSearchString) {
+  let searchString = aSearchString.trim();
+  if (searchString == "")
+    return [];
+
+  let quotedTerms = [];
+
+  // Split up multiple search words to create a *foo* and *bar* search against
+  // search fields, using the OR-search template from modelQuery for each word.
+  // If the search query has quoted terms as "foo bar", extract them as is.
+  let startIndex;
+  while ((startIndex = searchString.indexOf('"')) != -1) {
+    let endIndex = searchString.indexOf('"', startIndex + 1);
+    if (endIndex == -1)
+      endIndex = searchString.length;
+
+    quotedTerms.push(searchString.substring(startIndex + 1, endIndex));
+    let query = searchString.substring(0, startIndex);
+    if (endIndex < searchString.length)
+      query += searchString.substr(endIndex + 1);
+
+    searchString = query.trim();
+  }
+
+  let searchWords = [];
+  if (searchString.length != 0) {
+    searchWords = quotedTerms.concat(searchString.split(/\s+/));
+  } else {
+    searchWords = quotedTerms;
+  }
+
+  return searchWords;
+}
+
+/**
+ * For AB quicksearch or recipient autocomplete, get the normal or phonetic model
+ * query URL part from prefs, allowing users to customize these searches.
+ * @param aBasePrefName  the full pref name of default, non-phonetic model query,
+ *                       e.g. mail.addr_book.quicksearchquery.format
+ *                       If phonetic search is used, corresponding pref must exist:
+ *                       e.g. mail.addr_book.quicksearchquery.format.phonetic
+ * @return               depending on mail.addr_book.show_phonetic_fields pref,
+ *                       the value of aBasePrefName or aBasePrefName + ".phonetic"
+ */
+function getModelQuery(aBasePrefName) {
+  let modelQuery = "";
+  if (Services.prefs.getComplexValue("mail.addr_book.show_phonetic_fields",
+      Components.interfaces.nsIPrefLocalizedString).data == "true") {
+    modelQuery = Services.prefs.getCharPref(aBasePrefName + ".phonetic");
+  } else {
+    modelQuery = Services.prefs.getCharPref(aBasePrefName);
+  }
+  // remove leading "?" to migrate existing customized values for mail.addr_book.quicksearchquery.format
+  // todo: could this be done in a once-off migration at install time to avoid repetitive calls?
+  if (modelQuery.startsWith("?"))
+    modelQuery = modelQuery.slice(1);
+  return modelQuery;
+}
+
+/**
+ * Check if the currently used pref with the model query was customized by user.
+ * @param aBasePrefName  the full pref name of default, non-phonetic model query,
+ *                       e.g. mail.addr_book.quicksearchquery.format
+ *                       If phonetic search is used, corresponding pref must exist:
+ *                       e.g. mail.addr_book.quicksearchquery.format.phonetic
+ * @return               true or false
+ */
+function modelQueryHasUserValue(aBasePrefName) {
+  if (Services.prefs.getComplexValue("mail.addr_book.show_phonetic_fields",
+      Components.interfaces.nsIPrefLocalizedString).data == "true")
+    return Services.prefs.prefHasUserValue(aBasePrefName + ".phonetic");
+  return Services.prefs.prefHasUserValue(aBasePrefName);
+}
+
+/*
+ * Given a database model query and a list of search tokens,
+ * return query URI.
+ *
+ * @param aModelQuery database model query
+ * @param aSearchWords an array of search tokens.
+ *
+ * @return query URI.
+ */
+function generateQueryURI(aModelQuery, aSearchWords) {
+  // If there are no search tokens, we simply return an empty string.
+  if (!aSearchWords || aSearchWords.length == 0)
+    return "";
+
+  let queryURI = "";
+  aSearchWords.forEach(searchWord =>
+    queryURI += aModelQuery.replace(/@V/g, encodeABTermValue(searchWord)));
+
+  // queryURI has all the (or(...)) searches, link them up with (and(...)).
+  queryURI = "?(and" + queryURI + ")";
+
+  return queryURI;
+}
+
+
+/**
+ * Encode the string passed as value into an addressbook search term.
+ * The '(' and ')' characters are special for the addressbook
+ * search query language, but are not escaped in encodeURIComponent()
+ * so must be done manually on top of it.
+ */
+function encodeABTermValue(aString) {
+  return encodeURIComponent(aString).replace(/\(/g, "%28").replace(/\)/g, "%29");
+}

mailnews/base/util/IOUtils.js

@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["IOUtils"];
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var kStringBlockSize = 4096;
+var kStreamBlockSize = 8192;
+
+var IOUtils =
+{
+  /**
+   * Read a file containing ASCII text into a string.
+   *
+   * @param aFile  An nsIFile representing the file to read or a string containing
+   *               the file name of a file under user's profile.
+   * @returns A string containing the contents of the file, presumed to be ASCII
+   *          text. If the file didn't exist, returns null.
+   */
+  loadFileToString: function(aFile) {
+    let file;
+    if (!(aFile instanceof Ci.nsIFile)) {
+      file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+      file.append(aFile);
+    } else {
+      file = aFile;
+    }
+
+    if (!file.exists())
+      return null;
+
+    let fstream = Cc["@mozilla.org/network/file-input-stream;1"]
+                    .createInstance(Ci.nsIFileInputStream);
+    // PR_RDONLY
+    fstream.init(file, 0x01, 0, 0);
+
+    let sstream = Cc["@mozilla.org/scriptableinputstream;1"]
+                    .createInstance(Ci.nsIScriptableInputStream);
+    sstream.init(fstream);
+
+    let data = "";
+    while (sstream.available()) {
+      data += sstream.read(kStringBlockSize);
+    }
+
+    sstream.close();
+    fstream.close();
+
+    return data;
+  },
+
+  /**
+   * Save a string containing ASCII text into a file. The file will be overwritten
+   * and contain only the given text.
+   *
+   * @param aFile   An nsIFile representing the file to write or a string containing
+   *                the file name of a file under user's profile.
+   * @param aData   The string to write.
+   * @param aPerms  The octal file permissions for the created file. If unset
+   *                the default of 0o600 is used.
+   */
+  saveStringToFile: function(aFile, aData, aPerms = 0o600) {
+    let file;
+    if (!(aFile instanceof Ci.nsIFile)) {
+      file = Services.dirsvc.get("ProfD", Ci.nsIFile);
+      file.append(aFile);
+    } else {
+      file = aFile;
+    }
+
+    let foStream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
+                     .createInstance(Ci.nsIFileOutputStream);
+
+    // PR_WRONLY + PR_CREATE_FILE + PR_TRUNCATE
+    foStream.init(file, 0x02 | 0x08 | 0x20, aPerms, 0);
+    // safe-file-output-stream appears to throw an error if it doesn't write everything at once
+    // so we won't worry about looping to deal with partial writes.
+    // In case we try to use this function for big files where buffering
+    // is needed we could use the implementation in saveStreamToFile().
+    foStream.write(aData, aData.length);
+    foStream.QueryInterface(Ci.nsISafeOutputStream).finish();
+    foStream.close();
+  },
+
+  /**
+   * Saves the given input stream to a file.
+   *
+   * @param aIStream The input stream to save.
+   * @param aFile    The file to which the stream is saved.
+   * @param aPerms   The octal file permissions for the created file. If unset
+   *                 the default of 0o600 is used.
+   */
+  saveStreamToFile: function(aIStream, aFile, aPerms = 0o600) {
+    if (!(aIStream instanceof Ci.nsIInputStream))
+      throw new Error("Invalid stream passed to saveStreamToFile");
+    if (!(aFile instanceof Ci.nsIFile))
+      throw new Error("Invalid file passed to saveStreamToFile");
+
+    let fstream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
+                    .createInstance(Ci.nsIFileOutputStream);
+    let buffer  = Cc["@mozilla.org/network/buffered-output-stream;1"]
+                    .createInstance(Ci.nsIBufferedOutputStream);
+
+    // Write the input stream to the file.
+    // PR_WRITE + PR_CREATE + PR_TRUNCATE
+    fstream.init(aFile, 0x04 | 0x08 | 0x20, aPerms, 0);
+    buffer.init(fstream, kStreamBlockSize);
+
+    buffer.writeFrom(aIStream, aIStream.available());
+
+    // Close the output streams.
+    if (buffer instanceof Components.interfaces.nsISafeOutputStream)
+      buffer.finish();
+    else
+      buffer.close();
+    if (fstream instanceof Components.interfaces.nsISafeOutputStream)
+      fstream.finish();
+    else
+      fstream.close();
+
+    // Close the input stream.
+    aIStream.close();
+    return aFile;
+  },
+
+  /**
+   * Returns size of system memory.
+   */
+  getPhysicalMemorySize: function() {
+    return Services.sysinfo.getPropertyAsInt64("memsize");
+  },
+};

mailnews/base/util/JXON.js

@@ -0,0 +1,180 @@
+/* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This is a modification of the JXON parsers found on the page
+// <https://developer.mozilla.org/en-US/docs/JXON>
+
+var EXPORTED_SYMBOLS = ["JXON"];
+
+var JXON = new (function() {
+  const sValueProp = "value"; /* you can customize these values */
+  const sAttributesProp = "attr";
+  const sAttrPref = "@";
+  const sElementListPrefix = "$";
+  const sConflictSuffix = "_"; // used when there's a name conflict with special JXON properties
+  const aCache = [];
+  const rIsNull = /^\s*$/;
+  const rIsBool = /^(?:true|false)$/i;
+
+  function parseText(sValue) {
+    //if (rIsNull.test(sValue))
+    //  return null;
+    if (rIsBool.test(sValue)) 
+      return sValue.toLowerCase() === "true";
+    if (isFinite(sValue))
+      return parseFloat(sValue);
+    if (isFinite(Date.parse(sValue)))
+      return new Date(sValue);
+    return sValue;
+  };
+
+  function EmptyTree() {
+  }
+  EmptyTree.prototype = {
+    toString : function () {
+      return "null";
+    },
+    valueOf : function () {
+      return null;
+    },
+  };
+
+  function objectify(vValue) {
+    if (vValue === null)
+      return new EmptyTree();
+    else if (vValue instanceof Object)
+      return vValue;
+    else
+      return new vValue.constructor(vValue); // What does this? copy?
+  };
+
+  function createObjTree(oParentNode, nVerb, bFreeze, bNesteAttr) {
+    const nLevelStart = aCache.length;
+    const bChildren = oParentNode.hasChildNodes();
+    const bAttributes = oParentNode.attributes &&
+                        oParentNode.attributes.length;
+    const bHighVerb = Boolean(nVerb & 2);
+
+    var sProp = 0;
+    var vContent = 0;
+    var nLength = 0;
+    var sCollectedTxt = "";
+    var vResult = bHighVerb ? {} : /* put here the default value for empty nodes: */ true;
+
+    if (bChildren) {
+      for (var oNode, nItem = 0; nItem < oParentNode.childNodes.length; nItem++) {
+        oNode = oParentNode.childNodes.item(nItem);
+        if (oNode.nodeType === 4) // CDATASection
+          sCollectedTxt += oNode.nodeValue; 
+        else if (oNode.nodeType === 3) // Text
+          sCollectedTxt += oNode.nodeValue;
+        else if (oNode.nodeType === 1) // Element
+          aCache.push(oNode);
+      }
+    }
+
+    const nLevelEnd = aCache.length;
+    const vBuiltVal = parseText(sCollectedTxt);
+
+    if (!bHighVerb && (bChildren || bAttributes))
+      vResult = nVerb === 0 ? objectify(vBuiltVal) : {};
+
+    for (var nElId = nLevelStart; nElId < nLevelEnd; nElId++) {
+      sProp = aCache[nElId].nodeName;
+      if (sProp == sValueProp || sProp == sAttributesProp)
+        sProp = sProp + sConflictSuffix;
+      vContent = createObjTree(aCache[nElId], nVerb, bFreeze, bNesteAttr);
+      if (!vResult.hasOwnProperty(sProp)) {
+        vResult[sProp] = vContent;
+        vResult[sElementListPrefix + sProp] = [];
+      }
+      vResult[sElementListPrefix + sProp].push(vContent);
+      nLength++;
+    }
+
+    if (bAttributes) {
+      const nAttrLen = oParentNode.attributes.length;
+      const sAPrefix = bNesteAttr ? "" : sAttrPref;
+      const oAttrParent = bNesteAttr ? {} : vResult;
+
+      for (var oAttrib, nAttrib = 0; nAttrib < nAttrLen; nLength++, nAttrib++) {
+        oAttrib = oParentNode.attributes.item(nAttrib);
+        oAttrParent[sAPrefix + oAttrib.name] = parseText(oAttrib.value);
+      }
+
+      if (bNesteAttr) {
+        if (bFreeze)
+          Object.freeze(oAttrParent);
+        vResult[sAttributesProp] = oAttrParent;
+        nLength -= nAttrLen - 1;
+      }
+    }
+
+    if (nVerb === 3 || (nVerb === 2 || nVerb === 1 && nLength > 0) && sCollectedTxt)
+      vResult[sValueProp] = vBuiltVal;
+    else if (!bHighVerb && nLength === 0 && sCollectedTxt)
+      vResult = vBuiltVal;
+
+    if (bFreeze && (bHighVerb || nLength > 0))
+      Object.freeze(vResult);
+
+    aCache.length = nLevelStart;
+
+    return vResult;
+  };
+
+  function loadObjTree(oXMLDoc, oParentEl, oParentObj) {
+    var vValue, oChild;
+
+    if (oParentObj instanceof String || oParentObj instanceof Number ||
+        oParentObj instanceof Boolean)
+      oParentEl.appendChild(oXMLDoc.createTextNode(oParentObj.toString())); /* verbosity level is 0 */
+    else if (oParentObj.constructor === Date)
+      oParentEl.appendChild(oXMLDoc.createTextNode(oParentObj.toGMTString()));
+
+    for (var sName in oParentObj) {
+      vValue = oParentObj[sName];
+      if (isFinite(sName) || vValue instanceof Function)
+        continue; /* verbosity level is 0 */
+      if (sName === sValueProp) {
+        if (vValue !== null && vValue !== true) {
+          oParentEl.appendChild(oXMLDoc.createTextNode(
+              vValue.constructor === Date ? vValue.toGMTString() : String(vValue)));
+        }
+      } else if (sName === sAttributesProp) { /* verbosity level is 3 */
+        for (var sAttrib in vValue)
+          oParentEl.setAttribute(sAttrib, vValue[sAttrib]);
+      } else if (sName.charAt(0) === sAttrPref) {
+        oParentEl.setAttribute(sName.slice(1), vValue);
+      } else if (vValue.constructor === Array) {
+        for (var nItem = 0; nItem < vValue.length; nItem++) {
+          oChild = oXMLDoc.createElement(sName);
+          loadObjTree(oXMLDoc, oChild, vValue[nItem]);
+          oParentEl.appendChild(oChild);
+        }
+      } else {
+        oChild = oXMLDoc.createElement(sName);
+        if (vValue instanceof Object)
+          loadObjTree(oXMLDoc, oChild, vValue);
+        else if (vValue !== null && vValue !== true)
+          oChild.appendChild(oXMLDoc.createTextNode(vValue.toString()));
+        oParentEl.appendChild(oChild);
+     }
+   }
+  };
+
+  this.build = function(oXMLParent, nVerbosity /* optional */, bFreeze /* optional */, bNesteAttributes /* optional */) {
+    const _nVerb = arguments.length > 1 &&
+        typeof nVerbosity === "number" ? nVerbosity & 3 :
+        /* put here the default verbosity level: */ 1;
+    return createObjTree(oXMLParent, _nVerb, bFreeze || false,
+        arguments.length > 3 ? bNesteAttributes : _nVerb === 3);
+  };
+
+  this.unbuild = function(oObjTree) {
+    const oNewDoc = document.implementation.createDocument("", "", null);
+    loadObjTree(oNewDoc, oNewDoc, oObjTree);
+    return oNewDoc;
+  };
+})();

mailnews/base/util/OAuth2.jsm

@@ -0,0 +1,234 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Provides OAuth 2.0 authentication
+ */
+var EXPORTED_SYMBOLS = ["OAuth2"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Http.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+function parseURLData(aData) {
+  let result = {};
+  aData.split(/[?#]/, 2)[1].split("&").forEach(function (aParam) {
+    let [key, value] = aParam.split("=");
+    result[key] = value;
+  });
+  return result;
+}
+
+// Only allow one connecting window per endpoint.
+var gConnecting = {};
+
+function OAuth2(aBaseURI, aScope, aAppKey, aAppSecret) {
+    this.authURI = aBaseURI + "oauth2/auth";
+    this.tokenURI = aBaseURI + "oauth2/token";
+    this.consumerKey = aAppKey;
+    this.consumerSecret = aAppSecret;
+    this.scope = aScope;
+    this.extraAuthParams = [];
+
+    this.log = Log4Moz.getConfiguredLogger("TBOAuth");
+}
+
+OAuth2.CODE_AUTHORIZATION = "authorization_code";
+OAuth2.CODE_REFRESH = "refresh_token";
+
+OAuth2.prototype = {
+
+    responseType: "code",
+    consumerKey: null,
+    consumerSecret: null,
+    completionURI: "http://localhost",
+    requestWindowURI: "chrome://messenger/content/browserRequest.xul",
+    requestWindowFeatures: "chrome,private,centerscreen,width=980,height=600",
+    requestWindowTitle: "",
+    scope: null,
+
+    accessToken: null,
+    refreshToken: null,
+    tokenExpires: 0,
+
+    connect: function connect(aSuccess, aFailure, aWithUI, aRefresh) {
+
+        this.connectSuccessCallback = aSuccess;
+        this.connectFailureCallback = aFailure;
+
+        if (!aRefresh && this.accessToken) {
+            aSuccess();
+        } else if (this.refreshToken) {
+            this.requestAccessToken(this.refreshToken, OAuth2.CODE_REFRESH);
+        } else {
+            if (!aWithUI) {
+                aFailure('{ "error": "auth_noui" }');
+                return;
+            }
+            if (gConnecting[this.authURI]) {
+                aFailure("Window already open");
+                return;
+            }
+            this.requestAuthorization();
+        }
+    },
+
+    requestAuthorization: function requestAuthorization() {
+        let params = [
+            ["response_type", this.responseType],
+            ["client_id", this.consumerKey],
+            ["redirect_uri", this.completionURI],
+        ];
+        // The scope can be optional.
+        if (this.scope) {
+            params.push(["scope", this.scope]);
+        }
+
+        // Add extra parameters
+        params.push(...this.extraAuthParams);
+
+        // Now map the parameters to a string
+        params = params.map(([k,v]) => k + "=" + encodeURIComponent(v)).join("&");
+
+        this._browserRequest = {
+            account: this,
+            url: this.authURI + "?" + params,
+            _active: true,
+            iconURI: "",
+            cancelled: function() {
+                if (!this._active) {
+                    return;
+                }
+
+                this.account.finishAuthorizationRequest();
+                this.account.onAuthorizationFailed(Components.results.NS_ERROR_ABORT, '{ "error": "cancelled"}');
+            },
+
+            loaded: function (aWindow, aWebProgress) {
+                if (!this._active) {
+                    return;
+                }
+
+                this._listener = {
+                    window: aWindow,
+                    webProgress: aWebProgress,
+                    _parent: this.account,
+
+                    QueryInterface: XPCOMUtils.generateQI([Ci.nsIWebProgressListener,
+                                                           Ci.nsISupportsWeakReference]),
+
+                    _cleanUp: function() {
+                      this.webProgress.removeProgressListener(this);
+                      this.window.close();
+                      delete this.window;
+                    },
+
+                    _checkForRedirect: function(aURL) {
+                      if (aURL.indexOf(this._parent.completionURI) != 0)
+                        return;
+
+                      this._parent.finishAuthorizationRequest();
+                      this._parent.onAuthorizationReceived(aURL);
+                    },
+
+                    onStateChange: function(aWebProgress, aRequest, aStateFlags, aStatus) {
+                      const wpl = Ci.nsIWebProgressListener;
+                      if (aStateFlags & (wpl.STATE_START | wpl.STATE_IS_NETWORK))
+                        this._checkForRedirect(aRequest.name);
+                    },
+                    onLocationChange: function(aWebProgress, aRequest, aLocation) {
+                      this._checkForRedirect(aLocation.spec);
+                    },
+                    onProgressChange: function() {},
+                    onStatusChange: function() {},
+                    onSecurityChange: function() {},
+                };
+                aWebProgress.addProgressListener(this._listener,
+                                                 Ci.nsIWebProgress.NOTIFY_ALL);
+                aWindow.document.title = this.account.requestWindowTitle;
+            }
+        };
+
+        this.wrappedJSObject = this._browserRequest;
+        gConnecting[this.authURI] = true;
+        Services.ww.openWindow(null, this.requestWindowURI, null, this.requestWindowFeatures, this);
+    },
+    finishAuthorizationRequest: function() {
+        gConnecting[this.authURI] = false;
+        if (!("_browserRequest" in this)) {
+            return;
+        }
+
+        this._browserRequest._active = false;
+        if ("_listener" in this._browserRequest) {
+            this._browserRequest._listener._cleanUp();
+        }
+        delete this._browserRequest;
+    },
+
+    onAuthorizationReceived: function(aData) {
+        this.log.info("authorization received" + aData);
+        let results = parseURLData(aData);
+        if (this.responseType == "code" && results.code) {
+            this.requestAccessToken(results.code, OAuth2.CODE_AUTHORIZATION);
+        } else if (this.responseType == "token") {
+            this.onAccessTokenReceived(JSON.stringify(results));
+        }
+        else
+          this.onAuthorizationFailed(null, aData);
+    },
+
+    onAuthorizationFailed: function(aError, aData) {
+        this.connectFailureCallback(aData);
+    },
+
+    requestAccessToken: function requestAccessToken(aCode, aType) {
+        let params = [
+            ["client_id", this.consumerKey],
+            ["client_secret", this.consumerSecret],
+            ["grant_type", aType],
+        ];
+
+        if (aType == OAuth2.CODE_AUTHORIZATION) {
+            params.push(["code", aCode]);
+            params.push(["redirect_uri", this.completionURI]);
+        } else if (aType == OAuth2.CODE_REFRESH) {
+            params.push(["refresh_token", aCode]);
+        }
+
+        let options = {
+          postData: params,
+          onLoad: this.onAccessTokenReceived.bind(this),
+          onError: this.onAccessTokenFailed.bind(this)
+        }
+        httpRequest(this.tokenURI, options);
+    },
+
+    onAccessTokenFailed: function onAccessTokenFailed(aError, aData) {
+        if (aError != "offline") {
+            this.refreshToken = null;
+        }
+        this.connectFailureCallback(aData);
+    },
+
+    onAccessTokenReceived: function onRequestTokenReceived(aData) {
+        let result = JSON.parse(aData);
+
+        this.accessToken = result.access_token;
+        if ("refresh_token" in result) {
+            this.refreshToken = result.refresh_token;
+        }
+        if ("expires_in" in result) {
+            this.tokenExpires = (new Date()).getTime() + (result.expires_in * 1000);
+        } else {
+            this.tokenExpires = Number.MAX_VALUE;
+        }
+        this.tokenType = result.token_type;
+
+        this.connectSuccessCallback();
+    }
+};

mailnews/base/util/OAuth2Providers.jsm

@@ -0,0 +1,77 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Details of supported OAuth2 Providers.
+ */
+var EXPORTED_SYMBOLS = ["OAuth2Providers"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+// map of hostnames to [issuer, scope]
+var kHostnames = new Map([
+  ["imap.googlemail.com", ["accounts.google.com", "https://mail.google.com/"]],
+  ["smtp.googlemail.com", ["accounts.google.com", "https://mail.google.com/"]],
+  ["imap.gmail.com", ["accounts.google.com", "https://mail.google.com/"]],
+  ["smtp.gmail.com", ["accounts.google.com", "https://mail.google.com/"]],
+
+  ["imap.mail.ru", ["o2.mail.ru", "mail.imap"]],
+  ["smtp.mail.ru", ["o2.mail.ru", "mail.imap"]],
+]);
+
+// map of issuers to appKey, appSecret, authURI, tokenURI
+
+// For the moment, these details are hard-coded, since Google does not
+// provide dynamic client registration. Don't copy these values for your
+// own application--register it yourself. This code (and possibly even the
+// registration itself) will disappear when this is switched to dynamic
+// client registration.
+var kIssuers = new Map ([
+  ["accounts.google.com", [
+    '406964657835-aq8lmia8j95dhl1a2bvharmfk3t1hgqj.apps.googleusercontent.com',
+    'kSmqreRr0qwBWJgbf5Y-PjSU',
+    'https://accounts.google.com/o/oauth2/auth',
+    'https://www.googleapis.com/oauth2/v3/token'
+  ]],
+  ["o2.mail.ru", [
+    'thunderbird',
+    'I0dCAXrcaNFujaaY',
+    'https://o2.mail.ru/login',
+    'https://o2.mail.ru/token'
+  ]],
+]);
+
+/**
+ *  OAuth2Providers: Methods to lookup OAuth2 parameters for supported
+ *                   email providers.
+ */
+var OAuth2Providers = {
+
+  /**
+   * Map a hostname to the relevant issuer and scope.
+   *
+   * @param aHostname  String representing the url for an imap or smtp
+   *                   server (example "imap.googlemail.com").
+   *
+   * @returns          Array with [issuer, scope] for the hostname if found,
+   *                   else undefined. issuer is a string representing the
+   *                   organization, scope is an oauth parameter describing\
+   *                   the required access level.
+   */
+  getHostnameDetails: function (aHostname) { return kHostnames.get(aHostname);},
+
+  /**
+   * Map an issuer to OAuth2 account details.
+   *
+   * @param aIssuer    The organization issuing oauth2 parameters, example
+   *                   "accounts.google.com".
+   *
+   * @return           Array containing [appKey, appSecret, authURI, tokenURI]
+   *                   where appKey and appDetails are strings representing the
+   *                   account registered for Thunderbird with the organization,
+   *                   authURI and tokenURI are url strings representing
+   *                   endpoints to access OAuth2 authentication.
+   */
+  getIssuerDetails: function (aIssuer) { return kIssuers.get(aIssuer);}
+}

mailnews/base/util/ServiceList.h

@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// IWYU pragma: private, include "mozilla/mailnews/Services.h"
+
+MOZ_SERVICE(AbManager,         nsIAbManager,
+            "@mozilla.org/abmanager;1")
+MOZ_SERVICE(AccountManager,    nsIMsgAccountManager,
+            "@mozilla.org/messenger/account-manager;1")
+MOZ_SERVICE(ComposeService,    nsIMsgComposeService,
+            "@mozilla.org/messengercompose;1")
+MOZ_SERVICE(CopyService,       nsIMsgCopyService,
+            "@mozilla.org/messenger/messagecopyservice;1")
+MOZ_SERVICE(DBService,         nsIMsgDBService,
+            "@mozilla.org/msgDatabase/msgDBService;1")
+MOZ_SERVICE(FilterService,     nsIMsgFilterService,
+            "@mozilla.org/messenger/services/filters;1")
+MOZ_SERVICE(HeaderParser,      nsIMsgHeaderParser,
+            "@mozilla.org/messenger/headerparser;1")
+MOZ_SERVICE(ImapService,       nsIImapService,
+            "@mozilla.org/messenger/imapservice;1")
+MOZ_SERVICE(ImportService,     nsIImportService,
+            "@mozilla.org/import/import-service;1")
+MOZ_SERVICE(MailNotifyService, mozINewMailNotificationService,
+            "@mozilla.org/newMailNotificationService;1")
+MOZ_SERVICE(MailSession,       nsIMsgMailSession,
+            "@mozilla.org/messenger/services/session;1")
+MOZ_SERVICE(MimeConverter,     nsIMimeConverter,
+            "@mozilla.org/messenger/mimeconverter;1")
+MOZ_SERVICE(MFNService,        nsIMsgFolderNotificationService,
+            "@mozilla.org/messenger/msgnotificationservice;1")
+MOZ_SERVICE(NntpService,       nsINntpService,
+            "@mozilla.org/messenger/nntpservice;1")
+MOZ_SERVICE(Pop3Service,       nsIPop3Service,
+            "@mozilla.org/messenger/popservice;1")
+MOZ_SERVICE(SmtpService,       nsISmtpService,
+            "@mozilla.org/messengercompose/smtp;1")
+MOZ_SERVICE(TagService,        nsIMsgTagService,
+            "@mozilla.org/messenger/tagservice;1")

mailnews/base/util/Services.cpp

@@ -0,0 +1,106 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/mailnews/Services.h"
+
+#include "nsIObserverService.h"
+#include "nsIObserver.h"
+#include "nsServiceManagerUtils.h"
+
+// All of the includes for the services we initiate here
+#include "mozINewMailNotificationService.h"
+#include "nsIAbManager.h"
+#include "nsIImapService.h"
+#include "nsIImportService.h"
+#include "nsIMimeConverter.h"
+#include "nsIMsgAccountManager.h"
+#include "nsIMsgComposeService.h"
+#include "nsIMsgCopyService.h"
+#include "nsIMsgDatabase.h"
+#include "nsIMsgFilterService.h"
+#include "nsIMsgFolderNotificationService.h"
+#include "nsIMsgHeaderParser.h"
+#include "nsIMsgMailSession.h"
+#include "nsIMsgTagService.h"
+#include "nsINntpService.h"
+#include "nsIPop3Service.h"
+#include "nsISmtpService.h"
+
+namespace mozilla {
+namespace services {
+
+namespace {
+class ShutdownObserver final : public nsIObserver
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIOBSERVER
+
+  static void EnsureInitialized();
+private:
+  ~ShutdownObserver() {}
+
+  void ShutdownServices();
+  static ShutdownObserver *sShutdownObserver;
+  static bool sShuttingDown;
+};
+
+bool ShutdownObserver::sShuttingDown = false;
+ShutdownObserver *ShutdownObserver::sShutdownObserver = nullptr;
+}
+
+#define MOZ_SERVICE(NAME, TYPE, CONTRACT_ID) \
+  static TYPE *g##NAME = nullptr; \
+  already_AddRefed<TYPE> Get##NAME() \
+  { \
+    ShutdownObserver::EnsureInitialized(); \
+    if (!g##NAME) \
+    { \
+      nsCOMPtr<TYPE> os = do_GetService(CONTRACT_ID); \
+      os.forget(&g##NAME); \
+      MOZ_ASSERT(g##NAME, "This service is unexpectedly missing."); \
+    } \
+    nsCOMPtr<TYPE> ret = g##NAME; \
+    return ret.forget(); \
+  }
+#include "mozilla/mailnews/ServiceList.h"
+#undef MOZ_SERVICE
+
+NS_IMPL_ISUPPORTS(ShutdownObserver, nsIObserver)
+
+NS_IMETHODIMP ShutdownObserver::Observe(nsISupports *aSubject,
+    const char *aTopic, const char16_t *aData)
+{
+  if (!strcmp(aTopic, "xpcom-shutdown-threads"))
+    ShutdownServices();
+  return NS_OK;
+}
+
+void ShutdownObserver::EnsureInitialized()
+{
+  MOZ_ASSERT(!sShuttingDown, "It is illegal to use this code after shutdown!");
+  if (!sShutdownObserver)
+  {
+    sShutdownObserver = new ShutdownObserver;
+    sShutdownObserver->AddRef();
+    nsCOMPtr<nsIObserverService> obs(mozilla::services::GetObserverService());
+    MOZ_ASSERT(obs, "This should never be null");
+    obs->AddObserver(sShutdownObserver, "xpcom-shutdown-threads", false);
+  }
+}
+
+void ShutdownObserver::ShutdownServices()
+{
+  sShuttingDown = true;
+  MOZ_ASSERT(sShutdownObserver, "Shutting down twice?");
+  sShutdownObserver->Release();
+  sShutdownObserver = nullptr;
+#define MOZ_SERVICE(NAME, TYPE, CONTRACT_ID) NS_IF_RELEASE(g##NAME);
+#include "mozilla/mailnews/ServiceList.h"
+#undef MOZ_SERVICE
+}
+
+} // namespace services
+} // namespace mozilla

mailnews/base/util/Services.h

@@ -0,0 +1,26 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_mailnews_Services_h
+#define mozilla_mailnews_Services_h
+
+#include "mozilla/Services.h"
+
+#define MOZ_SERVICE(NAME, TYPE, SERVICE_CID) class TYPE;
+#include "mozilla/mailnews/ServiceList.h"
+#undef MOZ_SERVICE
+
+namespace mozilla {
+namespace services {
+
+#define MOZ_SERVICE(NAME, TYPE, SERVICE_CID) \
+  already_AddRefed<TYPE> Get##NAME();
+#include "ServiceList.h"
+#undef MOZ_SERVICE
+
+} // namespace services
+} // namespace mozilla
+
+#endif

mailnews/base/util/StringBundle.js

@@ -0,0 +1,189 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["StringBundle"];
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+/**
+ * A string bundle.
+ *
+ * This object presents two APIs: a deprecated one that is equivalent to the API
+ * for the stringbundle XBL binding, to make it easy to switch from that binding
+ * to this module, and a new one that is simpler and easier to use.
+ *
+ * The benefit of this module over the XBL binding is that it can also be used
+ * in JavaScript modules and components, not only in chrome JS.
+ *
+ * To use this module, import it, create a new instance of StringBundle,
+ * and then use the instance's |get| and |getAll| methods to retrieve strings
+ * (you can get both plain and formatted strings with |get|):
+ *
+ *   let strings =
+ *     new StringBundle("chrome://example/locale/strings.properties");
+ *   let foo = strings.get("foo");
+ *   let barFormatted = strings.get("bar", [arg1, arg2]);
+ *   for (let string of strings.getAll())
+ *     dump (string.key + " = " + string.value + "\n");
+ *
+ * @param url {String}
+ *        the URL of the string bundle
+ */
+function StringBundle(url) {
+  this.url = url;
+}
+
+StringBundle.prototype = {
+  /**
+   * the locale associated with the application
+   * @type nsILocale
+   * @private
+   */
+  get _appLocale() {
+    try {
+      return Services.locale.getApplicationLocale();
+    }
+    catch(ex) {
+      return null;
+    }
+  },
+
+  /**
+   * the wrapped nsIStringBundle
+   * @type nsIStringBundle
+   * @private
+   */
+  get _stringBundle() {
+    let stringBundle = Services.strings.createBundle(this.url, this._appLocale);
+    this.__defineGetter__("_stringBundle", () => stringBundle);
+    return this._stringBundle;
+  },
+
+
+  // the new API
+
+  /**
+   * the URL of the string bundle
+   * @type String
+   */
+  _url: null,
+  get url() {
+    return this._url;
+  },
+  set url(newVal) {
+    this._url = newVal;
+    delete this._stringBundle;
+  },
+
+  /**
+   * Get a string from the bundle.
+   *
+   * @param key {String}
+   *        the identifier of the string to get
+   * @param args {array} [optional]
+   *        an array of arguments that replace occurrences of %S in the string
+   *
+   * @returns {String} the value of the string
+   */
+  get: function(key, args) {
+    if (args)
+      return this.stringBundle.formatStringFromName(key, args, args.length);
+    else
+      return this.stringBundle.GetStringFromName(key);
+  },
+
+  /**
+   * Get all the strings in the bundle.
+   *
+   * @returns {Array}
+   *          an array of objects with key and value properties
+   */
+  getAll: function() {
+    let strings = [];
+
+    // FIXME: for performance, return an enumerable array that wraps the string
+    // bundle's nsISimpleEnumerator (does JavaScript already support this?).
+
+    let enumerator = this.stringBundle.getSimpleEnumeration();
+
+    while (enumerator.hasMoreElements()) {
+      // We could simply return the nsIPropertyElement objects, but I think
+      // it's better to return standard JS objects that behave as consumers
+      // expect JS objects to behave (f.e. you can modify them dynamically).
+      let string = enumerator.getNext()
+        .QueryInterface(Components.interfaces.nsIPropertyElement);
+      strings.push({ key: string.key, value: string.value });
+    }
+
+    return strings;
+  },
+
+
+  // the deprecated XBL binding-compatible API
+
+  /**
+   * the URL of the string bundle
+   * @deprecated because its name doesn't make sense outside of an XBL binding
+   * @type String
+   */
+  get src() {
+    return this.url;
+  },
+  set src(newVal) {
+    this.url = newVal;
+  },
+
+  /**
+   * the locale associated with the application
+   * @deprecated because it has never been used outside the XBL binding itself,
+   * and consumers should obtain it directly from the locale service anyway.
+   * @type nsILocale
+   */
+  get appLocale() {
+    return this._appLocale;
+  },
+
+  /**
+   * the wrapped nsIStringBundle
+   * @deprecated because this module should provide all necessary functionality
+   * @type nsIStringBundle
+   *
+   * If you do ever need to use this, let the authors of this module know why
+   * so they can surface functionality for your use case in the module itself
+   * and you don't have to access this underlying XPCOM component.
+   */
+  get stringBundle() {
+    return this._stringBundle;
+  },
+
+  /**
+   * Get a string from the bundle.
+   * @deprecated use |get| instead
+   *
+   * @param key {String}
+   *        the identifier of the string to get
+   *
+   * @returns {String}
+   *          the value of the string
+   */
+  getString: function(key) {
+    return this.get(key);
+  },
+
+  /**
+   * Get a formatted string from the bundle.
+   * @deprecated use |get| instead
+   *
+   * @param key {string}
+   *        the identifier of the string to get
+   * @param args {array}
+   *        an array of arguments that replace occurrences of %S in the string
+   *
+   * @returns {String}
+   *          the formatted value of the string
+   */
+  getFormattedString: function(key, args) {
+    return this.get(key, args);
+  }
+}

mailnews/base/util/errUtils.js

@@ -0,0 +1,309 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains helper methods for debugging -- things like logging
+ * exception objects, dumping DOM nodes, Events, and generic object dumps.
+ */
+
+this.EXPORTED_SYMBOLS = ["logObject", "logException", "logElement", "logEvent",
+                          "errorWithDebug"];
+
+/**
+ * Report on an object to stdout.
+ * @param aObj  the object to be dumped
+ * @param aName the name of the object, for informational purposes
+ */
+function logObject(aObj, aName) {
+  dump("Dumping Object: " + aName + "\n");
+  stringifier.dumpObj(aObj, aName);
+}
+
+/**
+ * Log an exception to stdout. This function should not be called in
+ * expected circumstances.
+ * @param aException  the exception to log
+ * @param [aRethrow]  set to true to rethrow the exception after logging
+ * @param [aMsg]      optional message to log
+ */
+function logException(aException, aRethrow, aMsg) {
+  stringifier.dumpException(aException, aMsg);
+
+  if (aMsg)
+    Components.utils.reportError(aMsg);
+  Components.utils.reportError(aException);
+
+  if (aRethrow)
+    throw aException;
+}
+
+/**
+ * Log an DOM element to stdout.
+ * @param aElement the DOM element to dump
+ */
+function logElement(aElement) {
+  stringifier.dumpDOM(aElement);
+}
+
+/**
+ * Log an DOM event to stdout.
+ * @param aEvent the DOM event object to dump
+ */
+function logEvent(aEvent) {
+  stringifier.dumpEvent(aEvent);
+}
+
+/**
+ * Dump the current stack and return an Error suitable for throwing.  We return
+ *  the new Error so that your code can use a "throw" statement which makes it
+ *  obvious to syntactic analysis that there is an exit occuring at that point.
+ *
+ * Example:
+ *   throw errorWithDebug("I did not expect this!");
+ *
+ * @param aString The message payload for the exception.
+ */
+function errorWithDebug(aString) {
+  dump("PROBLEM: " + aString + "\n");
+  dump("CURRENT STACK (and throwing):\n");
+  // skip this frame.
+  dump(stringifier.getStack(1));
+  return new Error(aString);
+}
+
+function Stringifier() {};
+
+Stringifier.prototype = {
+  dumpObj: function (o, name) {
+    this._reset();
+    this._append(this.objectTreeAsString(o, true, true, 0));
+    dump(this._asString());
+  },
+
+  dumpDOM: function(node, level, recursive) {
+    this._reset();
+    let s = this.DOMNodeAsString(node, level, recursive);
+    dump(s);
+  },
+
+  dumpEvent: function(event) {
+    dump(this.eventAsString(event));
+  },
+
+  dumpException: function(exc, message) {
+    dump(exc + "\n");
+    this._reset();
+    if (message)
+      this._append("Exception (" + message + ")\n");
+
+    this._append("-- Exception object --\n");
+    this._append(this.objectTreeAsString(exc));
+    if (exc.stack) {
+      this._append("-- Stack Trace --\n");
+      this._append(exc.stack); // skip dumpException and logException
+    }
+    dump(this._asString());
+  },
+
+  _reset: function() {
+    this._buffer = [];
+  },
+
+  _append: function(string) {
+    this._buffer.push(string);
+  },
+
+  _asString: function() {
+    let str = this._buffer.join('');
+    this._reset();
+    return str;
+  },
+
+  getStack: function(skipCount) {
+    if (!((typeof Components == "object") &&
+          (typeof Components.classes == "object")))
+      return "No stack trace available.";
+    if (typeof(skipCount) === undefined)
+      skipCount = 0;
+
+    let frame = Components.stack.caller;
+    let str = "<top>";
+
+    while (frame) {
+      if (skipCount > 0) {
+        // Skip this frame.
+        skipCount -= 1;
+      }
+      else {
+        // Include the data from this frame.
+        let name = frame.name ? frame.name : "[anonymous]";
+        str += "\n" + name + "@" + frame.filename + ':' + frame.lineNumber;
+      }
+      frame = frame.caller;
+    }
+    return str + "\n";
+  },
+
+  objectTreeAsString: function(o, recurse, compress, level) {
+    let s = "";
+    if (recurse === undefined)
+      recurse = 0;
+    if (level === undefined)
+      level = 0;
+    if (compress === undefined)
+      compress = true;
+    let pfx = "";
+
+    for (var junk = 0; junk < level; junk++)
+      pfx += (compress) ? "| " : "|  ";
+
+    let tee = (compress) ? "+ " : "+- ";
+
+    if (typeof(o) != "object") {
+      s += pfx + tee + " (" + typeof(o) + ") " + o + "\n";
+    }
+    else {
+      for (let i in o) {
+        try {
+          let t = typeof o[i];
+          switch (t) {
+            case "function":
+              let sfunc = String(o[i]).split("\n");
+              if (sfunc[2] == "    [native code]")
+                sfunc = "[native code]";
+              else
+                sfunc = sfunc.length + " lines";
+              s += pfx + tee + i + " (function) " + sfunc + "\n";
+              break;
+            case "object":
+              s += pfx + tee + i + " (object) " + o[i] + "\n";
+              if (!compress)
+                s += pfx + "|\n";
+              if ((i != "parent") && (recurse))
+                s += this.objectTreeAsString(o[i], recurse - 1,
+                                             compress, level + 1);
+              break;
+            case "string":
+              if (o[i].length > 200)
+                s += pfx + tee + i + " (" + t + ") " + o[i].length + " chars\n";
+              else
+                s += pfx + tee + i + " (" + t + ") '" + o[i] + "'\n";
+              break;
+            default:
+              s += pfx + tee + i + " (" + t + ") " + o[i] + "\n";
+          }
+        } catch (ex) {
+          s += pfx + tee + " (exception) " + ex + "\n";
+        }
+        if (!compress)
+          s += pfx + "|\n";
+      }
+    }
+    s += pfx + "*\n";
+    return s;
+  },
+
+  _repeatStr: function (str, aCount) {
+    let res = "";
+    while (--aCount >= 0)
+      res += str;
+    return res;
+  },
+
+  DOMNodeAsString: function(node, level, recursive) {
+    if (level === undefined)
+      level = 0
+    if (recursive === undefined)
+      recursive = true;
+    this._append(this._repeatStr(" ", 2*level) + "<" + node.nodeName + "\n");
+
+    if (node.nodeType == 3) {
+        this._append(this._repeatStr(" ", (2*level) + 4) + node.nodeValue + "'\n");
+    }
+    else {
+      if (node.attributes) {
+        for (let i = 0; i < node.attributes.length; i++) {
+          this._append(this._repeatStr(
+                         " ", (2*level) + 4) + node.attributes[i].nodeName +
+                         "='" + node.attributes[i].nodeValue + "'\n");
+        }
+      }
+      if (node.childNodes.length == 0) {
+        this._append(this._repeatStr(" ", (2*level)) + "/>\n");
+      }
+      else if (recursive) {
+        this._append(this._repeatStr(" ", (2*level)) + ">\n");
+        for (let i = 0; i < node.childNodes.length; i++) {
+          this._append(this.DOMNodeAsString(node.childNodes[i], level + 1));
+        }
+        this._append(this._repeatStr(" ", 2*level) + "</" + node.nodeName + ">\n");
+      }
+    }
+    return this._asString();
+  },
+
+  eventAsString: function (event) {
+    this._reset();
+    this._append("-EVENT --------------------------\n");
+    this._append("type:           " + event.type + "\n");
+    this._append("eventPhase:     " + event.eventPhase + "\n");
+    if ("charCode" in event) {
+      this._append("charCode: " + event.charCode + "\n");
+      if ("name" in event)
+        this._append("str(charCode):  '" + String.fromCharCode(event.charCode) + "'\n");
+    }
+    if (("target" in event) && event.target) {
+      this._append("target: " + event.target + "\n");
+      if ("nodeName" in event.target)
+        this._append("target.nodeName: " + event.target.nodeName + "\n");
+      if ("getAttribute" in event.target)
+        this._append("target.id: " + event.target.getAttribute("id") + "\n");
+    }
+    if (("currentTarget" in event) && event.currentTarget) {
+      this._append("currentTarget: " + event.currentTarget + "\n");
+      if ("nodeName" in event.currentTarget)
+        this._append("currentTarget.nodeName: "+ event.currentTarget.nodeName + "\n");
+      if ("getAttribute" in event.currentTarget)
+        this._append("currentTarget.id: "+ event.currentTarget.getAttribute("id") + "\n");
+    }
+    if (("originalTarget" in event) && event.originalTarget) {
+      this._append("originalTarget: " + event.originalTarget + "\n");
+      if ("nodeName" in event.originalTarget)
+        this._append("originalTarget.nodeName: "+ event.originalTarget.nodeName + "\n");
+      if ("getAttribute" in event.originalTarget)
+        this._append("originalTarget.id: "+ event.originalTarget.getAttribute("id") + "\n");
+    }
+    let names = [
+        "bubbles",
+        "cancelable",
+        "detail",
+        "button",
+        "keyCode",
+        "isChar",
+        "shiftKey",
+        "altKey",
+        "ctrlKey",
+        "metaKey",
+        "clientX",
+        "clientY",
+        "screenX",
+        "screenY",
+        "layerX",
+        "layerY",
+        "isTrusted",
+        "timeStamp",
+        "currentTargetXPath",
+        "targetXPath",
+        "originalTargetXPath"
+                ];
+    for (let i in names) {
+      if (names[i] in event)
+        this._append(names[i] + ": " + event[names[i]] + "\n");
+    }
+    this._append("-------------------------------------\n");
+    return this._asString();
+  }
+};
+
+var stringifier = new Stringifier();

mailnews/base/util/folderUtils.jsm

@@ -0,0 +1,234 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains helper methods for dealing with nsIMsgFolders.
+ */
+
+this.EXPORTED_SYMBOLS = ["getFolderProperties", "getSpecialFolderString",
+                          "getFolderFromUri", "allAccountsSorted",
+                          "getMostRecentFolders", "folderNameCompare"];
+
+Components.utils.import("resource:///modules/mailServices.js");
+Components.utils.import("resource:///modules/iteratorUtils.jsm");
+
+/**
+ * Returns a string representation of a folder's "special" type.
+ *
+ * @param aFolder  the nsIMsgFolder whose special type should be returned
+ */
+function getSpecialFolderString(aFolder) {
+  const nsMsgFolderFlags = Components.interfaces.nsMsgFolderFlags;
+  let flags = aFolder.flags;
+  if (flags & nsMsgFolderFlags.Inbox)
+    return "Inbox";
+  if (flags & nsMsgFolderFlags.Trash)
+    return "Trash";
+  if (flags & nsMsgFolderFlags.Queue)
+    return "Outbox";
+  if (flags & nsMsgFolderFlags.SentMail)
+    return "Sent";
+  if (flags & nsMsgFolderFlags.Drafts)
+    return "Drafts";
+  if (flags & nsMsgFolderFlags.Templates)
+    return "Templates";
+  if (flags & nsMsgFolderFlags.Junk)
+    return "Junk";
+  if (flags & nsMsgFolderFlags.Archive)
+    return "Archive";
+  if (flags & nsMsgFolderFlags.Virtual)
+    return "Virtual";
+  return "none";
+}
+
+/**
+ * This function is meant to be used with trees. It returns the property list
+ * for all of the common properties that css styling is based off of.
+ *
+ * @param nsIMsgFolder aFolder  the folder whose properties should be returned
+ *                               as a string
+ * @param bool aOpen            true if the folder is open/expanded
+ *
+ * @return         A string of the property names, delimited by space.
+ */
+function getFolderProperties(aFolder, aOpen) {
+  const nsIMsgFolder = Components.interfaces.nsIMsgFolder;
+  let properties = [];
+
+  properties.push("folderNameCol");
+
+  properties.push("serverType-" + aFolder.server.type);
+
+  // set the SpecialFolder attribute
+  properties.push("specialFolder-" + getSpecialFolderString(aFolder));
+
+  // Now set the biffState
+  switch (aFolder.biffState) {
+    case nsIMsgFolder.nsMsgBiffState_NewMail:
+      properties.push("biffState-NewMail");
+      break;
+    case nsIMsgFolder.nsMsgBiffState_NoMail:
+      properties.push("biffState-NoMail");
+      break;
+    default:
+      properties.push("biffState-UnknownMail");
+  }
+
+  properties.push("isSecure-" + aFolder.server.isSecure);
+
+  // A folder has new messages, or a closed folder or any subfolder has new messages.
+  if (aFolder.hasNewMessages ||
+      (!aOpen && aFolder.hasSubFolders && aFolder.hasFolderOrSubfolderNewMessages))
+    properties.push("newMessages-true");
+
+  if (aFolder.isServer) {
+    properties.push("isServer-true");
+  }
+  else
+  {
+    // We only set this if we're not a server
+    let shallowUnread = aFolder.getNumUnread(false);
+    if (shallowUnread > 0) {
+      properties.push("hasUnreadMessages-true");
+    }
+    else
+    {
+      // Make sure that shallowUnread isn't negative
+      shallowUnread = 0;
+    }
+    let deepUnread = aFolder.getNumUnread(true);
+    if (deepUnread - shallowUnread > 0)
+      properties.push("subfoldersHaveUnreadMessages-true");
+  }
+
+  properties.push("noSelect-" + aFolder.noSelect);
+  properties.push("imapShared-" + aFolder.imapShared);
+
+  return properties.join(" ");
+}
+
+/**
+ * Returns a folder for a particular uri
+ *
+ * @param aUri  the rdf uri of the folder to return
+ */
+function getFolderFromUri(aUri) {
+  const Cc = Components.classes;
+  const Ci = Components.interfaces;
+  return Cc["@mozilla.org/mail/folder-lookup;1"].
+         getService(Ci.nsIFolderLookupService).getFolderById(aUri);
+}
+
+/**
+ * Returns the sort order value based on the server type to be used for sorting.
+ * The servers (accounts) go in the following order:
+ * (0) default account, (1) other mail accounts, (2) Local Folders,
+ * (3) IM accounts, (4) RSS, (5) News, (9) others (no server)
+ * This ordering is encoded in the .sortOrder property of each server type.
+ *
+ * @param aServer  the server object to be tested
+ */
+function getServerSortOrder(aServer) {
+  // If there is no server sort this object to the end.
+  if (!aServer)
+    return 999999999;
+
+  // Otherwise get the server sort order from the Account manager.
+  return MailServices.accounts.getSortOrder(aServer);
+}
+
+/**
+ * Compares the passed in accounts according to their precedence.
+ */
+function compareAccounts(aAccount1, aAccount2) {
+  return getServerSortOrder(aAccount1.incomingServer)
+           - getServerSortOrder(aAccount2.incomingServer);
+}
+
+/**
+ * Returns a list of accounts sorted by server type.
+ *
+ * @param aExcludeIMAccounts  Remove IM accounts from the list?
+ */
+function allAccountsSorted(aExcludeIMAccounts) {
+  // Get the account list, and add the proper items.
+  let accountList = toArray(fixIterator(MailServices.accounts.accounts,
+                                        Components.interfaces.nsIMsgAccount));
+
+  // This is a HACK to work around bug 41133. If we have one of the
+  // dummy "news" accounts there, that account won't have an
+  // incomingServer attached to it, and everything will blow up.
+  accountList = accountList.filter(function hasServer(a) {
+    return a.incomingServer;
+  });
+
+  // Remove IM servers.
+  if (aExcludeIMAccounts) {
+    accountList = accountList.filter(function(a) {
+      return a.incomingServer.type != "im";
+    });
+  }
+
+  return accountList.sort(compareAccounts);
+}
+
+/**
+ * Returns the most recently used/modified folders from the passed in list.
+ *
+ * @param aFolderList       The array of nsIMsgFolders to search for recent folders.
+ * @param aMaxHits          How many folders to return.
+ * @param aTimeProperty     Which folder time property to use.
+ *                          Use "MRMTime" for most recently modified time.
+ *                          Use "MRUTime" for most recently used time.
+ */
+function getMostRecentFolders(aFolderList, aMaxHits, aTimeProperty) {
+  let recentFolders = [];
+
+  /**
+   * This sub-function will add a folder to the recentFolders array if it
+   * is among the aMaxHits most recent. If we exceed aMaxHits folders,
+   * it will pop the oldest folder, ensuring that we end up with the
+   * right number.
+   *
+   * @param aFolder  The folder to check for recency.
+   */
+  let oldestTime = 0;
+  function addIfRecent(aFolder) {
+    let time = 0;
+    try {
+      time = Number(aFolder.getStringProperty(aTimeProperty)) || 0;
+    } catch(e) {}
+    if (time <= oldestTime)
+      return;
+
+    if (recentFolders.length == aMaxHits) {
+      recentFolders.sort(function sort_folders_by_time(a, b) {
+                         return a.time < b.time; });
+      recentFolders.pop();
+      oldestTime = recentFolders[recentFolders.length - 1].time;
+    }
+    recentFolders.push({ folder: aFolder, time: time });
+  }
+
+  for (let folder of aFolderList) {
+    addIfRecent(folder);
+  }
+
+  return recentFolders.map(function (f) { return f.folder; });
+}
+
+/**
+ * A locale dependent comparison function to produce a case-insensitive sort order
+ * used to sort folder names.
+ * Returns positive number if aString1 > aString2, negative number if aString1 > aString2,
+ *         otherwise 0.
+ *
+ * @param aString1  first string to compare
+ * @param aString2  second string to compare
+ */
+function folderNameCompare(aString1, aString2) {
+  // TODO: improve this as described in bug 992651.
+  return aString1.toLocaleLowerCase()
+                 .localeCompare(aString2.toLocaleLowerCase());
+}

mailnews/base/util/hostnameUtils.jsm

@@ -0,0 +1,341 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Generic shared utility code for checking of IP and hostname validity.
+ */
+
+this.EXPORTED_SYMBOLS = [ "isLegalHostNameOrIP",
+                           "isLegalHostName",
+                           "isLegalIPv4Address",
+                           "isLegalIPv6Address",
+                           "isLegalIPAddress",
+                           "isLegalLocalIPAddress",
+                           "cleanUpHostName",
+                           "kMinPort",
+                           "kMaxPort" ];
+
+var kMinPort = 1;
+var kMaxPort = 65535;
+
+/**
+ * Check if aHostName is an IP address or a valid hostname.
+ *
+ * @param aHostName                The string to check for validity.
+ * @param aAllowExtendedIPFormats  Allow hex/octal formats in addition to decimal.
+ * @return  Unobscured host name if aHostName is valid.
+ *          Returns null if it's not.
+ */
+function isLegalHostNameOrIP(aHostName, aAllowExtendedIPFormats)
+{
+  /*
+   RFC 1123:
+   Whenever a user inputs the identity of an Internet host, it SHOULD
+   be possible to enter either (1) a host domain name or (2) an IP
+   address in dotted-decimal ("#.#.#.#") form.  The host SHOULD check
+   the string syntactically for a dotted-decimal number before
+   looking it up in the Domain Name System.
+  */
+
+  return isLegalIPAddress(aHostName, aAllowExtendedIPFormats) ||
+         isLegalHostName(aHostName);
+}
+
+/**
+ * Check if aHostName is a valid hostname.
+ *
+ * @return  The host name if it is valid.
+ *          Returns null if it's not.
+ */
+function isLegalHostName(aHostName)
+{
+  /*
+   RFC 952:
+   A "name" (Net, Host, Gateway, or Domain name) is a text string up
+   to 24 characters drawn from the alphabet (A-Z), digits (0-9), minus
+   sign (-), and period (.).  Note that periods are only allowed when
+   they serve to delimit components of "domain style names". (See
+   RFC-921, "Domain Name System Implementation Schedule", for
+   background).  No blank or space characters are permitted as part of a
+   name. No distinction is made between upper and lower case.  The first
+   character must be an alpha character.  The last character must not be
+   a minus sign or period.
+
+   RFC 1123:
+   The syntax of a legal Internet host name was specified in RFC-952
+   [DNS:4].  One aspect of host name syntax is hereby changed: the
+   restriction on the first character is relaxed to allow either a
+   letter or a digit.  Host software MUST support this more liberal
+   syntax.
+
+   Host software MUST handle host names of up to 63 characters and
+   SHOULD handle host names of up to 255 characters.
+
+   RFC 1034:
+   Relative names are either taken relative to a well known origin, or to a
+   list of domains used as a search list.  Relative names appear mostly at
+   the user interface, where their interpretation varies from
+   implementation to implementation, and in master files, where they are
+   relative to a single origin domain name.  The most common interpretation
+   uses the root "." as either the single origin or as one of the members
+   of the search list, so a multi-label relative name is often one where
+   the trailing dot has been omitted to save typing.
+
+   Since a complete domain name ends with the root label, this leads to
+   a printed form which ends in a dot.
+  */
+
+  const hostPattern = /^(([a-z0-9]|[a-z0-9][a-z0-9\-]{0,61}[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9\-]{0,61}[a-z0-9])\.?$/i;
+  return ((aHostName.length <= 255) && hostPattern.test(aHostName)) ? aHostName : null;
+}
+
+/**
+ * Check if aHostName is a valid IPv4 address.
+ *
+ * @param aHostName                The string to check for validity.
+ * @param aAllowExtendedIPFormats  If false, only IPv4 addresses in the common
+                                   decimal format (4 components, each up to 255)
+ *                                 will be accepted, no hex/octal formats.
+ * @return  Unobscured canonicalized address if aHostName is an IPv4 address.
+ *          Returns null if it's not.
+ */
+function isLegalIPv4Address(aHostName, aAllowExtendedIPFormats)
+{
+  // Scammers frequently obscure the IP address by encoding each component as
+  // decimal, octal, hex or in some cases a mix match of each. There can even
+  // be less than 4 components where the last number covers the missing components.
+  // See the test at mailnews/base/test/unit/test_hostnameUtils.js for possible
+  // combinations.
+
+  if (!aHostName)
+    return null;
+
+  // Break the IP address down into individual components.
+  let ipComponents = aHostName.split(".");
+  let componentCount = ipComponents.length;
+  if (componentCount > 4 || (componentCount < 4 && !aAllowExtendedIPFormats))
+    return null;
+
+  /**
+   * Checks validity of an IP address component.
+   *
+   * @param aValue  The component string.
+   * @param aWidth  How many components does this string cover.
+   * @return        The value of the component in decimal if it is valid.
+   *                Returns null if it's not.
+   */
+  const kPowersOf256 = [ 1, 256, 65536, 16777216, 4294967296 ];
+  function isLegalIPv4Component(aValue, aWidth) {
+    let component;
+    // Is the component decimal?
+    if (/^(0|([1-9][0-9]{0,9}))$/.test(aValue)) {
+      component = parseInt(aValue, 10);
+    } else if (aAllowExtendedIPFormats) {
+      // Is the component octal?
+      if (/^(0[0-7]{1,12})$/.test(aValue))
+        component = parseInt(aValue, 8);
+      // Is the component hex?
+      else if (/^(0x[0-9a-f]{1,8})$/i.test(aValue))
+        component = parseInt(aValue, 16);
+      else
+        return null;
+    } else {
+      return null;
+    }
+
+    // Make sure the component in not larger than the expected maximum.
+    if (component >= kPowersOf256[aWidth])
+      return null;
+
+    return component;
+  }
+
+  for (let i = 0; i < componentCount; i++) {
+    // If we are on the last supplied component but we do not have 4,
+    // the last one covers the remaining ones.
+    let componentWidth = (i == componentCount - 1 ? 4 - i : 1);
+    let componentValue = isLegalIPv4Component(ipComponents[i], componentWidth);
+    if (componentValue == null)
+      return null;
+
+    // If we have a component spanning multiple ones, split it.
+    for (let j = 0; j < componentWidth; j++) {
+      ipComponents[i + j] = (componentValue >> ((componentWidth - 1 - j) * 8)) & 255;
+    }
+  }
+
+  // First component of zero is not valid.
+  if (ipComponents[0] == 0)
+    return null;
+
+  return ipComponents.join(".");
+}
+
+/**
+ * Check if aHostName is a valid IPv6 address.
+ *
+ * @param aHostName  The string to check for validity.
+ * @return  Unobscured canonicalized address if aHostName is an IPv6 address.
+ *          Returns null if it's not.
+ */
+function isLegalIPv6Address(aHostName)
+{
+  if (!aHostName)
+    return null;
+
+  // Break the IP address down into individual components.
+  let ipComponents = aHostName.toLowerCase().split(":");
+
+  // Make sure there are at least 3 components.
+  if (ipComponents.length < 3)
+    return null;
+
+  let ipLength = ipComponents.length - 1;
+
+  // Take care if the last part is written in decimal using dots as separators.
+  let lastPart = isLegalIPv4Address(ipComponents[ipLength], false);
+  if (lastPart)
+  {
+    let lastPartComponents = lastPart.split(".");
+    // Convert it into standard IPv6 components.
+    ipComponents[ipLength] =
+      ((lastPartComponents[0] << 8) | lastPartComponents[1]).toString(16);
+    ipComponents[ipLength + 1] =
+      ((lastPartComponents[2] << 8) | lastPartComponents[3]).toString(16);
+  }
+
+  // Make sure that there is only one empty component.
+  let emptyIndex;
+  for (let i = 1; i < ipComponents.length - 1; i++)
+  {
+    if (ipComponents[i] == "")
+    {
+      // If we already found an empty component return null.
+      if (emptyIndex)
+        return null;
+
+      emptyIndex = i;
+    }
+  }
+
+  // If we found an empty component, extend it.
+  if (emptyIndex)
+  {
+    ipComponents[emptyIndex] = 0;
+
+    // Add components so we have a total of 8.
+    for (let count = ipComponents.length; count < 8; count++)
+      ipComponents.splice(emptyIndex, 0, 0);
+  }
+
+  // Make sure there are 8 components.
+  if (ipComponents.length != 8)
+    return null;
+
+  // Format all components to 4 character hex value.
+  for (let i = 0; i < ipComponents.length; i++)
+  {
+    if (ipComponents[i] == "")
+      ipComponents[i] = 0;
+
+    // Make sure the component is a number and it isn't larger than 0xffff.
+    if (/^[0-9a-f]{1,4}$/.test(ipComponents[i])) {
+      ipComponents[i] = parseInt(ipComponents[i], 16);
+      if (isNaN(ipComponents[i]) || ipComponents[i] > 0xffff)
+        return null;
+    }
+    else {
+      return null;
+    }
+
+    // Pad the component with 0:s.
+    ipComponents[i] = ("0000" + ipComponents[i].toString(16)).substr(-4);
+  }
+
+  // TODO: support Zone indices in Link-local addresses? Currently they are rejected.
+  // http://en.wikipedia.org/wiki/IPv6_address#Link-local_addresses_and_zone_indices
+
+  let hostName = ipComponents.join(":");
+  // Treat 0000:0000:0000:0000:0000:0000:0000:0000 as an invalid IPv6 address.
+  return (hostName != "0000:0000:0000:0000:0000:0000:0000:0000") ?
+          hostName : null;
+}
+
+/**
+ * Check if aHostName is a valid IP address (IPv4 or IPv6).
+ *
+ * @param aHostName                The string to check for validity.
+ * @param aAllowExtendedIPFormats  Allow hex/octal formats in addition to decimal.
+ * @return  Unobscured canonicalized IPv4 or IPv6 address if it is valid,
+ *          otherwise null.
+ */
+function isLegalIPAddress(aHostName, aAllowExtendedIPFormats)
+{
+  return isLegalIPv4Address(aHostName, aAllowExtendedIPFormats) ||
+         isLegalIPv6Address(aHostName);
+}
+
+/**
+ * Check if aIPAddress is a local or private IP address.
+ *
+ * @param aIPAddress  A valid IP address literal in canonical (unobscured) form.
+ * @return            True if it is a local/private IPv4 or IPv6 address,
+ *                    otherwise false.
+ *
+ * Note: if the passed in address is not in canonical (unobscured form),
+ *       the result may be wrong.
+ */
+function isLegalLocalIPAddress(aIPAddress)
+{
+  // IPv4 address?
+  let ipComponents = aIPAddress.split(".");
+  if (ipComponents.length == 4)
+  {
+     // Check if it's a local or private IPv4 address.
+    return ipComponents[0] == 10 ||
+           ipComponents[0] == 127 || // loopback address
+          (ipComponents[0] == 192 && ipComponents[1] == 168) ||
+          (ipComponents[0] == 169 && ipComponents[1] == 254) ||
+          (ipComponents[0] == 172 && ipComponents[1] >= 16 && ipComponents[1] < 32);
+  }
+
+  // IPv6 address?
+  ipComponents = aIPAddress.split(":");
+  if (ipComponents.length == 8)
+  {
+    // ::1/128 - localhost
+    if (ipComponents[0] == "0000" && ipComponents[1] == "0000" &&
+        ipComponents[2] == "0000" && ipComponents[3] == "0000" &&
+        ipComponents[4] == "0000" && ipComponents[5] == "0000" &&
+        ipComponents[6] == "0000" && ipComponents[7] == "0001")
+      return true;
+
+    // fe80::/10 - link local addresses
+    if (ipComponents[0] == "fe80")
+      return true;
+
+    // fc00::/7 - unique local addresses
+    if (ipComponents[0].startsWith("fc") || // usage has not been defined yet
+        ipComponents[0].startsWith("fd"))
+      return true;
+
+    return false;
+  }
+
+  return false;
+}
+
+/**
+ * Clean up the hostname or IP. Usually used to sanitize a value input by the user.
+ * It is usually applied before we know if the hostname is even valid.
+ *
+ * @param aHostName  The hostname or IP string to clean up.
+ */
+function cleanUpHostName(aHostName)
+{
+  // TODO: Bug 235312: if UTF8 string was input, convert to punycode using convertUTF8toACE()
+  // but bug 563172 needs resolving first.
+  return aHostName.trim();
+}

mailnews/base/util/iteratorUtils.jsm

@@ -0,0 +1,166 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains helper methods for dealing with XPCOM iterators (arrays
+ * and enumerators) in JS-friendly ways.
+ */
+
+this.EXPORTED_SYMBOLS = ["fixIterator", "toXPCOMArray", "toArray"];
+
+Components.utils.import("resource://gre/modules/Deprecated.jsm");
+
+var Ci = Components.interfaces;
+
+var JS_HAS_SYMBOLS = typeof Symbol === "function";
+var ITERATOR_SYMBOL = JS_HAS_SYMBOLS ? Symbol.iterator : "@@iterator";
+
+/**
+ * This function will take a number of objects and convert them to an array.
+ *
+ * Currently, we support the following objects:
+ *   Anything you can for (let x of aObj) on
+ *                (e.g. toArray(fixIterator(enum))[4],
+ *                 also a NodeList from element.childNodes)
+ *
+ * @param aObj        The object to convert
+ */
+function toArray(aObj) {
+  if (ITERATOR_SYMBOL in aObj) {
+    return Array.from(aObj);
+  }
+
+  // We got something unexpected, notify the caller loudly.
+  throw new Error("An unsupported object sent to toArray: " +
+                  (("toString" in aObj) ? aObj.toString() : aObj));
+}
+
+/**
+ * Given a JS array, JS iterator, or one of a variety of XPCOM collections or
+ * iterators, return a JS iterator suitable for use in a for...of expression.
+ *
+ * Currently, we support the following types of XPCOM iterators:
+ *   nsIArray
+ *   nsISupportsArray
+ *   nsISimpleEnumerator
+ *
+ * This intentionally does not support nsIEnumerator as it is obsolete and
+ * no longer used in the base code.
+ *
+ * Note that old-style JS iterators are explicitly not supported in this
+ * method, as they are going away. For a limited time, the resulting iterator
+ * can be used in a for...in loop, but this is a legacy compatibility shim that
+ * will not work forever. See bug 1098412.
+ *
+ *   @param aEnum  the enumerator to convert
+ *   @param aIface (optional) an interface to QI each object to prior to
+ *                 returning
+ *
+ *   @note This returns an object that can be used in 'for...of' loops.
+ *         Do not use 'for each...in'. 'for...in' may be used, but only as a
+ *         legacy feature.
+ *         This does *not* return an Array object. To create such an array, use
+ *         let array = toArray(fixIterator(xpcomEnumerator));
+ */
+function fixIterator(aEnum, aIface) {
+  // Minor internal details: to support both for (let x of fixIterator()) and
+  // for (let x in fixIterator()), we need to add in a __iterator__ kludge
+  // property. __iterator__ is to go away in bug 1098412; we could theoretically
+  // make it work beyond that by using Proxies, but that's far to go for
+  // something we want to get rid of anyways.
+  // Note that the new-style iterator uses Symbol.iterator to work, and anything
+  // that has Symbol.iterator works with for-of.
+  function makeDualIterator(newStyle) {
+    newStyle.__iterator__ = function() {
+      for (let item of newStyle)
+        yield item;
+    };
+    return newStyle;
+  }
+
+  // If the input is an array or something that sports Symbol.iterator, then
+  // the original input is sufficient to directly return. However, if we want
+  // to support the aIface parameter, we need to do a lazy version of Array.map.
+  if (Array.isArray(aEnum) || ITERATOR_SYMBOL in aEnum) {
+    if (!aIface) {
+      return makeDualIterator(aEnum);
+    } else {
+      return makeDualIterator((function*() {
+        for (let o of aEnum)
+          yield o.QueryInterface(aIface);
+      })());
+    }
+  }
+
+  let face = aIface || Ci.nsISupports;
+  // Figure out which kind of array object we have.
+  // First try nsIArray (covers nsIMutableArray too).
+  if (aEnum instanceof Ci.nsIArray) {
+    return makeDualIterator((function*() {
+      let count = aEnum.length;
+      for (let i = 0; i < count; i++)
+        yield aEnum.queryElementAt(i, face);
+    })());
+  }
+
+  // Try an nsISupportsArray.
+  // This object is deprecated, but we need to keep supporting it
+  // while anything in the base code (including mozilla-central) produces it.
+  if (aEnum instanceof Ci.nsISupportsArray) {
+    return makeDualIterator((function*() {
+      let count = aEnum.Count();
+      for (let i = 0; i < count; i++)
+        yield aEnum.QueryElementAt(i, face);
+    })());
+  }
+
+  // How about nsISimpleEnumerator? This one is nice and simple.
+  if (aEnum instanceof Ci.nsISimpleEnumerator) {
+    return makeDualIterator((function*() {
+      while (aEnum.hasMoreElements())
+        yield aEnum.getNext().QueryInterface(face);
+    })());
+  }
+
+  // We got something unexpected, notify the caller loudly.
+  throw new Error("An unsupported object sent to fixIterator: " +
+                  (("toString" in aEnum) ? aEnum.toString() : aEnum));
+}
+
+/**
+ * This function takes an Array object and returns an XPCOM array
+ * of the desired type. It will *not* work if you extend Array.prototype.
+ *
+ * @param aArray      the array (anything fixIterator supports) to convert to an XPCOM array
+ * @param aInterface  the type of XPCOM array to convert
+ *
+ * @note The returned array is *not* dynamically updated.  Changes made to the
+ *       JS array after a call to this function will not be reflected in the
+ *       XPCOM array.
+ */
+function toXPCOMArray(aArray, aInterface) {
+  if (aInterface.equals(Ci.nsISupportsArray)) {
+    Deprecated.warning("nsISupportsArray object is deprecated, avoid creating new ones.",
+                       "https://developer.mozilla.org/en-US/docs/XPCOM_array_guide");
+    let supportsArray = Components.classes["@mozilla.org/supports-array;1"]
+                                  .createInstance(Ci.nsISupportsArray);
+    for (let item of fixIterator(aArray)) {
+      supportsArray.AppendElement(item);
+    }
+    return supportsArray;
+  }
+
+  if (aInterface.equals(Ci.nsIMutableArray)) {
+    let mutableArray = Components.classes["@mozilla.org/array;1"]
+                                 .createInstance(Ci.nsIMutableArray);
+    for (let item of fixIterator(aArray)) {
+      mutableArray.appendElement(item, false);
+    }
+    return mutableArray;
+  }
+
+  // We got something unexpected, notify the caller loudly.
+  throw new Error("An unsupported interface requested from toXPCOMArray: " +
+                  aInterface);
+}

mailnews/base/util/jsTreeSelection.js

@@ -0,0 +1,654 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['JSTreeSelection'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+/**
+ * Partial nsITreeSelection implementation so that we can have nsMsgDBViews that
+ *  exist only for message display but do not need to be backed by a full
+ *  tree view widget.  This could also hopefully be used for more xpcshell unit
+ *  testing of the FolderDisplayWidget.  It might also be useful for creating
+ *  transient selections when right-click selection happens.
+ *
+ * Our current limitations:
+ * - We do not support any single selection modes.  This is mainly because we
+ *   need to look at the box object for that and we don't want to do it.
+ * - Timed selection.  Our expected consumers don't use it.
+ *
+ * Our current laziness:
+ * - We aren't very precise about invalidation when it would be potentially
+ *   complicated.  The theory is that if there is a tree box object, it's
+ *   probably native and the XPConnect overhead is probably a lot more than
+ *   any potential savings, at least for now when the tree display is
+ *   generally C++ XPCOM backed rather than JS XPCOM backed.  Also, we
+ *   aren't intended to actually be used with a real tree display; you should
+ *   be using the C++ object in that case!
+ *
+ * If documentation is omitted for something, it is because we have little to
+ *  add to the documentation of nsITreeSelection and really hope that our
+ *  documentation tool will copy-down that documentation.
+ *
+ * This implementation attempts to mimic the behavior of nsTreeSelection.  In
+ *  a few cases, this leads to potentially confusing actions.  I attempt to note
+ *  when we are doing this and why we do it.
+ *
+ * Unit test is in mailnews/base/util/test_jsTreeSelection.js
+ */
+function JSTreeSelection(aTreeBoxObject) {
+  this._treeBoxObject = aTreeBoxObject;
+
+  this._currentIndex = null;
+  this._shiftSelectPivot = null;
+  this._ranges = [];
+  this._count = 0;
+
+  this._selectEventsSuppressed = false;
+}
+JSTreeSelection.prototype = {
+  /**
+   * The current nsITreeBoxObject, appropriately QueryInterfaced.  May be null.
+   */
+  _treeBoxObject: null,
+
+  /**
+   * Where the focus rectangle (that little dotted thing) shows up.  Just
+   *  because something is focused does not mean it is actually selected.
+   */
+  _currentIndex: null,
+  /**
+   * The view index where the shift is anchored when it is not (conceptually)
+   *  the same as _currentIndex.  This only happens when you perform a ranged
+   *  selection.  In that case, the start index of the ranged selection becomes
+   *  the shift pivot (and the _currentIndex becomes the end of the ranged
+   *  selection.)
+   * It gets cleared whenever the selection changes and it's not the result of
+   *  a call to rangedSelect.
+   */
+  _shiftSelectPivot: null,
+  /**
+   * A list of [lowIndexInclusive, highIndexInclusive] non-overlapping,
+   *  non-adjacent 'tuples' sort in ascending order.
+   */
+  _ranges: [],
+  /**
+   * The number of currently selected rows.
+   */
+  _count: 0,
+
+  // In the case of the stand-alone message window, there's no tree, but
+  // there's a view.
+  _view: null,
+
+  get tree() {
+    return this._treeBoxObject;
+  },
+  set tree(aTreeBoxObject) {
+    this._treeBoxObject = aTreeBoxObject;
+  },
+
+  set view(aView) {
+    this._view = aView;
+  },
+  /**
+   * Although the nsITreeSelection documentation doesn't say, what this method
+   *  is supposed to do is check if the seltype attribute on the XUL tree is any
+   *  of the following: "single" (only a single row may be selected at a time,
+   *  "cell" (a single cell may be selected), or "text" (the row gets selected
+   *  but only the primary column shows up as selected.)
+   *
+   * @return false because we don't support single-selection.
+   */
+  get single() {
+    return false;
+  },
+
+  _updateCount: function JSTreeSelection__updateCount() {
+    this._count = 0;
+    for (let [low, high] of this._ranges) {
+      this._count += high - low + 1;
+    }
+  },
+
+  get count() {
+    return this._count;
+  },
+
+  isSelected: function JSTreeSelection_isSelected(aViewIndex) {
+    for (let [low, high] of this._ranges) {
+      if (aViewIndex >= low && aViewIndex <= high)
+        return true;
+    }
+    return false;
+  },
+
+  /**
+   * Select the given row.  It does nothing if that row was already selected.
+   */
+  select: function JSTreeSelection_select(aViewIndex) {
+    // current index will provide our effective shift pivot
+    this._shiftSelectPivot = null;
+    this.currentIndex = aViewIndex;
+
+    if (this._count == 1 && this._ranges[0][0] == aViewIndex)
+      return;
+
+    this._count = 1;
+    this._ranges = [[aViewIndex, aViewIndex]];
+
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+
+    this._fireSelectionChanged();
+  },
+
+  timedSelect: function JSTreeSelection_timedSelect(aIndex, aDelay) {
+    throw new Error("We do not implement timed selection.");
+  },
+
+  toggleSelect: function JSTreeSelection_toggleSelect(aIndex) {
+    this.currentIndex = aIndex;
+    // If nothing's selected, select aIndex
+    if (this._count == 0) {
+      this._count = 1;
+      this._ranges = [[aIndex, aIndex]];
+    }
+    else for (let [iTupe, [low, high]] of this._ranges.entries()) {
+      // below the range? add it to the existing range or create a new one
+      if (aIndex < low) {
+        this._count++;
+        // is it just below an existing range? (range fusion only happens in the
+        //  high case, not here.)
+        if (aIndex == low - 1) {
+          this._ranges[iTupe][0] = aIndex;
+          break;
+        }
+        // then it gets its own range
+        this._ranges.splice(iTupe, 0, [aIndex, aIndex]);
+        break;
+      }
+      // in the range?  will need to either nuke, shrink, or split the range to
+      //  remove it
+      if (aIndex >= low && aIndex <= high) {
+        this._count--;
+        // nuke
+        if (aIndex == low && aIndex == high)
+          this._ranges.splice(iTupe, 1);
+        // lower shrink
+        else if (aIndex == low)
+          this._ranges[iTupe][0] = aIndex + 1;
+        // upper shrink
+        else if (aIndex == high)
+          this._ranges[iTupe][1] = aIndex - 1;
+        // split
+        else
+          this._ranges.splice(iTupe, 1, [low, aIndex - 1], [aIndex + 1, high]);
+        break;
+      }
+      // just above the range?  fuse into the range, and possibly the next
+      //  range up.
+      if (aIndex == high + 1) {
+        this._count++;
+        // see if there is another range and there was just a gap of one between
+        //  the two ranges.
+        if ((iTupe + 1 < this._ranges.length) &&
+            (this._ranges[iTupe+1][0] == aIndex + 1)) {
+          // yes, merge the ranges
+          this._ranges.splice(iTupe, 2, [low, this._ranges[iTupe+1][1]]);
+          break;
+        }
+        // nope, no merge required, just update the range
+        this._ranges[iTupe][1] = aIndex;
+        break;
+      }
+      // otherwise we need to keep going
+    }
+
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidateRow(aIndex);
+    this._fireSelectionChanged();
+  },
+
+  /**
+   * @param aRangeStart If omitted, it implies a shift-selection is happening,
+   *     in which case we use _shiftSelectPivot as the start if we have it,
+   *     _currentIndex if we don't, and if we somehow didn't have a
+   *     _currentIndex, we use the range end.
+   * @param aRangeEnd Just the inclusive end of the range.
+   * @param aAugment Does this set a new selection or should it be merged with
+   *     the existing selection?
+   */
+  rangedSelect: function JSTreeSelection_rangedSelect(aRangeStart, aRangeEnd,
+                                                      aAugment) {
+    if (aRangeStart == -1) {
+      if (this._shiftSelectPivot != null)
+        aRangeStart = this._shiftSelectPivot;
+      else if (this._currentIndex != null)
+        aRangeStart = this._currentIndex;
+      else
+        aRangeStart = aRangeEnd;
+    }
+
+    this._shiftSelectPivot = aRangeStart;
+    this.currentIndex = aRangeEnd;
+
+    // enforce our ordering constraint for our ranges
+    if (aRangeStart > aRangeEnd)
+      [aRangeStart, aRangeEnd] = [aRangeEnd, aRangeStart];
+
+    // if we're not augmenting, then this is really easy.
+    if (!aAugment) {
+      this._count = aRangeEnd - aRangeStart + 1;
+      this._ranges = [[aRangeStart, aRangeEnd]];
+      if (this._treeBoxObject)
+        this._treeBoxObject.invalidate();
+      this._fireSelectionChanged();
+      return;
+    }
+
+    // Iterate over our existing set of ranges, finding the 'range' of ranges
+    //  that our new range overlaps or simply obviates.
+    // Overlap variables track blocks we need to keep some part of, Nuke
+    //  variables are for blocks that get spliced out.  For our purposes, all
+    //  overlap blocks are also nuke blocks.
+    let lowOverlap, lowNuke, highNuke, highOverlap;
+    // in case there is no overlap, also figure an insertionPoint
+    let insertionPoint = this._ranges.length; // default to the end
+    for (let [iTupe, [low, high]] of this._ranges.entries()) {
+      // If it's completely include the range, it should be nuked
+      if (aRangeStart <= low && aRangeEnd >= high) {
+        if (lowNuke == null) // only the first one we see is the low one
+          lowNuke = iTupe;
+        highNuke = iTupe;
+      }
+      // If our new range start is inside a range or is adjacent, it's overlap
+      if (aRangeStart >= low - 1 && aRangeStart <= high + 1 &&
+          lowOverlap == null)
+        lowOverlap = lowNuke = highNuke = iTupe;
+      // If our new range ends inside a range or is adjacent, it's overlap
+      if (aRangeEnd >= low - 1 && aRangeEnd <= high + 1) {
+        highOverlap = highNuke = iTupe;
+        if (lowNuke == null)
+          lowNuke = iTupe;
+      }
+
+      // we're done when no more overlap is possible
+      if (aRangeEnd < low) {
+        insertionPoint = iTupe;
+        break;
+      }
+    }
+
+    if (lowOverlap != null)
+      aRangeStart = Math.min(aRangeStart, this._ranges[lowOverlap][0]);
+    if (highOverlap != null)
+      aRangeEnd = Math.max(aRangeEnd, this._ranges[highOverlap][1]);
+    if (lowNuke != null)
+      this._ranges.splice(lowNuke, highNuke - lowNuke + 1,
+                          [aRangeStart, aRangeEnd]);
+    else
+      this._ranges.splice(insertionPoint, 0, [aRangeStart, aRangeEnd]);
+
+    this._updateCount();
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+    this._fireSelectionChanged();
+  },
+
+  /**
+   * This is basically RangedSelect but without insertion of a new range and we
+   *  don't need to worry about adjacency.
+   * Oddly, nsTreeSelection doesn't fire a selection changed event here...
+   */
+  clearRange: function JSTreeSelection_clearRange(aRangeStart, aRangeEnd) {
+    // Iterate over our existing set of ranges, finding the 'range' of ranges
+    //  that our clear range overlaps or simply obviates.
+    // Overlap variables track blocks we need to keep some part of, Nuke
+    //  variables are for blocks that get spliced out.  For our purposes, all
+    //  overlap blocks are also nuke blocks.
+    let lowOverlap, lowNuke, highNuke, highOverlap;
+    for (let [iTupe, [low, high]] of this._ranges.entries()) {
+      // If we completely include the range, it should be nuked
+      if (aRangeStart <= low && aRangeEnd >= high) {
+        if (lowNuke == null) // only the first one we see is the low one
+          lowNuke = iTupe;
+        highNuke = iTupe;
+      }
+      // If our new range start is inside a range, it's nuke and maybe overlap
+      if (aRangeStart >= low && aRangeStart <= high && lowNuke == null) {
+        lowNuke = highNuke = iTupe;
+        // it's only overlap if we don't match at the low end
+        if (aRangeStart > low)
+          lowOverlap = iTupe;
+      }
+      // If our new range ends inside a range, it's nuke and maybe overlap
+      if (aRangeEnd >= low && aRangeEnd <= high) {
+        highNuke = iTupe;
+        // it's only overlap if we don't match at the high end
+        if (aRangeEnd < high)
+          highOverlap = iTupe;
+        if (lowNuke == null)
+          lowNuke = iTupe;
+      }
+
+      // we're done when no more overlap is possible
+      if (aRangeEnd < low)
+        break;
+    }
+    // nothing to do since there's nothing to nuke
+    if (lowNuke == null)
+      return;
+    let args = [lowNuke, highNuke - lowNuke + 1];
+    if (lowOverlap != null)
+      args.push([this._ranges[lowOverlap][0], aRangeStart - 1]);
+    if (highOverlap != null)
+      args.push([aRangeEnd + 1, this._ranges[highOverlap][1]]);
+    this._ranges.splice.apply(this._ranges, args);
+
+    this._updateCount();
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+    // note! nsTreeSelection doesn't fire a selection changed event, so neither
+    //  do we, but it seems like we should
+  },
+
+  /**
+   * nsTreeSelection always fires a select notification when the range is
+   *  cleared, even if there is no effective chance in selection.
+   */
+  clearSelection: function JSTreeSelection_clearSelection() {
+    this._shiftSelectPivot = null;
+    this._count = 0;
+    this._ranges = [];
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+    this._fireSelectionChanged();
+  },
+
+  /**
+   * Not even nsTreeSelection implements this.
+   */
+  invertSelection: function JSTreeSelection_invertSelection() {
+    throw new Error("Who really was going to use this?");
+  },
+
+  /**
+   * Select all with no rows is a no-op, otherwise we select all and notify.
+   */
+  selectAll: function JSTreeSelection_selectAll() {
+    if (!this._view)
+      return;
+
+    let view = this._view;
+    let rowCount = view.rowCount;
+
+    // no-ops-ville
+    if (!rowCount)
+      return;
+
+    this._count = rowCount;
+    this._ranges = [[0, rowCount - 1]];
+
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+    this._fireSelectionChanged();
+  },
+
+  getRangeCount: function JSTreeSelection_getRangeCount() {
+    return this._ranges.length;
+  },
+  getRangeAt: function JSTreeSelection_getRangeAt(aRangeIndex, aMinObj,
+                                                  aMaxObj) {
+    if (aRangeIndex < 0 || aRangeIndex > this._ranges.length)
+      throw new Exception("Try a real range index next time.");
+    [aMinObj.value, aMaxObj.value] = this._ranges[aRangeIndex];
+  },
+
+  invalidateSelection: function JSTreeSelection_invalidateSelection() {
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+  },
+
+  /**
+   * Helper method to adjust points in the face of row additions/removal.
+   * @param aPoint The point, null if there isn't one, or an index otherwise.
+   * @param aDeltaAt The row at which the change is happening.
+   * @param aDelta The number of rows added if positive, or the (negative)
+   *     number of rows removed.
+   */
+  _adjustPoint: function JSTreeSelection__adjustPoint(aPoint, aDeltaAt,
+                                                      aDelta) {
+    // if there is no point, no change
+    if (aPoint == null)
+      return aPoint;
+    // if the point is before the change, no change
+    if (aPoint < aDeltaAt)
+      return aPoint;
+    // if it's a deletion and it includes the point, clear it
+    if (aDelta < 0 && aPoint >= aDeltaAt && (aPoint + aDelta < aDeltaAt))
+      return null;
+    // (else) the point is at/after the change, compensate
+    return aPoint + aDelta;
+  },
+  /**
+   * Find the index of the range, if any, that contains the given index, and
+   *  the index at which to insert a range if one does not exist.
+   *
+   * @return A tuple containing: 1) the index if there is one, null otherwise,
+   *     2) the index at which to insert a range that would contain the point.
+   */
+  _findRangeContainingRow:
+      function JSTreeSelection__findRangeContainingRow(aIndex) {
+    for (let [iTupe, [low, high]] of this._ranges.entries()) {
+      if (aIndex >= low && aIndex <= high)
+        return [iTupe, iTupe];
+      if (aIndex < low)
+        return [null, iTupe];
+    }
+    return [null, this._ranges.length];
+  },
+
+
+  /**
+   * When present, a list of calls made to adjustSelection.  See
+   *  |logAdjustSelectionForReplay| and |replayAdjustSelectionLog|.
+   */
+  _adjustSelectionLog: null,
+  /**
+   * Start logging calls to adjustSelection made against this instance.  You
+   *  would do this because you are replacing an existing selection object
+   *  with this instance for the purposes of creating a transient selection.
+   *  Of course, you want the original selection object to be up-to-date when
+   *  you go to put it back, so then you can call replayAdjustSelectionLog
+   *  with that selection object and everything will be peachy.
+   */
+  logAdjustSelectionForReplay:
+      function JSTreeSelection_logAdjustSelectionForReplay() {
+    this._adjustSelectionLog = [];
+  },
+  /**
+   * Stop logging calls to adjustSelection and replay the existing log against
+   *  aSelection.
+   *
+   * @param aSelection {nsITreeSelection}.
+   */
+  replayAdjustSelectionLog:
+      function JSTreeSelection_replayAdjustSelectionLog(aSelection) {
+    if (this._adjustSelectionLog.length) {
+      // Temporarily disable selection events because adjustSelection is going
+      //  to generate an event each time otherwise, and better 1 event than
+      //  many.
+      aSelection.selectEventsSuppressed = true;
+      for (let [index, count] of this._adjustSelectionLog) {
+        aSelection.adjustSelection(index, count);
+      }
+      aSelection.selectEventsSuppressed = false;
+    }
+    this._adjustSelectionLog = null;
+  },
+
+  adjustSelection: function JSTreeSelection_adjustSelection(aIndex, aCount) {
+    // nothing to do if there is no actual change
+    if (!aCount)
+      return;
+
+    if (this._adjustSelectionLog)
+      this._adjustSelectionLog.push([aIndex, aCount]);
+
+    // adjust our points
+    this._shiftSelectPivot = this._adjustPoint(this._shiftSelectPivot,
+                                               aIndex, aCount);
+    this._currentIndex = this._adjustPoint(this._currentIndex, aIndex, aCount);
+
+    // If we are adding rows, we want to split any range at aIndex and then
+    //  translate all of the ranges above that point up.
+    if (aCount > 0) {
+      let [iContain, iInsert] = this._findRangeContainingRow(aIndex);
+      if (iContain != null) {
+        let [low, high] = this._ranges[iContain];
+        // if it is the low value, we just want to shift the range entirely, so
+        //  do nothing (and keep iInsert pointing at it for translation)
+        // if it is not the low value, then there must be at least two values so
+        //  we should split it and only translate the new/upper block
+        if (aIndex != low) {
+          this._ranges.splice(iContain, 1, [low, aIndex - 1], [aIndex, high]);
+          iInsert++;
+        }
+      }
+      // now translate everything from iInsert on up
+      for (let iTrans = iInsert; iTrans < this._ranges.length; iTrans++) {
+        let [low, high] = this._ranges[iTrans];
+        this._ranges[iTrans] = [low + aCount, high + aCount];
+      }
+      // invalidate and fire selection change notice
+      if (this._treeBoxObject)
+        this._treeBoxObject.invalidate();
+      this._fireSelectionChanged();
+      return;
+    }
+
+    // If we are removing rows, we are basically clearing the range that is
+    //  getting deleted and translating everyone above the remaining point
+    //  downwards.  The one trick is we may have to merge the lowest translated
+    //  block.
+    let saveSuppress = this.selectEventsSuppressed;
+    this.selectEventsSuppressed = true;
+    this.clearRange(aIndex, aIndex - aCount - 1);
+    // translate
+    let iTrans = this._findRangeContainingRow(aIndex)[1];
+    for (; iTrans < this._ranges.length; iTrans++) {
+      let [low, high] = this._ranges[iTrans];
+      // for the first range, low may be below the index, in which case it
+      //  should not get translated
+      this._ranges[iTrans] = [(low >= aIndex) ? low + aCount : low,
+                              high + aCount];
+    }
+    // we may have to merge the lowest translated block because it may now be
+    //  adjacent to the previous block
+    if (iTrans > 0 && iTrans < this._ranges.length &&
+        this._ranges[iTrans-1][1] == this_ranges[iTrans][0]) {
+      this._ranges[iTrans-1][1] = this._ranges[iTrans][1];
+      this._ranges.splice(iTrans, 1);
+    }
+
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidate();
+    this.selectEventsSuppressed = saveSuppress;
+  },
+
+  get selectEventsSuppressed() {
+    return this._selectEventsSuppressed;
+  },
+  /**
+   * Control whether selection events are suppressed.  For consistency with
+   *  nsTreeSelection, we always generate a selection event when a value of
+   *  false is assigned, even if the value was already false.
+   */
+  set selectEventsSuppressed(aSuppress) {
+    this._selectEventsSuppressed = aSuppress;
+    if (!aSuppress)
+      this._fireSelectionChanged();
+  },
+
+  /**
+   * Note that we bypass any XUL "onselect" handler that may exist and go
+   *  straight to the view.  If you have a tree, you shouldn't be using us,
+   *  so this seems aboot right.
+   */
+  _fireSelectionChanged: function JSTreeSelection__fireSelectionChanged() {
+    // don't fire if we are suppressed; we will fire when un-suppressed
+    if (this.selectEventsSuppressed)
+      return;
+    let view;
+    if (this._treeBoxObject && this._treeBoxObject.view)
+      view = this._treeBoxObject.view;
+    else
+      view = this._view;
+
+    // We might not have a view if we're in the middle of setting up things
+    if (view) {
+      view = view.QueryInterface(Ci.nsITreeView);
+      view.selectionChanged();
+    }
+  },
+
+  get currentIndex() {
+    if (this._currentIndex == null)
+      return -1;
+    return this._currentIndex;
+  },
+  /**
+   * Sets the current index.  Other than updating the variable, this just
+   *  invalidates the tree row if we have a tree.
+   * The real selection object would send a DOM event we don't care about.
+   */
+  set currentIndex(aIndex) {
+    if (aIndex == this.currentIndex)
+      return;
+
+    this._currentIndex = (aIndex != -1) ? aIndex : null;
+    if (this._treeBoxObject)
+      this._treeBoxObject.invalidateRow(aIndex);
+  },
+
+  currentColumn: null,
+
+  get shiftSelectPivot() {
+    return this._shiftSelectPivot != null ? this._shiftSelectPivot : -1;
+  },
+
+  QueryInterface: XPCOMUtils.generateQI(
+    [Ci.nsITreeSelection]),
+
+  /*
+   * Functions after this aren't part of the nsITreeSelection interface.
+   */
+
+  /**
+   * Duplicate this selection on another nsITreeSelection. This is useful
+   * when you would like to discard this selection for a real tree selection.
+   * We assume that both selections are for the same tree.
+   *
+   * @note We don't transfer the correct shiftSelectPivot over.
+   * @note This will fire a selectionChanged event on the tree view.
+   *
+   * @param aSelection an nsITreeSelection to duplicate this selection onto
+   */
+  duplicateSelection: function JSTreeSelection_duplicateSelection(aSelection) {
+    aSelection.selectEventsSuppressed = true;
+    aSelection.clearSelection();
+    for (let [iTupe, [low, high]] of this._ranges.entries())
+      aSelection.rangedSelect(low, high, iTupe > 0);
+
+    aSelection.currentIndex = this.currentIndex;
+    // This will fire a selectionChanged event
+    aSelection.selectEventsSuppressed = false;
+  },
+};

mailnews/base/util/mailServices.js

@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["MailServices"];
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var MailServices = {};
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "mailSession",
+                                   "@mozilla.org/messenger/services/session;1",
+                                   "nsIMsgMailSession");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "accounts",
+                                   "@mozilla.org/messenger/account-manager;1",
+                                   "nsIMsgAccountManager");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "pop3",
+                                   "@mozilla.org/messenger/popservice;1",
+                                   "nsIPop3Service");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "imap",
+                                   "@mozilla.org/messenger/imapservice;1",
+                                   "nsIImapService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "nntp",
+                                   "@mozilla.org/messenger/nntpservice;1",
+                                   "nsINntpService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "smtp",
+                                   "@mozilla.org/messengercompose/smtp;1",
+                                   "nsISmtpService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "compose",
+                                   "@mozilla.org/messengercompose;1",
+                                   "nsIMsgComposeService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "ab",
+                                   "@mozilla.org/abmanager;1",
+                                   "nsIAbManager");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "copy",
+                                   "@mozilla.org/messenger/messagecopyservice;1",
+                                   "nsIMsgCopyService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "mfn",
+                                   "@mozilla.org/messenger/msgnotificationservice;1",
+                                   "nsIMsgFolderNotificationService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "headerParser",
+                                   "@mozilla.org/messenger/headerparser;1",
+                                   "nsIMsgHeaderParser");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "mimeConverter",
+                                   "@mozilla.org/messenger/mimeconverter;1",
+                                   "nsIMimeConverter");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "tags",
+                                   "@mozilla.org/messenger/tagservice;1",
+                                   "nsIMsgTagService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "filters",
+                                   "@mozilla.org/messenger/services/filters;1",
+                                   "nsIMsgFilterService");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "junk",
+                                   "@mozilla.org/messenger/filter-plugin;1?name=bayesianfilter",
+                                   "nsIJunkMailPlugin");
+
+XPCOMUtils.defineLazyServiceGetter(MailServices, "newMailNotification",
+                                   "@mozilla.org/newMailNotificationService;1",
+                                   "mozINewMailNotificationService");

mailnews/base/util/mailnewsMigrator.js

@@ -0,0 +1,203 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Migrate profile (prefs and other files) from older versions of Mailnews to
+ * current.
+ * This should be run at startup. It migrates as needed: each migration
+ * function should be written to be a no-op when the value is already migrated
+ * or was never used in the old version.
+ */
+
+this.EXPORTED_SYMBOLS = [ "migrateMailnews" ];
+
+Components.utils.import("resource:///modules/errUtils.js");
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource:///modules/mailServices.js");
+var Ci = Components.interfaces;
+var kServerPrefVersion = 1;
+var kSmtpPrefVersion = 1;
+var kABRemoteContentPrefVersion = 1;
+var kDefaultCharsetsPrefVersion = 1;
+
+function migrateMailnews()
+{
+  try {
+    MigrateServerAuthPref();
+  } catch (e) { logException(e); }
+
+  try {
+    MigrateABRemoteContentSettings();
+  } catch (e) { logException(e); }
+
+  try {
+    MigrateDefaultCharsets();
+  } catch (e) { logException(e); }
+}
+
+/**
+ * Migrates from pref useSecAuth to pref authMethod
+ */
+function MigrateServerAuthPref()
+{
+  try {
+    // comma-separated list of all accounts.
+    var accounts = Services.prefs.getCharPref("mail.accountmanager.accounts")
+        .split(",");
+    for (let i = 0; i < accounts.length; i++)
+    {
+      let accountKey = accounts[i]; // e.g. "account1"
+      if (!accountKey)
+        continue;
+      let serverKey = Services.prefs.getCharPref("mail.account." + accountKey +
+         ".server");
+      let server = "mail.server." + serverKey + ".";
+      if (Services.prefs.prefHasUserValue(server + "authMethod"))
+        continue;
+      if (!Services.prefs.prefHasUserValue(server + "useSecAuth") &&
+          !Services.prefs.prefHasUserValue(server + "auth_login"))
+        continue;
+      if (Services.prefs.prefHasUserValue(server + "migrated"))
+        continue;
+      // auth_login = false => old-style auth
+      // else: useSecAuth = true => "secure auth"
+      // else: cleartext pw
+      let auth_login = true;
+      let useSecAuth = false; // old default, default pref now removed
+      try {
+        auth_login = Services.prefs.getBoolPref(server + "auth_login");
+      } catch (e) {}
+      try {
+        useSecAuth = Services.prefs.getBoolPref(server + "useSecAuth");
+      } catch (e) {}
+
+      Services.prefs.setIntPref(server + "authMethod",
+          auth_login ? (useSecAuth ?
+                           Ci.nsMsgAuthMethod.secure :
+                           Ci.nsMsgAuthMethod.passwordCleartext) :
+                       Ci.nsMsgAuthMethod.old);
+      Services.prefs.setIntPref(server + "migrated", kServerPrefVersion);
+    }
+
+    // same again for SMTP servers
+    var smtpservers = Services.prefs.getCharPref("mail.smtpservers").split(",");
+    for (let i = 0; i < smtpservers.length; i++)
+    {
+      if (!smtpservers[i])
+        continue;
+      let server = "mail.smtpserver." + smtpservers[i] + ".";
+      if (Services.prefs.prefHasUserValue(server + "authMethod"))
+        continue;
+      if (!Services.prefs.prefHasUserValue(server + "useSecAuth") &&
+          !Services.prefs.prefHasUserValue(server + "auth_method"))
+        continue;
+      if (Services.prefs.prefHasUserValue(server + "migrated"))
+        continue;
+      // auth_method = 0 => no auth
+      // else: useSecAuth = true => "secure auth"
+      // else: cleartext pw
+      let auth_method = 1;
+      let useSecAuth = false;
+      try {
+        auth_method = Services.prefs.getIntPref(server + "auth_method");
+      } catch (e) {}
+      try {
+        useSecAuth = Services.prefs.getBoolPref(server + "useSecAuth");
+      } catch (e) {}
+
+      Services.prefs.setIntPref(server + "authMethod",
+          auth_method ? (useSecAuth ?
+                            Ci.nsMsgAuthMethod.secure :
+                            Ci.nsMsgAuthMethod.passwordCleartext) :
+                        Ci.nsMsgAuthMethod.none);
+      Services.prefs.setIntPref(server + "migrated", kSmtpPrefVersion);
+    }
+  } catch(e) { logException(e); }
+}
+
+/**
+ * The address book used to contain information about wheather to allow remote
+ * content for a given contact. Now we use the permission manager for that.
+ * Do a one-time migration for it.
+ */
+function MigrateABRemoteContentSettings()
+{
+  if (Services.prefs.prefHasUserValue("mail.ab_remote_content.migrated"))
+    return;
+
+  // Search through all of our local address books looking for a match.
+  let enumerator = MailServices.ab.directories;
+  while (enumerator.hasMoreElements())
+  {
+    let migrateAddress = function(aEmail) {
+      let uri = Services.io.newURI(
+        "chrome://messenger/content/email=" + aEmail, null, null);
+      Services.perms.add(uri, "image", Services.perms.ALLOW_ACTION);
+    }
+
+    let addrbook = enumerator.getNext()
+      .QueryInterface(Components.interfaces.nsIAbDirectory);
+    try {
+      // If it's a read-only book, don't try to find a card as we we could never
+      // have set the AllowRemoteContent property.
+      if (addrbook.readOnly)
+        continue;
+
+      let childCards = addrbook.childCards;
+      while (childCards.hasMoreElements())
+      {
+        let card = childCards.getNext()
+                             .QueryInterface(Components.interfaces.nsIAbCard);
+
+        if (card.getProperty("AllowRemoteContent", false) == false)
+          continue; // not allowed for this contact
+
+        if (card.primaryEmail)
+          migrateAddress(card.primaryEmail);
+
+        if (card.getProperty("SecondEmail", ""))
+          migrateAddress(card.getProperty("SecondEmail", ""));
+      }
+    } catch (e) { logException(e); }
+  }
+
+  Services.prefs.setIntPref("mail.ab_remote_content.migrated",
+                            kABRemoteContentPrefVersion);
+}
+
+/**
+ * If the default sending or viewing charset is one that is no longer available,
+ * change it back to the default.
+ */
+function MigrateDefaultCharsets()
+{
+  if (Services.prefs.prefHasUserValue("mail.default_charsets.migrated"))
+    return;
+
+  let charsetConvertManager = Components.classes['@mozilla.org/charset-converter-manager;1']
+    .getService(Components.interfaces.nsICharsetConverterManager);
+
+  let sendCharsetStr = Services.prefs.getComplexValue(
+      "mailnews.send_default_charset",
+      Components.interfaces.nsIPrefLocalizedString).data;
+
+  try {
+    charsetConvertManager.getCharsetTitle(sendCharsetStr);
+  } catch (e) {
+    Services.prefs.clearUserPref("mailnews.send_default_charset");
+  }
+
+  let viewCharsetStr = Services.prefs.getComplexValue(
+      "mailnews.view_default_charset",
+      Components.interfaces.nsIPrefLocalizedString).data;
+
+  try {
+    charsetConvertManager.getCharsetTitle(viewCharsetStr);
+  } catch (e) {
+    Services.prefs.clearUserPref("mailnews.view_default_charset");
+  }
+
+  Services.prefs.setIntPref("mail.default_charsets.migrated",
+                            kDefaultCharsetsPrefVersion);
+}

mailnews/base/util/moz.build

@@ -0,0 +1,78 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXPORTS += [
+    'nsImapMoveCoalescer.h',
+    'nsMsgCompressIStream.h',
+    'nsMsgCompressOStream.h',
+    'nsMsgDBFolder.h',
+    'nsMsgDBFolderAtomList.h',
+    'nsMsgI18N.h',
+    'nsMsgIdentity.h',
+    'nsMsgIncomingServer.h',
+    'nsMsgKeyArray.h',
+    'nsMsgKeySet.h',
+    'nsMsgLineBuffer.h',
+    'nsMsgMailNewsUrl.h',
+    'nsMsgProtocol.h',
+    'nsMsgReadStateTxn.h',
+    'nsMsgTxn.h',
+    'nsMsgUtils.h',
+]
+
+EXPORTS.mozilla.mailnews += [
+    'ServiceList.h',
+    'Services.h',
+]
+
+SOURCES += [
+    'nsImapMoveCoalescer.cpp',
+    'nsMsgCompressIStream.cpp',
+    'nsMsgCompressOStream.cpp',
+    'nsMsgDBFolder.cpp',
+    'nsMsgFileStream.cpp',
+    'nsMsgI18N.cpp',
+    'nsMsgIdentity.cpp',
+    'nsMsgIncomingServer.cpp',
+    'nsMsgKeyArray.cpp',
+    'nsMsgKeySet.cpp',
+    'nsMsgLineBuffer.cpp',
+    'nsMsgMailNewsUrl.cpp',
+    'nsMsgProtocol.cpp',
+    'nsMsgReadStateTxn.cpp',
+    'nsMsgTxn.cpp',
+    'nsMsgUtils.cpp',
+    'nsStopwatch.cpp',
+    'Services.cpp',
+]
+
+EXTRA_JS_MODULES += [
+    'ABQueryUtils.jsm',
+    'errUtils.js',
+    'folderUtils.jsm',
+    'hostnameUtils.jsm',
+    'IOUtils.js',
+    'iteratorUtils.jsm',
+    'jsTreeSelection.js',
+    'JXON.js',
+    'mailnewsMigrator.js',
+    'mailServices.js',
+    'msgDBCacheManager.js',
+    'OAuth2.jsm',
+    'OAuth2Providers.jsm',
+    'StringBundle.js',
+    'templateUtils.js',
+    'traceHelper.js',
+]
+
+LOCAL_INCLUDES += [
+  '/mozilla/netwerk/base'
+]
+
+FINAL_LIBRARY = 'mail'
+
+Library('msgbsutl_s')
+
+DEFINES['_IMPL_NS_MSG_BASE'] = True

mailnews/base/util/msgDBCacheManager.js

@@ -0,0 +1,175 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Message DB Cache manager
+ */
+
+/* :::::::: Constants and Helpers ::::::::::::::: */
+
+this.EXPORTED_SYMBOLS = ["msgDBCacheManager"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/mailServices.js");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource:///modules/gloda/log4moz.js");
+var log = Log4Moz.getConfiguredLogger("mailnews.database.dbcache");
+
+/**
+ */
+var DBCACHE_INTERVAL_DEFAULT_MS = 60000; // 1 minute
+
+/* :::::::: The Module ::::::::::::::: */
+
+var msgDBCacheManager =
+{
+  _initialized: false,
+
+  _msgDBCacheTimer: null,
+
+  _msgDBCacheTimerIntervalMS: DBCACHE_INTERVAL_DEFAULT_MS,
+
+  _dbService: null,
+
+  /**
+   * This is called on startup
+   */
+  init: function dbcachemgr_init()
+  {
+    if (this._initialized)
+      return;
+
+    this._dbService = Cc["@mozilla.org/msgDatabase/msgDBService;1"]
+                        .getService(Ci.nsIMsgDBService);
+
+    // we listen for "quit-application-granted" instead of
+    // "quit-application-requested" because other observers of the
+    // latter can cancel the shutdown.
+    Services.obs.addObserver(this, "quit-application-granted", false);
+
+    this.startPeriodicCheck();
+
+    this._initialized = true;
+  },
+
+/* ........ Timer Callback ................*/
+
+  _dbCacheCheckTimerCallback: function dbCache_CheckTimerCallback()
+  {
+    msgDBCacheManager.checkCachedDBs();
+  },
+
+/* ........ Observer Notification Handler ................*/
+
+  observe: function dbCache_observe(aSubject, aTopic, aData) {
+    switch (aTopic) {
+    // This is observed before any windows start unloading if something other
+    // than the last 3pane window closing requested the application be
+    // shutdown. For example, when the user quits via the file menu.
+    case "quit-application-granted":
+      Services.obs.removeObserver(this, "quit-application-granted");
+      this.stopPeriodicCheck();
+      break;
+    }
+  },
+
+/* ........ Public API ................*/
+
+  /**
+   * Stops db cache check
+   */
+  stopPeriodicCheck: function dbcache_stopPeriodicCheck()
+  {
+    if (this._dbCacheCheckTimer) {
+      this._dbCacheCheckTimer.cancel();
+
+      delete this._dbCacheCheckTimer;
+      this._dbCacheCheckTimer = null;
+    }
+  },
+
+  /**
+   * Starts periodic db cache check
+   */
+  startPeriodicCheck: function dbcache_startPeriodicCheck()
+  {
+    if (!this._dbCacheCheckTimer) {
+      this._dbCacheCheckTimer = Cc["@mozilla.org/timer;1"]
+                                   .createInstance(Ci.nsITimer);
+
+      this._dbCacheCheckTimer.initWithCallback(
+                                   this._dbCacheCheckTimerCallback,
+                                   this._msgDBCacheTimerIntervalMS,
+                                   Ci.nsITimer.TYPE_REPEATING_SLACK);
+    }
+  },
+
+  /**
+   * Checks if any DBs need to be closed due to inactivity or too many of them open.
+   */
+  checkCachedDBs: function()
+  {
+    let idleLimit = Services.prefs.getIntPref("mail.db.idle_limit");
+    let maxOpenDBs = Services.prefs.getIntPref("mail.db.max_open");
+
+    // db.lastUseTime below is in microseconds while Date.now and idleLimit pref
+    // is in milliseconds.
+    let closeThreshold = (Date.now() - idleLimit) * 1000;
+    let cachedDBs = this._dbService.openDBs;
+    log.info("Periodic check of cached folder databases (DBs), count=" + cachedDBs.length);
+    // Count databases that are already closed or get closed now due to inactivity.
+    let numClosing = 0;
+    // Count databases whose folder is open in a window.
+    let numOpenInWindow = 0;
+    let dbs = [];
+    for (let i = 0; i < cachedDBs.length; i++) {
+      let db = cachedDBs.queryElementAt(i, Ci.nsIMsgDatabase);
+      if (!db.folder.databaseOpen) {
+        // The DB isn't really open anymore.
+        log.debug("Skipping, DB not open for folder: " + db.folder.name);
+        numClosing++;
+        continue;
+      }
+
+      if (MailServices.mailSession.IsFolderOpenInWindow(db.folder)) {
+        // The folder is open in a window so this DB must not be closed.
+        log.debug("Skipping, DB open in window for folder: " + db.folder.name);
+        numOpenInWindow++;
+        continue;
+      }
+
+      if (db.lastUseTime < closeThreshold)
+      {
+        // DB open too log without activity.
+        log.debug("Closing expired DB for folder: " + db.folder.name);
+        db.folder.msgDatabase = null;
+        numClosing++;
+        continue;
+      }
+
+      // Database eligible for closing.
+      dbs.push(db);
+    }
+    log.info("DBs open in a window: " + numOpenInWindow + ", DBs open: " + dbs.length + ", DBs already closing: " + numClosing);
+    let dbsToClose = Math.max(dbs.length - Math.max(maxOpenDBs - numOpenInWindow, 0), 0);
+    if (dbsToClose > 0) {
+      // Close some DBs so that we do not have more than maxOpenDBs.
+      // However, we skipped DBs for folders that are open in a window
+      // so if there are so many windows open, it may be possible for
+      // more than maxOpenDBs folders to stay open after this loop.
+      log.info("Need to close " + dbsToClose + " more DBs");
+      // Order databases by lowest lastUseTime (oldest) at the end.
+      dbs.sort((a, b) => b.lastUseTime - a.lastUseTime);
+      while (dbsToClose > 0) {
+        let db = dbs.pop();
+        log.debug("Closing DB for folder: " + db.folder.name);
+        db.folder.msgDatabase = null;
+        dbsToClose--;
+      }
+    }
+  },
+};

mailnews/base/util/nsImapMoveCoalescer.cpp

@@ -0,0 +1,233 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsMsgImapCID.h"
+#include "nsImapMoveCoalescer.h"
+#include "nsIImapService.h"
+#include "nsIMsgCopyService.h"
+#include "nsMsgBaseCID.h"
+#include "nsIMsgFolder.h" // TO include biffState enum. Change to bool later...
+#include "nsMsgFolderFlags.h"
+#include "nsIMsgHdr.h"
+#include "nsIMsgImapMailFolder.h"
+#include "nsThreadUtils.h"
+#include "nsServiceManagerUtils.h"
+#include "nsIMutableArray.h"
+#include "nsArrayUtils.h"
+#include "nsComponentManagerUtils.h"
+#include "mozilla/ArrayUtils.h"
+
+NS_IMPL_ISUPPORTS(nsImapMoveCoalescer, nsIUrlListener)
+
+nsImapMoveCoalescer::nsImapMoveCoalescer(nsIMsgFolder *sourceFolder, nsIMsgWindow *msgWindow)
+{
+  m_sourceFolder = sourceFolder; 
+  m_msgWindow = msgWindow;
+  m_hasPendingMoves = false;
+}
+
+nsImapMoveCoalescer::~nsImapMoveCoalescer()
+{
+}
+
+nsresult nsImapMoveCoalescer::AddMove(nsIMsgFolder *folder, nsMsgKey key)
+{
+  m_hasPendingMoves = true;
+  int32_t folderIndex = m_destFolders.IndexOf(folder);
+  nsTArray<nsMsgKey> *keysToAdd = nullptr;
+
+  if (folderIndex >= 0)
+    keysToAdd = &(m_sourceKeyArrays[folderIndex]);
+  else
+  {
+    m_destFolders.AppendObject(folder);
+    keysToAdd = m_sourceKeyArrays.AppendElement();
+    if (!keysToAdd)
+      return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  if (!keysToAdd->Contains(key))
+    keysToAdd->AppendElement(key);
+
+  return NS_OK;
+}
+
+nsresult nsImapMoveCoalescer::PlaybackMoves(bool doNewMailNotification /* = false */)
+{
+  int32_t numFolders = m_destFolders.Count();
+  // Nothing to do, so don't change the member variables.
+  if (numFolders == 0)
+    return NS_OK;
+
+  nsresult rv = NS_OK;
+  m_hasPendingMoves = false;
+  m_doNewMailNotification = doNewMailNotification;
+  m_outstandingMoves = 0;
+
+  for (int32_t i = 0; i < numFolders; ++i)
+  {
+    // XXX TODO
+    // JUNK MAIL RELATED
+    // is this the right place to make sure dest folder exists
+    // (and has proper flags?), before we start copying?
+    nsCOMPtr <nsIMsgFolder> destFolder(m_destFolders[i]);
+    nsTArray<nsMsgKey>& keysToAdd = m_sourceKeyArrays[i];
+    int32_t numNewMessages = 0;
+    int32_t numKeysToAdd = keysToAdd.Length();
+    if (numKeysToAdd == 0)
+      continue;
+
+    nsCOMPtr<nsIMutableArray> messages(do_CreateInstance(NS_ARRAY_CONTRACTID));
+    for (uint32_t keyIndex = 0; keyIndex < keysToAdd.Length(); keyIndex++)
+    {
+      nsCOMPtr<nsIMsgDBHdr> mailHdr = nullptr;
+      rv = m_sourceFolder->GetMessageHeader(keysToAdd.ElementAt(keyIndex), getter_AddRefs(mailHdr));
+      if (NS_SUCCEEDED(rv) && mailHdr)
+      {
+        messages->AppendElement(mailHdr, false);
+        bool isRead = false;
+        mailHdr->GetIsRead(&isRead);
+        if (!isRead)
+          numNewMessages++;
+      }
+    }
+    uint32_t destFlags;
+    destFolder->GetFlags(&destFlags);
+    if (! (destFlags & nsMsgFolderFlags::Junk)) // don't set has new on junk folder
+    {
+      destFolder->SetNumNewMessages(numNewMessages);
+      if (numNewMessages > 0)
+        destFolder->SetHasNewMessages(true);
+    }
+    // adjust the new message count on the source folder
+    int32_t oldNewMessageCount = 0;
+    m_sourceFolder->GetNumNewMessages(false, &oldNewMessageCount);
+    if (oldNewMessageCount >= numKeysToAdd)
+      oldNewMessageCount -= numKeysToAdd;
+    else
+      oldNewMessageCount = 0;
+
+    m_sourceFolder->SetNumNewMessages(oldNewMessageCount);
+
+    nsCOMPtr <nsISupports> sourceSupports = do_QueryInterface(m_sourceFolder, &rv);
+    nsCOMPtr <nsIUrlListener> urlListener(do_QueryInterface(sourceSupports));
+
+    keysToAdd.Clear();
+    nsCOMPtr<nsIMsgCopyService> copySvc = do_GetService(NS_MSGCOPYSERVICE_CONTRACTID);
+    if (copySvc)
+    {
+      nsCOMPtr <nsIMsgCopyServiceListener> listener;
+      if (m_doNewMailNotification)
+      {
+        nsMoveCoalescerCopyListener *copyListener = new nsMoveCoalescerCopyListener(this, destFolder);
+        if (copyListener)
+          listener = do_QueryInterface(copyListener);
+      }
+      rv = copySvc->CopyMessages(m_sourceFolder, messages, destFolder, true,
+                                 listener, m_msgWindow, false /*allowUndo*/);
+      if (NS_SUCCEEDED(rv))
+        m_outstandingMoves++;
+    }
+  }
+  return rv;
+}
+
+NS_IMETHODIMP
+nsImapMoveCoalescer::OnStartRunningUrl(nsIURI *aUrl)
+{
+  NS_PRECONDITION(aUrl, "just a sanity check");
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsImapMoveCoalescer::OnStopRunningUrl(nsIURI *aUrl, nsresult aExitCode)
+{
+  m_outstandingMoves--;
+  if (m_doNewMailNotification && !m_outstandingMoves)
+  {
+    nsCOMPtr <nsIMsgImapMailFolder> imapFolder = do_QueryInterface(m_sourceFolder);
+    if (imapFolder)
+      imapFolder->NotifyIfNewMail();
+  }
+  return NS_OK;
+}
+
+nsTArray<nsMsgKey> *nsImapMoveCoalescer::GetKeyBucket(uint32_t keyArrayIndex)
+{
+  NS_ASSERTION(keyArrayIndex < MOZ_ARRAY_LENGTH(m_keyBuckets), "invalid index");
+
+  return keyArrayIndex < mozilla::ArrayLength(m_keyBuckets) ?
+           &(m_keyBuckets[keyArrayIndex]) : nullptr;
+}
+
+NS_IMPL_ISUPPORTS(nsMoveCoalescerCopyListener, nsIMsgCopyServiceListener)
+
+nsMoveCoalescerCopyListener::nsMoveCoalescerCopyListener(nsImapMoveCoalescer * coalescer, 
+                                                         nsIMsgFolder *destFolder)
+{
+  m_destFolder = destFolder;
+  m_coalescer = coalescer;
+}
+
+nsMoveCoalescerCopyListener::~nsMoveCoalescerCopyListener()
+{
+}
+
+NS_IMETHODIMP nsMoveCoalescerCopyListener::OnStartCopy()
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* void OnProgress (in uint32_t aProgress, in uint32_t aProgressMax); */
+NS_IMETHODIMP nsMoveCoalescerCopyListener::OnProgress(uint32_t aProgress, uint32_t aProgressMax)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* void SetMessageKey (in uint32_t aKey); */
+NS_IMETHODIMP nsMoveCoalescerCopyListener::SetMessageKey(uint32_t aKey)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* void GetMessageId (in nsACString aMessageId); */
+NS_IMETHODIMP nsMoveCoalescerCopyListener::GetMessageId(nsACString& messageId)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* void OnStopCopy (in nsresult aStatus); */
+NS_IMETHODIMP nsMoveCoalescerCopyListener::OnStopCopy(nsresult aStatus)
+{
+  nsresult rv = NS_OK;
+  if (NS_SUCCEEDED(aStatus))
+  {
+    // if the dest folder is imap, update it.
+    nsCOMPtr <nsIMsgImapMailFolder> imapFolder = do_QueryInterface(m_destFolder);
+    if (imapFolder)
+    {
+      uint32_t folderFlags;
+      m_destFolder->GetFlags(&folderFlags);
+      if (!(folderFlags & (nsMsgFolderFlags::Junk | nsMsgFolderFlags::Trash)))
+      {
+        nsCOMPtr<nsIImapService> imapService = do_GetService(NS_IMAPSERVICE_CONTRACTID, &rv); 
+        NS_ENSURE_SUCCESS(rv, rv);
+        nsCOMPtr <nsIURI> url;
+        nsCOMPtr <nsIUrlListener> listener = do_QueryInterface(m_coalescer);
+        rv = imapService->SelectFolder(m_destFolder, listener, nullptr, getter_AddRefs(url));
+      }
+    }
+    else // give junk filters a chance to run on new msgs in destination local folder
+    {
+      bool filtersRun;
+      m_destFolder->CallFilterPlugins(nullptr, &filtersRun);
+    }
+  }
+  return rv;
+}
+
+
+

mailnews/base/util/nsImapMoveCoalescer.h

@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsImapMoveCoalescer_H
+#define _nsImapMoveCoalescer_H
+
+#include "msgCore.h"
+#include "nsCOMArray.h"
+#include "nsIMsgWindow.h"
+#include "nsCOMPtr.h"
+#include "MailNewsTypes.h"
+#include "nsTArray.h"
+#include "nsIUrlListener.h"
+#include "nsIMsgCopyServiceListener.h"
+
+// imap move coalescer class - in order to keep nsImapMailFolder from growing like Topsy
+// Logically, we want to keep track of an nsTArray<nsMsgKey> per nsIMsgFolder, and then
+// be able to retrieve them one by one and play back the moves.
+// This utility class will be used by both the filter code and the offline playback code,
+// to avoid multiple moves to the same folder.
+
+class NS_MSG_BASE nsImapMoveCoalescer : public nsIUrlListener
+{
+public:
+  friend class nsMoveCoalescerCopyListener;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIURLLISTENER
+
+  nsImapMoveCoalescer(nsIMsgFolder *sourceFolder, nsIMsgWindow *msgWindow);
+
+  nsresult AddMove(nsIMsgFolder *folder, nsMsgKey key);
+  nsresult PlaybackMoves(bool doNewMailNotification = false);
+  // this lets the caller store keys in an arbitrary number of buckets. If the bucket
+  // for the passed in index doesn't exist, it will get created.
+  nsTArray<nsMsgKey> *GetKeyBucket(uint32_t keyArrayIndex);
+  nsIMsgWindow *GetMsgWindow() {return m_msgWindow;}
+  bool HasPendingMoves() {return m_hasPendingMoves;}
+protected:
+  virtual ~nsImapMoveCoalescer();
+  // m_sourceKeyArrays and m_destFolders are parallel arrays.
+  nsTArray<nsTArray<nsMsgKey> > m_sourceKeyArrays;
+  nsCOMArray<nsIMsgFolder> m_destFolders;
+  nsCOMPtr <nsIMsgWindow> m_msgWindow;
+  nsCOMPtr <nsIMsgFolder> m_sourceFolder;
+  bool m_doNewMailNotification;
+  bool m_hasPendingMoves;
+  nsTArray<nsMsgKey> m_keyBuckets[2];
+  int32_t m_outstandingMoves;
+};
+
+class nsMoveCoalescerCopyListener final : public nsIMsgCopyServiceListener
+{
+public:
+    nsMoveCoalescerCopyListener(nsImapMoveCoalescer * coalescer, nsIMsgFolder *destFolder);
+    NS_DECL_ISUPPORTS
+    NS_DECL_NSIMSGCOPYSERVICELISTENER
+
+    nsCOMPtr <nsIMsgFolder> m_destFolder;
+
+      nsImapMoveCoalescer *m_coalescer;
+    // when we get OnStopCopy, update the folder. When we've finished all the copies,
+    // send the biff notification.
+
+private:
+    ~nsMoveCoalescerCopyListener();
+};
+
+
+#endif // _nsImapMoveCoalescer_H
+

mailnews/base/util/nsMsgCompressIStream.cpp

@@ -0,0 +1,228 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsMsgCompressIStream.h"
+#include "prio.h"
+#include "prmem.h"
+#include "nsAlgorithm.h"
+#include <algorithm>
+
+#define BUFFER_SIZE 16384
+
+nsMsgCompressIStream::nsMsgCompressIStream() :
+  m_dataptr(nullptr),
+  m_dataleft(0),
+  m_inflateAgain(false)
+{
+}
+
+nsMsgCompressIStream::~nsMsgCompressIStream()
+{
+  Close();
+}
+
+NS_IMPL_ISUPPORTS(nsMsgCompressIStream, nsIInputStream,
+                              nsIAsyncInputStream)
+
+nsresult nsMsgCompressIStream::InitInputStream(nsIInputStream *rawStream)
+{
+  // protect against repeat calls
+  if (m_iStream)
+    return NS_ERROR_UNEXPECTED;
+
+  // allocate some memory for buffering
+  m_zbuf = mozilla::MakeUnique<char[]>(BUFFER_SIZE);
+  if (!m_zbuf)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  // allocate some memory for buffering
+  m_databuf = mozilla::MakeUnique<char[]>(BUFFER_SIZE);
+  if (!m_databuf)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  // set up zlib object
+  m_zstream.zalloc = Z_NULL;
+  m_zstream.zfree = Z_NULL;
+  m_zstream.opaque = Z_NULL;
+
+  // http://zlib.net/manual.html is rather silent on the topic, but 
+  // perl's Compress::Raw::Zlib manual says:
+  // -WindowBits
+  //  To compress an RFC 1951 data stream, set WindowBits to -MAX_WBITS.
+  if (inflateInit2(&m_zstream, -MAX_WBITS) != Z_OK)
+    return NS_ERROR_FAILURE;
+
+  m_iStream = rawStream;
+
+  return NS_OK;
+}
+
+nsresult nsMsgCompressIStream::DoInflation()
+{
+  // if there's something in the input buffer of the zstream, process it.
+  m_zstream.next_out = (Bytef *) m_databuf.get();
+  m_zstream.avail_out = BUFFER_SIZE;
+  int zr = inflate(&m_zstream, Z_SYNC_FLUSH);
+
+  // inflate() should normally be called until it returns 
+  // Z_STREAM_END or an error, and Z_BUF_ERROR just means
+  // unable to progress any further (possible if we filled
+  // an output buffer exactly)
+  if (zr == Z_BUF_ERROR || zr == Z_STREAM_END)
+    zr = Z_OK;
+
+  // otherwise it's an error
+  if (zr != Z_OK) 
+    return NS_ERROR_FAILURE;
+
+  // http://www.zlib.net/manual.html says:
+  // If inflate returns Z_OK and with zero avail_out, it must be called 
+  // again after making room in the output buffer because there might be
+  // more output pending. 
+  m_inflateAgain = m_zstream.avail_out ? false : true;
+
+  // set the pointer to the start of the buffer, and the count to how
+  // based on how many bytes are left unconsumed.
+  m_dataptr = m_databuf.get();
+  m_dataleft = BUFFER_SIZE - m_zstream.avail_out;
+
+  return NS_OK;
+}
+
+/* void close (); */
+NS_IMETHODIMP nsMsgCompressIStream::Close()
+{
+  return CloseWithStatus(NS_OK);
+}
+
+NS_IMETHODIMP nsMsgCompressIStream::CloseWithStatus(nsresult reason)
+{
+  nsresult rv = NS_OK;
+
+  if (m_iStream)
+  {
+    // pass the status through to our wrapped stream
+    nsCOMPtr <nsIAsyncInputStream> asyncInputStream = do_QueryInterface(m_iStream);
+    if (asyncInputStream)
+      rv = asyncInputStream->CloseWithStatus(reason);
+
+    // tidy up
+    m_iStream = nullptr;
+    inflateEnd(&m_zstream);
+  }
+
+  // clean up all the buffers
+  m_zbuf = nullptr;
+  m_databuf = nullptr;
+  m_dataptr = nullptr;
+  m_dataleft = 0;
+
+  return rv;
+}
+
+/* unsigned long long available (); */
+NS_IMETHODIMP nsMsgCompressIStream::Available(uint64_t *aResult)
+{
+  if (!m_iStream) 
+    return NS_BASE_STREAM_CLOSED;
+
+  // check if there's anything still in flight
+  if (!m_dataleft && m_inflateAgain)
+  {
+    nsresult rv = DoInflation();
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  // we'll be returning this many to the next read, guaranteed
+  if (m_dataleft)
+  {
+    *aResult = m_dataleft;
+    return NS_OK;
+  }
+
+  // this value isn't accurate, but will give a good true/false 
+  // indication for idle purposes, and next read will fill
+  // m_dataleft, so we'll have an accurate count for the next call.
+  return m_iStream->Available(aResult);
+}
+
+/* [noscript] unsigned long read (in charPtr aBuf, in unsigned long aCount); */
+NS_IMETHODIMP nsMsgCompressIStream::Read(char * aBuf, uint32_t aCount, uint32_t *aResult)
+{
+  if (!m_iStream) 
+  {
+    *aResult = 0;
+    return NS_OK;
+  }
+  
+  // There are two stages of buffering:
+  // * m_zbuf contains the compressed data from the remote server
+  // * m_databuf contains the uncompressed raw bytes for consumption
+  //   by the local client.
+  // 
+  // Each buffer will only be filled when the following buffers
+  // have been entirely consumed.
+  //
+  // m_dataptr and m_dataleft are respectively a pointer to the
+  // unconsumed portion of m_databuf and the number of bytes
+  // of uncompressed data remaining in m_databuf.
+  //
+  // both buffers have a maximum size of BUFFER_SIZE, so it is
+  // possible that multiple inflate passes will be required to
+  // consume all of m_zbuf.
+  while (!m_dataleft)
+  {
+    // get some more data if we don't already have any
+    if (!m_inflateAgain) 
+    {
+      uint32_t bytesRead;
+      nsresult rv = m_iStream->Read(m_zbuf.get(), (uint32_t)BUFFER_SIZE, &bytesRead);
+      NS_ENSURE_SUCCESS(rv, rv);
+      if (!bytesRead)
+        return NS_BASE_STREAM_CLOSED;
+      m_zstream.next_in = (Bytef *) m_zbuf.get();
+      m_zstream.avail_in = bytesRead;
+    }
+
+    nsresult rv = DoInflation();
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  *aResult = std::min(m_dataleft, aCount);
+
+  if (*aResult)
+  {
+    memcpy(aBuf, m_dataptr, *aResult);
+    m_dataptr += *aResult;
+    m_dataleft -= *aResult;
+  }
+
+  return NS_OK;
+}
+
+/* [noscript] unsigned long readSegments (in nsWriteSegmentFun aWriter, in voidPtr aClosure, in unsigned long aCount); */
+NS_IMETHODIMP nsMsgCompressIStream::ReadSegments(nsWriteSegmentFun aWriter, void * aClosure, uint32_t aCount, uint32_t *_retval)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP nsMsgCompressIStream::AsyncWait(nsIInputStreamCallback *callback, uint32_t flags, uint32_t amount, nsIEventTarget *target)
+{
+  if (!m_iStream)
+    return NS_BASE_STREAM_CLOSED;
+
+  nsCOMPtr <nsIAsyncInputStream> asyncInputStream = do_QueryInterface(m_iStream);
+  if (asyncInputStream)
+    return asyncInputStream->AsyncWait(callback, flags, amount, target);
+
+  return NS_OK;
+}
+
+/* boolean isNonBlocking (); */
+NS_IMETHODIMP nsMsgCompressIStream::IsNonBlocking(bool *aNonBlocking)
+{
+  *aNonBlocking = false;
+  return NS_OK;
+}
+

mailnews/base/util/nsMsgCompressIStream.h

@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsIAsyncInputStream.h"
+#include "nsIInputStream.h"
+#include "nsCOMPtr.h"
+#include "mozilla/UniquePtr.h"
+#include "zlib.h"
+
+class NS_MSG_BASE nsMsgCompressIStream final : public nsIAsyncInputStream
+{
+public:
+  nsMsgCompressIStream();
+
+  NS_DECL_THREADSAFE_ISUPPORTS
+    
+  NS_DECL_NSIINPUTSTREAM
+  NS_DECL_NSIASYNCINPUTSTREAM
+
+  nsresult InitInputStream(nsIInputStream *rawStream);
+
+protected:
+  ~nsMsgCompressIStream();
+  nsresult DoInflation();
+  nsCOMPtr<nsIInputStream> m_iStream;
+  mozilla::UniquePtr<char[]> m_zbuf;
+  mozilla::UniquePtr<char[]> m_databuf;
+  char *m_dataptr;
+  uint32_t m_dataleft;
+  bool m_inflateAgain;
+  z_stream m_zstream;
+};
+

mailnews/base/util/nsMsgCompressOStream.cpp

@@ -0,0 +1,145 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsMsgCompressOStream.h"
+#include "prio.h"
+#include "prmem.h"
+
+#define BUFFER_SIZE 16384
+
+nsMsgCompressOStream::nsMsgCompressOStream() :
+  m_zbuf(nullptr)
+{
+}
+
+nsMsgCompressOStream::~nsMsgCompressOStream()
+{
+  Close();
+}
+
+NS_IMPL_ISUPPORTS(nsMsgCompressOStream, nsIOutputStream)
+
+nsresult nsMsgCompressOStream::InitOutputStream(nsIOutputStream *rawStream)
+{
+  // protect against repeat calls
+  if (m_oStream)
+    return NS_ERROR_UNEXPECTED;
+
+  // allocate some memory for a buffer
+  m_zbuf = mozilla::MakeUnique<char[]>(BUFFER_SIZE);
+  if (!m_zbuf)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  // set up the zlib object
+  m_zstream.zalloc = Z_NULL;
+  m_zstream.zfree = Z_NULL;
+  m_zstream.opaque = Z_NULL;
+
+  // http://zlib.net/manual.html is rather silent on the topic, but
+  // perl's Compress::Raw::Zlib manual says:
+  // -WindowBits [...]
+  //  To compress an RFC 1951 data stream, set WindowBits to -MAX_WBITS.
+  if (deflateInit2(&m_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 
+                   -MAX_WBITS, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK)
+    return NS_ERROR_FAILURE;
+
+  m_oStream = rawStream;
+
+  return NS_OK;
+}
+
+/* void close (); */
+NS_IMETHODIMP nsMsgCompressOStream::Close()
+{
+  if (m_oStream)
+  {
+    m_oStream = nullptr;
+    deflateEnd(&m_zstream);
+  }
+  m_zbuf = nullptr;
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgCompressOStream::Write(const char *buf, uint32_t count, uint32_t *result)
+{
+  if (!m_oStream)
+    return NS_BASE_STREAM_CLOSED;
+
+  m_zstream.next_in = (Bytef *) buf;
+  m_zstream.avail_in = count;
+
+  // keep looping until the buffer doesn't get filled
+  do
+  {
+    m_zstream.next_out = (Bytef *) m_zbuf.get();
+    m_zstream.avail_out = BUFFER_SIZE;
+    // Using "Z_SYNC_FLUSH" may cause excess flushes if the calling
+    // code does a lot of small writes.  An option with the IMAP
+    // protocol is to check the buffer for "\n" at the end, but
+    // in the interests of keeping this generic, don't optimise
+    // yet.  An alternative is to require ->Flush always, but that
+    // is likely to break callers.
+    int zr = deflate(&m_zstream, Z_SYNC_FLUSH);
+    if (zr == Z_STREAM_END || zr == Z_BUF_ERROR)
+      zr = Z_OK; // not an error for our purposes
+    if (zr != Z_OK)
+      return NS_ERROR_FAILURE;
+
+    uint32_t out_size = BUFFER_SIZE - m_zstream.avail_out;
+    const char *out_buf = m_zbuf.get();
+
+    // push everything in the buffer before repeating
+    while (out_size)
+    {
+      uint32_t out_result;
+      nsresult rv = m_oStream->Write(out_buf, out_size, &out_result);
+      NS_ENSURE_SUCCESS(rv, rv);
+      if (!out_result)
+	return NS_BASE_STREAM_CLOSED;
+      out_size -= out_result;
+      out_buf += out_result;
+    }
+
+  // http://www.zlib.net/manual.html says:
+  // If deflate returns with avail_out == 0, this function must be 
+  // called again with the same value of the flush parameter and
+  // more output space (updated avail_out), until the flush is 
+  // complete (deflate returns with non-zero avail_out). 
+  } while (!m_zstream.avail_out);
+
+  *result = count;
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgCompressOStream::Flush(void)
+{
+  if (!m_oStream)
+    return NS_BASE_STREAM_CLOSED;
+  
+  return m_oStream->Flush();
+}
+
+NS_IMETHODIMP
+nsMsgCompressOStream::WriteFrom(nsIInputStream *inStr, uint32_t count, uint32_t *_retval)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+nsMsgCompressOStream::WriteSegments(nsReadSegmentFun reader, void * closure, uint32_t count, uint32_t *_retval)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* boolean isNonBlocking (); */
+NS_IMETHODIMP nsMsgCompressOStream::IsNonBlocking(bool *aNonBlocking)
+{
+  *aNonBlocking = false;
+  return NS_OK;
+}
+

mailnews/base/util/nsMsgCompressOStream.h

@@ -0,0 +1,28 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsIOutputStream.h"
+#include "nsCOMPtr.h"
+#include "mozilla/UniquePtr.h"
+#include "zlib.h"
+
+class NS_MSG_BASE nsMsgCompressOStream final : public nsIOutputStream
+{
+public:
+  nsMsgCompressOStream();
+
+  NS_DECL_THREADSAFE_ISUPPORTS
+
+  NS_DECL_NSIOUTPUTSTREAM
+
+  nsresult InitOutputStream(nsIOutputStream *rawStream);
+
+protected:
+  ~nsMsgCompressOStream();
+  nsCOMPtr<nsIOutputStream> m_oStream;
+  mozilla::UniquePtr<char[]> m_zbuf;
+  z_stream m_zstream;
+};
+

mailnews/base/util/nsMsgDBFolder.h

@@ -0,0 +1,297 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgDBFolder_h__
+#define nsMsgDBFolder_h__
+
+#include "mozilla/Attributes.h"
+#include "msgCore.h"
+#include "nsIMsgFolder.h" 
+#include "nsRDFResource.h"
+#include "nsIDBFolderInfo.h"
+#include "nsIMsgDatabase.h"
+#include "nsIMsgIncomingServer.h"
+#include "nsCOMPtr.h"
+#include "nsStaticAtom.h"
+#include "nsIDBChangeListener.h"
+#include "nsIMsgPluggableStore.h"
+#include "nsIURL.h"
+#include "nsIFile.h"
+#include "nsWeakReference.h"
+#include "nsIMsgFilterList.h"
+#include "nsIUrlListener.h"
+#include "nsIMsgHdr.h"
+#include "nsIOutputStream.h"
+#include "nsITransport.h"
+#include "nsIStringBundle.h"
+#include "nsTObserverArray.h"
+#include "nsCOMArray.h"
+#include "nsMsgKeySet.h"
+#include "nsMsgMessageFlags.h"
+#include "nsIMsgFilterPlugin.h"
+class nsIMsgFolderCacheElement;
+class nsICollation;
+class nsMsgKeySetU;
+
+ /* 
+  * nsMsgDBFolder
+  * class derived from nsMsgFolder for those folders that use an nsIMsgDatabase
+  */ 
+
+#undef IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+class NS_MSG_BASE nsMsgDBFolder: public nsRDFResource,
+                                 public nsSupportsWeakReference,
+                                 public nsIMsgFolder,
+                                 public nsIDBChangeListener,
+                                 public nsIUrlListener,
+                                 public nsIJunkMailClassificationListener,
+                                 public nsIMsgTraitClassificationListener
+{
+public: 
+  nsMsgDBFolder(void);
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_NSIMSGFOLDER
+  NS_DECL_NSIDBCHANGELISTENER
+  NS_DECL_NSIURLLISTENER
+  NS_DECL_NSIJUNKMAILCLASSIFICATIONLISTENER
+  NS_DECL_NSIMSGTRAITCLASSIFICATIONLISTENER
+  
+  NS_IMETHOD WriteToFolderCacheElem(nsIMsgFolderCacheElement *element);
+  NS_IMETHOD ReadFromFolderCacheElem(nsIMsgFolderCacheElement *element);
+
+  // nsRDFResource overrides
+  NS_IMETHOD Init(const char* aURI) override;
+
+  nsresult CreateDirectoryForFolder(nsIFile **result);
+  nsresult CreateBackupDirectory(nsIFile **result);
+  nsresult GetBackupSummaryFile(nsIFile **result, const nsACString& newName);
+  nsresult GetMsgPreviewTextFromStream(nsIMsgDBHdr *msgHdr, nsIInputStream *stream);
+  nsresult HandleAutoCompactEvent(nsIMsgWindow *aMsgWindow);
+protected:
+  virtual ~nsMsgDBFolder();
+  
+  virtual nsresult CreateBaseMessageURI(const nsACString& aURI);
+
+  void compressQuotesInMsgSnippet(const nsString& aMessageText, nsAString& aCompressedQuotesStr);
+  void decodeMsgSnippet(const nsACString& aEncodingType, bool aIsComplete, nsCString& aMsgSnippet);
+
+  // helper routine to parse the URI and update member variables
+  nsresult parseURI(bool needServer=false);
+  nsresult GetBaseStringBundle(nsIStringBundle **aBundle);
+  nsresult GetStringFromBundle(const char* msgName, nsString& aResult);
+  nsresult ThrowConfirmationPrompt(nsIMsgWindow *msgWindow, const nsAString& confirmString, bool *confirmed);
+  nsresult GetWarnFilterChanged(bool *aVal);
+  nsresult SetWarnFilterChanged(bool aVal);
+  nsresult CreateCollationKey(const nsString &aSource,  uint8_t **aKey, uint32_t *aLength);
+
+protected:
+  // all children will override this to create the right class of object.
+  virtual nsresult CreateChildFromURI(const nsCString &uri, nsIMsgFolder **folder) = 0;
+  virtual nsresult ReadDBFolderInfo(bool force);
+  virtual nsresult FlushToFolderCache();
+  virtual nsresult GetDatabase() = 0;
+  virtual nsresult SendFlagNotifications(nsIMsgDBHdr *item, uint32_t oldFlags, uint32_t newFlags);
+  nsresult CheckWithNewMessagesStatus(bool messageAdded);
+  void     UpdateNewMessages();
+  nsresult OnHdrAddedOrDeleted(nsIMsgDBHdr *hdrChanged, bool added);
+  nsresult CreateFileForDB(const nsAString& userLeafName, nsIFile *baseDir,
+                           nsIFile **dbFile);
+
+  nsresult GetFolderCacheKey(nsIFile **aFile, bool createDBIfMissing = false);
+  nsresult GetFolderCacheElemFromFile(nsIFile *file, nsIMsgFolderCacheElement **cacheElement);
+  nsresult AddDirectorySeparator(nsIFile *path);
+  nsresult CheckIfFolderExists(const nsAString& newFolderName, nsIMsgFolder *parentFolder, nsIMsgWindow *msgWindow);
+  bool     ConfirmAutoFolderRename(nsIMsgWindow *aMsgWindow,
+                                   const nsString& aOldName,
+                                   const nsString& aNewName);
+
+  // Returns true if: a) there is no need to prompt or b) the user is already
+  // logged in or c) the user logged in successfully.
+  static bool PromptForMasterPasswordIfNecessary();
+
+  // offline support methods.
+  nsresult StartNewOfflineMessage();
+  nsresult WriteStartOfNewLocalMessage();
+  nsresult EndNewOfflineMessage();
+  nsresult CompactOfflineStore(nsIMsgWindow *inWindow, nsIUrlListener *aUrlListener);
+  nsresult AutoCompact(nsIMsgWindow *aWindow);
+  // this is a helper routine that ignores whether nsMsgMessageFlags::Offline is set for the folder
+  nsresult MsgFitsDownloadCriteria(nsMsgKey msgKey, bool *result);
+  nsresult GetPromptPurgeThreshold(bool *aPrompt);
+  nsresult GetPurgeThreshold(int32_t *aThreshold);
+  nsresult ApplyRetentionSettings(bool deleteViaFolder);
+  bool     VerifyOfflineMessage(nsIMsgDBHdr *msgHdr, nsIInputStream *fileStream);
+  nsresult AddMarkAllReadUndoAction(nsIMsgWindow *msgWindow,
+                                    nsMsgKey *thoseMarked, uint32_t numMarked);
+
+  nsresult PerformBiffNotifications(void); // if there are new, non spam messages, do biff
+  nsresult CloseDBIfFolderNotOpen();
+
+  virtual nsresult SpamFilterClassifyMessage(const char *aURI, nsIMsgWindow *aMsgWindow, nsIJunkMailPlugin *aJunkMailPlugin);
+  virtual nsresult SpamFilterClassifyMessages(const char **aURIArray, uint32_t aURICount, nsIMsgWindow *aMsgWindow, nsIJunkMailPlugin *aJunkMailPlugin);
+  // Helper function for Move code to call to update the MRU and MRM time.
+  void    UpdateTimestamps(bool allowUndo);
+  void    SetMRUTime();
+  void    SetMRMTime();
+  /**
+   * Clear all processing flags, presumably because message keys are no longer
+   * valid.
+   */
+  void ClearProcessingFlags();
+
+  nsresult NotifyHdrsNotBeingClassified();
+
+  /**
+   * Produce an array of messages ordered like the input keys.
+   */
+  nsresult MessagesInKeyOrder(nsTArray<nsMsgKey> &aKeyArray,
+                              nsIMsgFolder *srcFolder,
+                              nsIMutableArray* messages);
+
+protected:
+  nsCOMPtr<nsIMsgDatabase> mDatabase;
+  nsCOMPtr<nsIMsgDatabase> mBackupDatabase;
+  nsCString mCharset;
+  bool mCharsetOverride;
+  bool mAddListener;
+  bool mNewMessages;
+  bool mGettingNewMessages;
+  nsMsgKey mLastMessageLoaded;
+
+  nsCOMPtr <nsIMsgDBHdr> m_offlineHeader;
+  int32_t m_numOfflineMsgLines;
+  int32_t m_bytesAddedToLocalMsg;
+  // this is currently used when we do a save as of an imap or news message..
+  nsCOMPtr<nsIOutputStream> m_tempMessageStream;
+
+  nsCOMPtr <nsIMsgRetentionSettings> m_retentionSettings;
+  nsCOMPtr <nsIMsgDownloadSettings> m_downloadSettings;
+  static NS_MSG_BASE_STATIC_MEMBER_(nsrefcnt) mInstanceCount;
+
+protected:
+  uint32_t mFlags;
+  nsWeakPtr mParent;     //This won't be refcounted for ownership reasons.
+  int32_t mNumUnreadMessages;        /* count of unread messages (-1 means unknown; -2 means unknown but we already tried to find out.) */
+  int32_t mNumTotalMessages;         /* count of existing messages. */
+  bool mNotifyCountChanges;
+  int64_t mExpungedBytes;
+  nsCOMArray<nsIMsgFolder> mSubFolders;
+  // This can't be refcounted due to ownsership issues
+  nsTObserverArray<nsIFolderListener*> mListeners;
+
+  bool mInitializedFromCache;
+  nsISupports *mSemaphoreHolder; // set when the folder is being written to
+                                 //Due to ownership issues, this won't be AddRef'd.
+
+  nsWeakPtr mServer;
+
+  // These values are used for tricking the front end into thinking that we have more 
+  // messages than are really in the DB.  This is usually after and IMAP message copy where
+  // we don't want to do an expensive select until the user actually opens that folder
+  int32_t mNumPendingUnreadMessages;
+  int32_t mNumPendingTotalMessages;
+  int64_t mFolderSize;
+
+  int32_t mNumNewBiffMessages;
+
+  // these are previous set of new msgs, which we might
+  // want to run junk controls on. This is in addition to "new" hdrs
+  // in the db, which might get cleared because the user clicked away
+  // from the folder.
+  nsTArray<nsMsgKey> m_saveNewMsgs;
+
+  // These are the set of new messages for a folder who has had
+  // its db closed, without the user reading the folder. This 
+  // happens with pop3 mail filtered to a different local folder.
+  nsTArray<nsMsgKey> m_newMsgs;
+
+  //
+  // stuff from the uri
+  //
+  bool mHaveParsedURI;        // is the URI completely parsed?
+  bool mIsServerIsValid;
+  bool mIsServer;
+  nsString mName;
+  nsCOMPtr<nsIFile> mPath;
+  nsCString mBaseMessageURI; //The uri with the message scheme
+
+  bool mInVFEditSearchScope ; // non persistant state used by the virtual folder UI
+
+  // static stuff for cross-instance objects like atoms
+  static NS_MSG_BASE_STATIC_MEMBER_(nsrefcnt) gInstanceCount;
+
+  static nsresult initializeStrings();
+  static nsresult createCollationKeyGenerator();
+
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedInboxName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedTrashName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedSentName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedDraftsName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedTemplatesName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedUnsentName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedJunkName;
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedArchivesName;
+
+  static NS_MSG_BASE_STATIC_MEMBER_(char16_t*) kLocalizedBrandShortName;
+  
+#define MSGDBFOLDER_ATOM(name_, value) static NS_MSG_BASE_STATIC_MEMBER_(nsIAtom*) name_;
+#include "nsMsgDBFolderAtomList.h"
+#undef MSGDBFOLDER_ATOM
+
+  static NS_MSG_BASE_STATIC_MEMBER_(nsICollation*) gCollationKeyGenerator;
+
+  // store of keys that have a processing flag set
+  struct
+  {
+    uint32_t bit;
+    nsMsgKeySetU* keys;
+  } mProcessingFlag[nsMsgProcessingFlags::NumberOfFlags];
+
+  // list of nsIMsgDBHdrs for messages to process post-bayes
+  nsCOMPtr<nsIMutableArray> mPostBayesMessagesToFilter;
+
+  /**
+   * The list of message keys that have been classified for msgsClassified
+   * batch notification purposes.  We add to this list in OnMessageClassified
+   * when we are told about a classified message (a URI is provided), and we
+   * notify for the list and clear it when we are told all the messages in
+   * the batch were classified (a URI is not provided).
+   */
+  nsTArray<nsMsgKey> mClassifiedMsgKeys;
+  // Is the current bayes filtering doing junk classification?
+  bool mBayesJunkClassifying;
+  // Is the current bayes filtering doing trait classification?
+  bool mBayesTraitClassifying;
+};
+
+// This class is a kludge to allow nsMsgKeySet to be used with uint32_t keys
+class nsMsgKeySetU
+{
+public:
+    // Creates an empty set.
+  static nsMsgKeySetU* Create();
+  ~nsMsgKeySetU();
+  // IsMember() returns whether the given key is a member of this set.
+  bool IsMember(nsMsgKey key);
+  // Add() adds the given key to the set.  (Returns 1 if a change was
+  // made, 0 if it was already there, and negative on error.)
+  int Add(nsMsgKey key);
+  // Remove() removes the given article from the set.
+  int Remove(nsMsgKey key);
+  // Add the keys in the set to aArray.
+  nsresult ToMsgKeyArray(nsTArray<nsMsgKey> &aArray);
+
+protected:
+  nsMsgKeySetU();
+  nsMsgKeySet* loKeySet;
+  nsMsgKeySet* hiKeySet;
+};
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_HIDDEN
+
+#endif

mailnews/base/util/nsMsgDBFolderAtomList.h

@@ -0,0 +1,26 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+MSGDBFOLDER_ATOM(kTotalUnreadMessagesAtom, "TotalUnreadMessages")
+MSGDBFOLDER_ATOM(kBiffStateAtom, "BiffState")
+MSGDBFOLDER_ATOM(kNewMailReceivedAtom, "NewMailReceived")
+MSGDBFOLDER_ATOM(kNewMessagesAtom, "NewMessages")
+MSGDBFOLDER_ATOM(kInVFEditSearchScopeAtom, "inVFEditSearchScope")
+MSGDBFOLDER_ATOM(kNumNewBiffMessagesAtom, "NumNewBiffMessages")
+MSGDBFOLDER_ATOM(kTotalMessagesAtom, "TotalMessages")
+MSGDBFOLDER_ATOM(kFolderSizeAtom, "FolderSize")
+MSGDBFOLDER_ATOM(kStatusAtom, "Status")
+MSGDBFOLDER_ATOM(kFlaggedAtom, "Flagged")
+MSGDBFOLDER_ATOM(kNameAtom, "Name")
+MSGDBFOLDER_ATOM(kSynchronizeAtom, "Synchronize")
+MSGDBFOLDER_ATOM(kOpenAtom, "open")
+MSGDBFOLDER_ATOM(kIsDeferred, "isDeferred")
+MSGDBFOLDER_ATOM(kKeywords, "Keywords")
+MSGDBFOLDER_ATOM(mFolderLoadedAtom, "FolderLoaded")
+MSGDBFOLDER_ATOM(mDeleteOrMoveMsgCompletedAtom, "DeleteOrMoveMsgCompleted")
+MSGDBFOLDER_ATOM(mDeleteOrMoveMsgFailedAtom, "DeleteOrMoveMsgFailed")
+MSGDBFOLDER_ATOM(mJunkStatusChangedAtom, "JunkStatusChanged")
+MSGDBFOLDER_ATOM(mFiltersAppliedAtom, "FiltersApplied")
+MSGDBFOLDER_ATOM(mFolderFlagAtom, "FolderFlag")

mailnews/base/util/nsMsgFileStream.cpp

@@ -0,0 +1,196 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsIFile.h"
+#include "nsMsgFileStream.h"
+#include "prerr.h"
+#include "prerror.h"
+
+/* From nsDebugImpl.cpp: */
+static nsresult
+ErrorAccordingToNSPR()
+{
+    PRErrorCode err = PR_GetError();
+    switch (err) {
+      case PR_OUT_OF_MEMORY_ERROR:              return NS_ERROR_OUT_OF_MEMORY;
+      case PR_WOULD_BLOCK_ERROR:                return NS_BASE_STREAM_WOULD_BLOCK;
+      case PR_FILE_NOT_FOUND_ERROR:             return NS_ERROR_FILE_NOT_FOUND;
+      case PR_READ_ONLY_FILESYSTEM_ERROR:       return NS_ERROR_FILE_READ_ONLY;
+      case PR_NOT_DIRECTORY_ERROR:              return NS_ERROR_FILE_NOT_DIRECTORY;
+      case PR_IS_DIRECTORY_ERROR:               return NS_ERROR_FILE_IS_DIRECTORY;
+      case PR_LOOP_ERROR:                       return NS_ERROR_FILE_UNRESOLVABLE_SYMLINK;
+      case PR_FILE_EXISTS_ERROR:                return NS_ERROR_FILE_ALREADY_EXISTS;
+      case PR_FILE_IS_LOCKED_ERROR:             return NS_ERROR_FILE_IS_LOCKED;
+      case PR_FILE_TOO_BIG_ERROR:               return NS_ERROR_FILE_TOO_BIG;
+      case PR_NO_DEVICE_SPACE_ERROR:            return NS_ERROR_FILE_NO_DEVICE_SPACE;
+      case PR_NAME_TOO_LONG_ERROR:              return NS_ERROR_FILE_NAME_TOO_LONG;
+      case PR_DIRECTORY_NOT_EMPTY_ERROR:        return NS_ERROR_FILE_DIR_NOT_EMPTY;
+      case PR_NO_ACCESS_RIGHTS_ERROR:           return NS_ERROR_FILE_ACCESS_DENIED;
+      default:                                  return NS_ERROR_FAILURE;
+    }
+}
+
+nsMsgFileStream::nsMsgFileStream() 
+{
+  mFileDesc = nullptr;
+  mSeekedToEnd = false;
+}
+
+nsMsgFileStream::~nsMsgFileStream()
+{
+  if (mFileDesc)
+    PR_Close(mFileDesc);
+}
+
+NS_IMPL_ISUPPORTS(nsMsgFileStream, nsIInputStream, nsIOutputStream, nsISeekableStream)
+
+nsresult nsMsgFileStream::InitWithFile(nsIFile *file)
+{
+  return file->OpenNSPRFileDesc(PR_RDWR|PR_CREATE_FILE, 0664, &mFileDesc);
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::Seek(int32_t whence, int64_t offset)
+{
+  if (mFileDesc == nullptr)
+    return NS_BASE_STREAM_CLOSED;
+
+  bool seekingToEnd = whence == PR_SEEK_END && offset == 0;
+  if (seekingToEnd && mSeekedToEnd)
+    return NS_OK;
+
+  int64_t cnt = PR_Seek64(mFileDesc, offset, (PRSeekWhence)whence);
+  if (cnt == int64_t(-1)) {
+    return ErrorAccordingToNSPR();
+  }
+
+  mSeekedToEnd = seekingToEnd;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::Tell(int64_t *result)
+{
+  if (mFileDesc == nullptr)
+    return NS_BASE_STREAM_CLOSED;
+  
+  int64_t cnt = PR_Seek64(mFileDesc, 0, PR_SEEK_CUR);
+  if (cnt == int64_t(-1)) {
+    return ErrorAccordingToNSPR();
+  }
+  *result = cnt;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::SetEOF()
+{
+  if (mFileDesc == nullptr)
+    return NS_BASE_STREAM_CLOSED;
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* void close (); */
+NS_IMETHODIMP nsMsgFileStream::Close()
+{
+  nsresult rv = NS_OK;
+  if (mFileDesc && (PR_Close(mFileDesc) == PR_FAILURE))
+        rv = NS_BASE_STREAM_OSERROR;
+    mFileDesc = nullptr;
+  return rv;
+}
+
+/* unsigned long long available (); */
+NS_IMETHODIMP nsMsgFileStream::Available(uint64_t *aResult)
+{
+  if (!mFileDesc) 
+    return NS_BASE_STREAM_CLOSED;
+  
+  int64_t avail = PR_Available64(mFileDesc);
+  if (avail == -1)
+    return ErrorAccordingToNSPR();
+
+  *aResult = avail;
+  return NS_OK;
+}
+
+/* [noscript] unsigned long read (in charPtr aBuf, in unsigned long aCount); */
+NS_IMETHODIMP nsMsgFileStream::Read(char * aBuf, uint32_t aCount, uint32_t *aResult)
+{
+  if (!mFileDesc) 
+  {
+    *aResult = 0;
+    return NS_OK;
+  }
+  
+  int32_t bytesRead = PR_Read(mFileDesc, aBuf, aCount);
+  if (bytesRead == -1)
+    return ErrorAccordingToNSPR();
+  
+  *aResult = bytesRead;
+  return NS_OK;
+}
+
+/* [noscript] unsigned long readSegments (in nsWriteSegmentFun aWriter, in voidPtr aClosure, in unsigned long aCount); */
+NS_IMETHODIMP nsMsgFileStream::ReadSegments(nsWriteSegmentFun aWriter, void * aClosure, uint32_t aCount, uint32_t *_retval)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/* boolean isNonBlocking (); */
+NS_IMETHODIMP nsMsgFileStream::IsNonBlocking(bool *aNonBlocking)
+{
+  *aNonBlocking = false;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::Write(const char *buf, uint32_t count, uint32_t *result)
+{
+  if (mFileDesc == nullptr)
+    return NS_BASE_STREAM_CLOSED;
+  
+  int32_t cnt = PR_Write(mFileDesc, buf, count);
+  if (cnt == -1) {
+    return ErrorAccordingToNSPR();
+  }
+  *result = cnt;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::Flush(void)
+{
+  if (mFileDesc == nullptr)
+    return NS_BASE_STREAM_CLOSED;
+  
+  int32_t cnt = PR_Sync(mFileDesc);
+  if (cnt == -1) 
+    return ErrorAccordingToNSPR();
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::WriteFrom(nsIInputStream *inStr, uint32_t count, uint32_t *_retval)
+{
+  NS_NOTREACHED("WriteFrom (see source comment)");
+  return NS_ERROR_NOT_IMPLEMENTED;
+  // File streams intentionally do not support this method.
+  // If you need something like this, then you should wrap
+  // the file stream using nsIBufferedOutputStream
+}
+
+NS_IMETHODIMP
+nsMsgFileStream::WriteSegments(nsReadSegmentFun reader, void * closure, uint32_t count, uint32_t *_retval)
+{
+  NS_NOTREACHED("WriteSegments (see source comment)");
+  return NS_ERROR_NOT_IMPLEMENTED;
+  // File streams intentionally do not support this method.
+  // If you need something like this, then you should wrap
+  // the file stream using nsIBufferedOutputStream
+}
+
+
+

mailnews/base/util/nsMsgFileStream.h

@@ -0,0 +1,33 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Attributes.h"
+#include "msgCore.h"
+#include "nsIInputStream.h"
+#include "nsIOutputStream.h"
+#include "nsISeekableStream.h"
+#include "prio.h"
+
+class nsMsgFileStream final : public nsIInputStream,
+                                  public nsIOutputStream,
+                                  public nsISeekableStream
+{
+public:
+  nsMsgFileStream();
+
+  NS_DECL_ISUPPORTS
+
+  NS_IMETHOD Available(uint64_t *_retval) override; 
+  NS_IMETHOD Read(char * aBuf, uint32_t aCount, uint32_t *_retval) override; 
+  NS_IMETHOD ReadSegments(nsWriteSegmentFun aWriter, void * aClosure, uint32_t aCount, uint32_t *_retval) override;
+  NS_DECL_NSIOUTPUTSTREAM
+  NS_DECL_NSISEEKABLESTREAM
+
+  nsresult InitWithFile(nsIFile *localFile);
+protected:
+  ~nsMsgFileStream();
+
+  PRFileDesc *mFileDesc;
+  bool mSeekedToEnd;
+};

mailnews/base/util/nsMsgI18N.cpp

@@ -0,0 +1,479 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// as does this
+#include "nsICharsetConverterManager.h"
+#include "nsIPlatformCharset.h"
+#include "nsIServiceManager.h"
+
+#include "nsISupports.h"
+#include "nsIPrefBranch.h"
+#include "nsIPrefService.h"
+#include "nsIMimeConverter.h"
+#include "nsMsgUtils.h"
+#include "nsMsgI18N.h"
+#include "nsMsgMimeCID.h"
+#include "nsILineInputStream.h"
+#include "nsMimeTypes.h"
+#include "nsISaveAsCharset.h"
+#include "nsStringGlue.h"
+#include "prmem.h"
+#include "plstr.h"
+#include "nsUTF8Utils.h"
+#include "nsNetUtil.h"
+#include "nsCRTGlue.h"
+#include "nsComponentManagerUtils.h"
+#include "nsUnicharUtils.h"
+#include "nsIFileStreams.h"
+//
+// International functions necessary for composition
+//
+
+nsresult nsMsgI18NConvertFromUnicode(const char* aCharset,
+                                     const nsString& inString,
+                                     nsACString& outString,
+                                     bool aIsCharsetCanonical,
+                                     bool aReportUencNoMapping)
+{
+  if (inString.IsEmpty()) {
+    outString.Truncate();
+    return NS_OK;
+  }
+  // Note: This will hide a possible error if the Unicode contains more than one
+  // charset, e.g. Latin1 + Japanese.
+  else if (!aReportUencNoMapping && (!*aCharset ||
+             !PL_strcasecmp(aCharset, "us-ascii") ||
+             !PL_strcasecmp(aCharset, "ISO-8859-1"))) {
+    LossyCopyUTF16toASCII(inString, outString);
+    return NS_OK;
+  }
+  else if (!PL_strcasecmp(aCharset, "UTF-8")) {
+    CopyUTF16toUTF8(inString, outString);
+    return NS_OK;
+  }
+
+  nsresult rv;
+  nsCOMPtr <nsICharsetConverterManager> ccm = do_GetService(NS_CHARSETCONVERTERMANAGER_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsCOMPtr <nsIUnicodeEncoder> encoder;
+
+  // get an unicode converter
+  if (aIsCharsetCanonical)  // optimize for modified UTF-7 used by IMAP
+    rv = ccm->GetUnicodeEncoderRaw(aCharset, getter_AddRefs(encoder));
+  else
+    rv = ccm->GetUnicodeEncoder(aCharset, getter_AddRefs(encoder));
+  NS_ENSURE_SUCCESS(rv, rv);
+  // Must set behavior to kOnError_Signal if we want to receive the
+  // NS_ERROR_UENC_NOMAPPING signal, should it occur.
+  int32_t behavior = aReportUencNoMapping ? nsIUnicodeEncoder::kOnError_Signal:
+    nsIUnicodeEncoder::kOnError_Replace;
+  rv = encoder->SetOutputErrorBehavior(behavior, nullptr, '?');
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  const char16_t *originalSrcPtr = inString.get();
+  const char16_t *currentSrcPtr = originalSrcPtr;
+  int32_t originalUnicharLength = inString.Length();
+  int32_t srcLength;
+  int32_t dstLength;
+  char localbuf[512+10]; // We have seen cases were the buffer was overrun
+                         // by two (!!) bytes (Bug 1255863).
+                         // So give it ten bytes more for now to avoid a crash.
+  int32_t consumedLen = 0;
+
+  bool mappingFailure = false;
+  outString.Truncate();
+  // convert
+  while (consumedLen < originalUnicharLength) {
+    srcLength = originalUnicharLength - consumedLen;  
+    dstLength = 512;
+    rv = encoder->Convert(currentSrcPtr, &srcLength, localbuf, &dstLength);
+#ifdef DEBUG
+    if (dstLength > 512) {
+      char warning[100];
+      sprintf(warning, "encoder->Convert() returned %d bytes. Limit = 512", dstLength);
+      NS_WARNING(warning);
+    }
+#endif
+    if (rv == NS_ERROR_UENC_NOMAPPING) {
+      mappingFailure = true;
+    }
+    if (NS_FAILED(rv) || dstLength == 0)
+      break;
+    outString.Append(localbuf, dstLength);
+
+    currentSrcPtr += srcLength;
+    consumedLen = currentSrcPtr - originalSrcPtr; // src length used so far
+  }
+  dstLength = 512; // Reset available buffer size.
+  rv = encoder->Finish(localbuf, &dstLength);
+  if (NS_SUCCEEDED(rv)) {
+    if (dstLength)
+      outString.Append(localbuf, dstLength);
+    return !mappingFailure ? rv: NS_ERROR_UENC_NOMAPPING;
+  }
+  return rv;
+}
+
+nsresult nsMsgI18NConvertToUnicode(const char* aCharset,
+                                   const nsCString& inString, 
+                                   nsAString& outString,
+                                   bool aIsCharsetCanonical)
+{
+  if (inString.IsEmpty()) {
+    outString.Truncate();
+    return NS_OK;
+  }
+  else if (!*aCharset || !PL_strcasecmp(aCharset, "us-ascii") ||
+           !PL_strcasecmp(aCharset, "ISO-8859-1")) {
+    // Despite its name, it also works for Latin-1.
+    CopyASCIItoUTF16(inString, outString);
+    return NS_OK;
+  }
+  else if (!PL_strcasecmp(aCharset, "UTF-8")) {
+    if (MsgIsUTF8(inString)) {
+      nsAutoString tmp;
+      CopyUTF8toUTF16(inString, tmp);
+      if (!tmp.IsEmpty() && tmp.First() == char16_t(0xFEFF))
+        tmp.Cut(0, 1);
+      outString.Assign(tmp);
+      return NS_OK;
+    }
+    NS_WARNING("Invalid UTF-8 string");
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  nsresult rv;
+  nsCOMPtr <nsICharsetConverterManager> ccm = do_GetService(NS_CHARSETCONVERTERMANAGER_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCOMPtr <nsIUnicodeDecoder> decoder;
+
+  // get an unicode converter
+  if (aIsCharsetCanonical)  // optimize for modified UTF-7 used by IMAP
+    rv = ccm->GetUnicodeDecoderRaw(aCharset, getter_AddRefs(decoder));
+  else
+    rv = ccm->GetUnicodeDecoderInternal(aCharset, getter_AddRefs(decoder));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  const char *originalSrcPtr = inString.get();
+  const char *currentSrcPtr = originalSrcPtr;
+  int32_t originalLength = inString.Length();
+  int32_t srcLength;
+  int32_t dstLength;
+  char16_t localbuf[512];
+  int32_t consumedLen = 0;
+
+  outString.Truncate();
+
+  // convert
+  while (consumedLen < originalLength) {
+    srcLength = originalLength - consumedLen;  
+    dstLength = 512;
+    rv = decoder->Convert(currentSrcPtr, &srcLength, localbuf, &dstLength);
+    if (NS_FAILED(rv) || dstLength == 0)
+      break;
+    outString.Append(localbuf, dstLength);
+
+    currentSrcPtr += srcLength;
+    consumedLen = currentSrcPtr - originalSrcPtr; // src length used so far
+  }
+  return rv;
+}
+
+// Charset used by the file system.
+const char * nsMsgI18NFileSystemCharset()
+{
+  /* Get a charset used for the file. */
+  static nsAutoCString fileSystemCharset;
+
+  if (fileSystemCharset.IsEmpty()) 
+  {
+    nsresult rv;
+    nsCOMPtr <nsIPlatformCharset> platformCharset = do_GetService(NS_PLATFORMCHARSET_CONTRACTID, &rv);
+        if (NS_SUCCEEDED(rv)) {
+          rv = platformCharset->GetCharset(kPlatformCharsetSel_FileName,
+                                           fileSystemCharset);
+        }
+
+    if (NS_FAILED(rv)) 
+      fileSystemCharset.Assign("ISO-8859-1");
+  }
+  return fileSystemCharset.get();
+}
+
+// Charset used by the text file.
+void nsMsgI18NTextFileCharset(nsACString& aCharset)
+{
+  nsresult rv;
+  nsCOMPtr <nsIPlatformCharset> platformCharset =
+    do_GetService(NS_PLATFORMCHARSET_CONTRACTID, &rv);
+  if (NS_SUCCEEDED(rv)) {
+    rv = platformCharset->GetCharset(kPlatformCharsetSel_PlainTextInFile,
+                                     aCharset);
+  }
+
+  if (NS_FAILED(rv))
+    aCharset.Assign("ISO-8859-1");
+}
+
+// MIME encoder, output string should be freed by PR_FREE
+// XXX : fix callers later to avoid allocation and copy
+char * nsMsgI18NEncodeMimePartIIStr(const char *header, bool structured, const char *charset, int32_t fieldnamelen, bool usemime) 
+{
+  // No MIME, convert to the outgoing mail charset.
+  if (false == usemime) {
+    nsAutoCString convertedStr;
+    if (NS_SUCCEEDED(ConvertFromUnicode(charset, NS_ConvertUTF8toUTF16(header),
+                                        convertedStr)))
+      return PL_strdup(convertedStr.get());
+    else
+      return PL_strdup(header);
+  }
+
+  nsAutoCString encodedString;
+  nsresult res;
+  nsCOMPtr<nsIMimeConverter> converter = do_GetService(NS_MIME_CONVERTER_CONTRACTID, &res);
+  if (NS_SUCCEEDED(res) && nullptr != converter)
+    res = converter->EncodeMimePartIIStr_UTF8(nsDependentCString(header),
+      structured, "UTF-8", fieldnamelen,
+      nsIMimeConverter::MIME_ENCODED_WORD_SIZE, encodedString);
+
+  return NS_SUCCEEDED(res) ? PL_strdup(encodedString.get()) : nullptr;
+}
+
+// Return True if a charset is stateful (e.g. JIS).
+bool nsMsgI18Nstateful_charset(const char *charset)
+{
+  //TODO: use charset manager's service
+  return (PL_strcasecmp(charset, "ISO-2022-JP") == 0);
+}
+
+bool nsMsgI18Nmultibyte_charset(const char *charset)
+{
+  nsresult res;
+  nsCOMPtr <nsICharsetConverterManager> ccm = do_GetService(NS_CHARSETCONVERTERMANAGER_CONTRACTID, &res);
+  bool result = false;
+
+  if (NS_SUCCEEDED(res)) {
+    nsAutoString charsetData;
+    res = ccm->GetCharsetData(charset, u".isMultibyte", charsetData);
+    if (NS_SUCCEEDED(res)) {
+      result = charsetData.LowerCaseEqualsLiteral("true");
+    }
+  }
+
+  return result;
+}
+
+bool nsMsgI18Ncheck_data_in_charset_range(const char *charset, const char16_t* inString, char **fallbackCharset)
+{
+  if (!charset || !*charset || !inString || !*inString)
+    return true;
+
+  nsresult res;
+  bool result = true;
+  
+  nsCOMPtr <nsICharsetConverterManager> ccm = do_GetService(NS_CHARSETCONVERTERMANAGER_CONTRACTID, &res);
+
+  if (NS_SUCCEEDED(res)) {
+    nsCOMPtr <nsIUnicodeEncoder> encoder;
+
+    // get an unicode converter
+    res = ccm->GetUnicodeEncoderRaw(charset, getter_AddRefs(encoder));
+    if(NS_SUCCEEDED(res)) {
+      const char16_t *originalPtr = inString;
+      int32_t originalLen = NS_strlen(inString);
+      const char16_t *currentSrcPtr = originalPtr;
+      char localBuff[512];
+      int32_t consumedLen = 0;
+      int32_t srcLen;
+      int32_t dstLength;
+
+      // convert from unicode
+      while (consumedLen < originalLen) {
+        srcLen = originalLen - consumedLen;
+        dstLength = 512;
+        res = encoder->Convert(currentSrcPtr, &srcLen, localBuff, &dstLength);
+        if (NS_ERROR_UENC_NOMAPPING == res) {
+          result = false;
+          break;
+        }
+        else if (NS_FAILED(res) || (0 == dstLength))
+          break;
+
+        currentSrcPtr += srcLen;
+        consumedLen = currentSrcPtr - originalPtr; // src length used so far
+      }
+    }    
+  }
+
+  // if the conversion was not successful then try fallback to other charsets
+  if (!result && fallbackCharset) {
+    nsCString convertedString;
+    res = nsMsgI18NConvertFromUnicode(*fallbackCharset,
+      nsDependentString(inString), convertedString, false, true);
+    result = (NS_SUCCEEDED(res) && NS_ERROR_UENC_NOMAPPING != res);
+  }
+
+  return result;
+}
+
+// Simple parser to parse META charset. 
+// It only supports the case when the description is within one line. 
+const char * 
+nsMsgI18NParseMetaCharset(nsIFile* file) 
+{ 
+  static char charset[nsIMimeConverter::MAX_CHARSET_NAME_LENGTH+1];
+
+  *charset = '\0'; 
+
+  bool isDirectory = false;
+  file->IsDirectory(&isDirectory);
+  if (isDirectory) {
+    NS_ERROR("file is a directory");
+    return charset; 
+  }
+
+  nsresult rv;
+  nsCOMPtr <nsIFileInputStream> fileStream = do_CreateInstance(NS_LOCALFILEINPUTSTREAM_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, charset);
+  
+  rv = fileStream->Init(file, PR_RDONLY, 0664, false);
+  nsCOMPtr <nsILineInputStream> lineStream = do_QueryInterface(fileStream, &rv);
+
+  nsCString curLine;
+  bool more = true;
+  while (NS_SUCCEEDED(rv) && more) { 
+    rv = lineStream->ReadLine(curLine, &more); 
+    if (curLine.IsEmpty()) 
+      continue; 
+
+    ToUpperCase(curLine);
+
+    if (curLine.Find("/HEAD") != -1) 
+      break; 
+
+    if (curLine.Find("META") != -1 && 
+       curLine.Find("HTTP-EQUIV") != -1 && 
+        curLine.Find("CONTENT-TYPE") != -1 && 
+       curLine.Find("CHARSET") != -1) { 
+      char *cp = (char *) PL_strchr(PL_strstr(curLine.get(), "CHARSET"), '=');
+      char *token = nullptr;
+      if (cp)
+      {
+        char *newStr = cp + 1;
+        token = NS_strtok(" \"\'", &newStr);
+      }
+      if (token) { 
+        PL_strncpy(charset, token, sizeof(charset));
+        charset[sizeof(charset)-1] = '\0';
+
+        // this function cannot parse a file if it is really
+        // encoded by one of the following charsets
+        // so we can say that the charset label must be incorrect for
+        // the .html if we actually see those charsets parsed
+        // and we should ignore them
+        if (!PL_strncasecmp("UTF-16", charset, sizeof("UTF-16")-1) || 
+            !PL_strncasecmp("UTF-32", charset, sizeof("UTF-32")-1))
+          charset[0] = '\0';
+
+        break;
+      } 
+    } 
+  } 
+
+  return charset; 
+}
+
+nsresult nsMsgI18NShrinkUTF8Str(const nsCString &inString,
+                                uint32_t aMaxLength,
+                                nsACString &outString)
+{
+  if (inString.IsEmpty()) {
+    outString.Truncate();
+    return NS_OK;
+  }
+  if (inString.Length() < aMaxLength) {
+    outString.Assign(inString);
+    return NS_OK;
+  }
+  NS_ASSERTION(MsgIsUTF8(inString), "Invalid UTF-8 string is inputted");
+  const char* start = inString.get();
+  const char* end = start + inString.Length();
+  const char* last = start + aMaxLength;
+  const char* cur = start;
+  const char* prev = nullptr;
+  bool err = false;
+  while (cur < last) {
+    prev = cur;
+    if (!UTF8CharEnumerator::NextChar(&cur, end, &err) || err)
+      break;
+  }
+  if (!prev || err) {
+    outString.Truncate();
+    return NS_OK;
+  }
+  uint32_t len = prev - start;
+  outString.Assign(Substring(inString, 0, len));
+  return NS_OK;
+}
+
+void nsMsgI18NConvertRawBytesToUTF16(const nsCString& inString, 
+                                     const char* charset,
+                                     nsAString& outString)
+{
+  if (MsgIsUTF8(inString))
+  {
+    CopyUTF8toUTF16(inString, outString);
+    return;
+  }
+
+  nsresult rv = ConvertToUnicode(charset, inString, outString);
+  if (NS_SUCCEEDED(rv))
+    return;
+
+  const char* cur = inString.BeginReading();
+  const char* end = inString.EndReading();
+  outString.Truncate();
+  while (cur < end) {
+    char c = *cur++;
+    if (c & char(0x80))
+      outString.Append(UCS2_REPLACEMENT_CHAR);
+    else
+      outString.Append(c);
+  }
+}
+
+void nsMsgI18NConvertRawBytesToUTF8(const nsCString& inString, 
+                                    const char* charset,
+                                    nsACString& outString)
+{
+  if (MsgIsUTF8(inString))
+  {
+    outString.Assign(inString);
+    return;
+  }
+
+  nsAutoString utf16Text;
+  nsresult rv = ConvertToUnicode(charset, inString, utf16Text);
+  if (NS_SUCCEEDED(rv))
+  {
+    CopyUTF16toUTF8(utf16Text, outString);
+    return;
+  }
+
+  // EF BF BD (UTF-8 encoding of U+FFFD)
+  NS_NAMED_LITERAL_CSTRING(utf8ReplacementChar, "\357\277\275");
+  const char* cur = inString.BeginReading();
+  const char* end = inString.EndReading();
+  outString.Truncate();
+  while (cur < end) {
+    char c = *cur++;
+    if (c & char(0x80))
+      outString.Append(utf8ReplacementChar);
+    else
+      outString.Append(c);
+  }
+}

mailnews/base/util/nsMsgI18N.h

@@ -0,0 +1,198 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMsgI18N_H_
+#define _nsMsgI18N_H_
+
+#include "nscore.h"
+#include "msgCore.h"
+#include "nsStringGlue.h"
+class nsIFile;
+
+/**
+ * Encode an input string into RFC 2047 form.
+ *
+ * @param header       [IN] A header to encode.
+ * @param structured   [IN] Specify the header is structured or non-structured field (See RFC-822).
+ * @param charset      [IN] Charset name to convert.
+ * @param fieldnamelen [IN] Header field name length. (e.g. "From: " -> 6)
+ * @param usemime      [IN] If false then apply charset conversion only no MIME encoding.
+ * @return             Encoded buffer (in C string) or NULL in case of error.
+ */
+NS_MSG_BASE char      *nsMsgI18NEncodeMimePartIIStr(const char *header, bool structured, const char *charset, int32_t fieldnamelen, bool usemime);
+
+/**
+ * Check if given charset is stateful (e.g. ISO-2022-JP).
+ *
+ * @param charset     [IN] Charset name.
+ * @return            True if stateful
+ */
+NS_MSG_BASE bool      nsMsgI18Nstateful_charset(const char *charset);
+
+/**
+ * Check if given charset is multibye (e.g. Shift_JIS, Big5).
+ *
+ * @param charset     [IN] Charset name.
+ * @return            True if multibyte
+ */
+NS_MSG_BASE bool nsMsgI18Nmultibyte_charset(const char *charset);
+
+/**
+ * Check the input (unicode) string is in a range of the given charset after the conversion.
+ * Note, do not use this for large string (e.g. message body) since this actually applies the conversion to the buffer.
+ *
+ * @param charset     [IN] Charset to be converted.
+ * @param inString    [IN] Input unicode string to be examined.
+ * @param fallbackCharset [OUT]
+ *                         null if fallback charset is not needed.
+ *                         Otherwise, a fallback charset name may be set if that was used for the conversion. 
+ *                         Caller is responsible for freeing the memory.
+ * @return            True if the string can be converted within the charset range.
+ *                    False if one or more characters cannot be converted to the target charset.
+ */
+NS_MSG_BASE bool      nsMsgI18Ncheck_data_in_charset_range(const char *charset, const char16_t* inString,
+                                                           char **fallbackCharset=nullptr);
+
+/**
+ * Return charset name of file system (OS dependent).
+ *
+ * @return            File system charset name.
+ */
+NS_MSG_BASE const char * nsMsgI18NFileSystemCharset(void);
+
+/**
+ * Return charset name of text file (OS dependent).
+ *
+ * @param aCharset    [OUT] Text file charset name.
+ */
+NS_MSG_BASE void nsMsgI18NTextFileCharset(nsACString& aCharset);
+
+/**
+ * Convert from unicode to target charset.
+ *
+ * @param charset     [IN] Charset name.
+ * @param inString    [IN] Unicode string to convert.
+ * @param outString   [OUT] Converted output string.
+ * @param aIsCharsetCanonical  [IN] Whether the charset is canonical or not.
+ * @param aReportUencNoMapping [IN] Set encoder to report (instead of using
+ *                                  replacement char on errors). Set to true
+ *                                  to receive NS_ERROR_UENC_NOMAPPING when
+ *                                  that happens. Note that
+ *                                  NS_ERROR_UENC_NOMAPPING is a success code!
+ * @return            nsresult.
+ */
+NS_MSG_BASE nsresult nsMsgI18NConvertFromUnicode(const char* aCharset,
+                                                 const nsString& inString,
+                                                 nsACString& outString,
+                                                 bool aIsCharsetCanonical =
+                                                        false,
+                                                 bool reportUencNoMapping =
+                                                        false);
+/**
+ * Convert from charset to unicode.
+ *
+ * @param charset     [IN] Charset name.
+ * @param inString    [IN] Input string to convert.
+ * @param outString   [OUT] Output unicode string.
+ * @return            nsresult.
+ */
+NS_MSG_BASE nsresult nsMsgI18NConvertToUnicode(const char* aCharset,
+                                               const nsCString& inString,
+                                               nsAString& outString,
+                                               bool aIsCharsetCanonical =
+                                                      false);
+/**
+ * Parse for META charset.
+ *
+ * @param file    [IN] A nsIFile.
+ * @return            A charset name or empty string if not found.
+ */
+NS_MSG_BASE const char *nsMsgI18NParseMetaCharset(nsIFile* file);
+
+/**
+ * Shrink the aStr to aMaxLength bytes. Note that this doesn't check whether
+ * the aUTF8Str is valid UTF-8 string.
+ *
+ * @param inString   [IN] Input UTF-8 string (it must be valid UTF-8 string)
+ * @param aMaxLength [IN] Shrink to this length (it means bytes)
+ * @param outString  [OUT] Shrunken UTF-8 string
+ * @return           nsresult
+ */
+NS_MSG_BASE nsresult nsMsgI18NShrinkUTF8Str(const nsCString &inString,
+                                            uint32_t aMaxLength,
+                                            nsACString &outString);
+
+/*
+ * Convert raw bytes in header to UTF-16
+ *
+ * @param inString   [IN] Input raw octets
+ * @param outString  [OUT] Output UTF-16 string
+ */
+NS_MSG_BASE void nsMsgI18NConvertRawBytesToUTF16(const nsCString& inString, 
+                                                 const char* charset,
+                                                 nsAString& outString);
+
+/*
+ * Convert raw bytes in header to UTF-8
+ *
+ * @param inString   [IN] Input raw octets
+ * @param outString  [OUT] Output UTF-8 string
+ */
+NS_MSG_BASE void nsMsgI18NConvertRawBytesToUTF8(const nsCString& inString, 
+                                                const char* charset,
+                                                nsACString& outString);
+
+// inline forwarders to avoid littering with 'x-imap4-.....'
+inline nsresult CopyUTF16toMUTF7(const nsString &aSrc, nsACString& aDest)
+{
+    return nsMsgI18NConvertFromUnicode("x-imap4-modified-utf7", aSrc,
+                                       aDest, true);
+}
+
+inline nsresult CopyMUTF7toUTF16(const nsCString& aSrc, nsAString& aDest)
+{
+    return nsMsgI18NConvertToUnicode("x-imap4-modified-utf7", aSrc,
+                                     aDest, true);
+}
+
+inline nsresult ConvertToUnicode(const char* charset,
+                                 const nsCString &aSrc, nsAString& aDest)
+{
+    return nsMsgI18NConvertToUnicode(charset, aSrc, aDest);
+}
+
+inline nsresult ConvertToUnicode(const char* charset,
+                                 const char* aSrc, nsAString& aDest)
+{
+    return nsMsgI18NConvertToUnicode(charset, nsDependentCString(aSrc), aDest);
+}
+
+inline nsresult ConvertFromUnicode(const char* charset,
+                                   const nsString &aSrc, nsACString& aDest)
+{
+    return nsMsgI18NConvertFromUnicode(charset, aSrc, aDest);
+}
+
+inline void ConvertRawBytesToUTF16(const nsCString& inString, 
+                                   const char* charset, nsAString& outString)
+{
+    return nsMsgI18NConvertRawBytesToUTF16(inString, charset, outString);
+}
+
+inline void ConvertRawBytesToUTF16(const char* inString, 
+                                   const char* charset, nsAString& outString)
+{
+    return nsMsgI18NConvertRawBytesToUTF16(nsDependentCString(inString),
+                                           charset,
+                                           outString);
+}
+
+inline void ConvertRawBytesToUTF8(const nsCString& inString, 
+                                  const char* charset, nsACString& outString)
+{
+    return nsMsgI18NConvertRawBytesToUTF8(inString, charset, outString);
+}
+
+#endif /* _nsMsgI18N_H_ */

mailnews/base/util/nsMsgIdentity.cpp

@@ -0,0 +1,669 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h" // for pre-compiled headers
+#include "nsMsgIdentity.h"
+#include "nsIPrefService.h"
+#include "nsStringGlue.h"
+#include "nsMsgCompCID.h"
+#include "nsIRDFService.h"
+#include "nsIRDFResource.h"
+#include "nsRDFCID.h"
+#include "nsMsgFolderFlags.h"
+#include "nsIMsgFolder.h"
+#include "nsIMsgIncomingServer.h"
+#include "nsIMsgAccountManager.h"
+#include "mozilla/mailnews/MimeHeaderParser.h"
+#include "nsMsgBaseCID.h"
+#include "prprf.h"
+#include "nsISupportsPrimitives.h"
+#include "nsMsgUtils.h"
+#include "nsServiceManagerUtils.h"
+#include "nsComponentManagerUtils.h"
+#include "nsArrayUtils.h"
+
+static NS_DEFINE_CID(kRDFServiceCID, NS_RDFSERVICE_CID);
+
+#define REL_FILE_PREF_SUFFIX "-rel"
+
+NS_IMPL_ISUPPORTS(nsMsgIdentity,
+                   nsIMsgIdentity)
+
+/*
+ * accessors for pulling values directly out of preferences
+ * instead of member variables, etc
+ */
+
+NS_IMETHODIMP
+nsMsgIdentity::GetKey(nsACString& aKey)
+{
+  aKey = mKey;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::SetKey(const nsACString& identityKey)
+{
+  mKey = identityKey;
+  nsresult rv;
+  nsCOMPtr<nsIPrefService> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  if (NS_FAILED(rv))
+    return rv;
+
+  nsAutoCString branchName;
+  branchName.AssignLiteral("mail.identity.");
+  branchName += mKey;
+  branchName.Append('.');
+  rv = prefs->GetBranch(branchName.get(), getter_AddRefs(mPrefBranch));
+  if (NS_FAILED(rv))
+    return rv;
+
+  rv = prefs->GetBranch("mail.identity.default.", getter_AddRefs(mDefPrefBranch));
+  return rv;
+}
+
+nsresult
+nsMsgIdentity::GetIdentityName(nsAString& idName)
+{
+  idName.AssignLiteral("");
+  // Try to use "fullname <email>" as the name.
+  nsresult rv = GetFullAddress(idName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // If a non-empty label exists, append it.
+  nsString label;
+  rv = GetLabel(label);
+  if (NS_SUCCEEDED(rv) && !label.IsEmpty())
+  { // TODO: this should be localizable
+    idName.AppendLiteral(" (");
+    idName.Append(label);
+    idName.AppendLiteral(")");
+  }
+
+  if (!idName.IsEmpty())
+    return NS_OK;
+
+  // If we still found nothing to use, use our key.
+  return ToString(idName);
+}
+
+nsresult
+nsMsgIdentity::GetFullAddress(nsAString& fullAddress)
+{
+  nsAutoString fullName;
+  nsresult rv = GetFullName(fullName);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoCString email;
+  rv = GetEmail(email);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (fullName.IsEmpty() && email.IsEmpty())
+    fullAddress.Truncate();
+  else
+    mozilla::mailnews::MakeMimeAddress(fullName, NS_ConvertASCIItoUTF16(email), fullAddress);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::ToString(nsAString& aResult)
+{
+  aResult.AssignLiteral("[nsIMsgIdentity: ");
+  aResult.Append(NS_ConvertASCIItoUTF16(mKey));
+  aResult.AppendLiteral("]");
+  return NS_OK;
+}
+
+/* Identity attribute accessors */
+
+NS_IMETHODIMP
+nsMsgIdentity::GetSignature(nsIFile **sig)
+{
+  bool gotRelPref;
+  nsresult rv = NS_GetPersistentFile("sig_file" REL_FILE_PREF_SUFFIX, "sig_file", nullptr, gotRelPref, sig, mPrefBranch);
+  if (NS_SUCCEEDED(rv) && !gotRelPref)
+  {
+    rv = NS_SetPersistentFile("sig_file" REL_FILE_PREF_SUFFIX, "sig_file", *sig, mPrefBranch);
+    NS_ASSERTION(NS_SUCCEEDED(rv), "Failed to write signature file pref.");
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::SetSignature(nsIFile *sig)
+{
+  nsresult rv = NS_OK;
+  if (sig)
+    rv = NS_SetPersistentFile("sig_file" REL_FILE_PREF_SUFFIX, "sig_file", sig, mPrefBranch);
+  return rv;
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::ClearAllValues()
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  return mPrefBranch->DeleteBranch("");
+}
+
+NS_IMPL_IDPREF_STR(EscapedVCard, "escapedVCard")
+NS_IMPL_IDPREF_STR(SmtpServerKey, "smtpServer")
+NS_IMPL_IDPREF_WSTR(FullName, "fullName")
+NS_IMPL_IDPREF_STR(Email, "useremail")
+NS_IMPL_IDPREF_WSTR(Label, "label")
+NS_IMPL_IDPREF_STR(ReplyTo, "reply_to")
+NS_IMPL_IDPREF_WSTR(Organization, "organization")
+NS_IMPL_IDPREF_BOOL(ComposeHtml, "compose_html")
+NS_IMPL_IDPREF_BOOL(AttachVCard, "attach_vcard")
+NS_IMPL_IDPREF_BOOL(AttachSignature, "attach_signature")
+NS_IMPL_IDPREF_WSTR(HtmlSigText, "htmlSigText")
+NS_IMPL_IDPREF_BOOL(HtmlSigFormat, "htmlSigFormat")
+
+NS_IMPL_IDPREF_BOOL(AutoQuote, "auto_quote")
+NS_IMPL_IDPREF_INT(ReplyOnTop, "reply_on_top")
+NS_IMPL_IDPREF_BOOL(SigBottom, "sig_bottom")
+NS_IMPL_IDPREF_BOOL(SigOnForward, "sig_on_fwd")
+NS_IMPL_IDPREF_BOOL(SigOnReply, "sig_on_reply")
+
+NS_IMPL_IDPREF_INT(SignatureDate,"sig_date")
+
+NS_IMPL_IDPREF_BOOL(DoFcc, "fcc")
+
+NS_IMPL_FOLDERPREF_STR(FccFolder, "fcc_folder", "Sent", nsMsgFolderFlags::SentMail)
+NS_IMPL_IDPREF_STR(FccFolderPickerMode, "fcc_folder_picker_mode")
+NS_IMPL_IDPREF_BOOL(FccReplyFollowsParent, "fcc_reply_follows_parent")
+NS_IMPL_IDPREF_STR(DraftsFolderPickerMode, "drafts_folder_picker_mode")
+NS_IMPL_IDPREF_STR(ArchivesFolderPickerMode, "archives_folder_picker_mode")
+NS_IMPL_IDPREF_STR(TmplFolderPickerMode, "tmpl_folder_picker_mode")
+
+NS_IMPL_IDPREF_BOOL(BccSelf, "bcc_self")
+NS_IMPL_IDPREF_BOOL(BccOthers, "bcc_other")
+NS_IMPL_IDPREF_STR (BccList, "bcc_other_list")
+
+NS_IMPL_IDPREF_BOOL(SuppressSigSep, "suppress_signature_separator")
+
+NS_IMPL_IDPREF_BOOL(DoCc, "doCc")
+NS_IMPL_IDPREF_STR (DoCcList, "doCcList")
+
+NS_IMETHODIMP
+nsMsgIdentity::GetDoBcc(bool *aValue)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsresult rv = mPrefBranch->GetBoolPref("doBcc", aValue);
+  if (NS_SUCCEEDED(rv))
+    return rv;
+
+  bool bccSelf = false;
+  GetBccSelf(&bccSelf);
+
+  bool bccOthers = false;
+  GetBccOthers(&bccOthers);
+
+  nsCString others;
+  GetBccList(others);
+
+  *aValue = bccSelf || (bccOthers && !others.IsEmpty());
+
+  return SetDoBcc(*aValue);
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::SetDoBcc(bool aValue)
+{
+  return SetBoolAttribute("doBcc", aValue);
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::GetDoBccList(nsACString& aValue)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsCString val;
+  nsresult rv = mPrefBranch->GetCharPref("doBccList", getter_Copies(val));
+  aValue = val;
+  if (NS_SUCCEEDED(rv))
+    return rv;
+
+  bool bccSelf = false;
+  rv = GetBccSelf(&bccSelf);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  if (bccSelf)
+    GetEmail(aValue);
+
+  bool bccOthers = false;
+  rv = GetBccOthers(&bccOthers);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  nsCString others;
+  rv = GetBccList(others);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  if (bccOthers && !others.IsEmpty()) {
+    if (bccSelf)
+      aValue.AppendLiteral(",");
+    aValue.Append(others);
+  }
+
+  return SetDoBccList(aValue);
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::SetDoBccList(const nsACString& aValue)
+{
+  return SetCharAttribute("doBccList", aValue);
+}
+
+NS_IMPL_FOLDERPREF_STR(DraftFolder, "draft_folder", "Drafts", nsMsgFolderFlags::Drafts)
+NS_IMPL_FOLDERPREF_STR(ArchiveFolder, "archive_folder", "Archives", nsMsgFolderFlags::Archive)
+NS_IMPL_FOLDERPREF_STR(StationeryFolder, "stationery_folder", "Templates", nsMsgFolderFlags::Templates)
+
+NS_IMPL_IDPREF_BOOL(ArchiveEnabled, "archive_enabled")
+NS_IMPL_IDPREF_INT(ArchiveGranularity, "archive_granularity")
+NS_IMPL_IDPREF_BOOL(ArchiveKeepFolderStructure, "archive_keep_folder_structure")
+
+NS_IMPL_IDPREF_BOOL(ShowSaveMsgDlg, "showSaveMsgDlg")
+NS_IMPL_IDPREF_STR (DirectoryServer, "directoryServer")
+NS_IMPL_IDPREF_BOOL(OverrideGlobalPref, "overrideGlobal_Pref")
+NS_IMPL_IDPREF_BOOL(AutocompleteToMyDomain, "autocompleteToMyDomain")
+
+NS_IMPL_IDPREF_BOOL(Valid, "valid")
+
+nsresult
+nsMsgIdentity::getFolderPref(const char *prefname, nsCString& retval,
+                             const char *folderName, uint32_t folderflag)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsresult rv = mPrefBranch->GetCharPref(prefname, getter_Copies(retval));
+  if (NS_SUCCEEDED(rv) && !retval.IsEmpty()) {
+    // get the corresponding RDF resource
+    // RDF will create the folder resource if it doesn't already exist
+    nsCOMPtr<nsIRDFService> rdf(do_GetService(kRDFServiceCID, &rv));
+    if (NS_FAILED(rv)) return rv;
+    nsCOMPtr<nsIRDFResource> resource;
+    rdf->GetResource(retval, getter_AddRefs(resource));
+
+    nsCOMPtr <nsIMsgFolder> folderResource = do_QueryInterface(resource);
+    if (folderResource)
+    {
+      // don't check validity of folder - caller will handle creating it
+      nsCOMPtr<nsIMsgIncomingServer> server;
+      //make sure that folder hierarchy is built so that legitimate parent-child relationship is established
+      folderResource->GetServer(getter_AddRefs(server));
+      if (server)
+      {
+        nsCOMPtr<nsIMsgFolder> rootFolder;
+        nsCOMPtr<nsIMsgFolder> deferredToRootFolder;
+        server->GetRootFolder(getter_AddRefs(rootFolder));
+        server->GetRootMsgFolder(getter_AddRefs(deferredToRootFolder));
+        // check if we're using a deferred account - if not, use the uri;
+        // otherwise, fall through to code that will fix this pref.
+        if (rootFolder == deferredToRootFolder)
+        {
+          nsCOMPtr <nsIMsgFolder> msgFolder;
+          rv = server->GetMsgFolderFromURI(folderResource, retval, getter_AddRefs(msgFolder));
+          return NS_SUCCEEDED(rv) ? msgFolder->GetURI(retval) : rv;
+        }
+      }
+    }
+  }
+
+  // if the server doesn't exist, fall back to the default pref.
+  rv = mDefPrefBranch->GetCharPref(prefname, getter_Copies(retval));
+  if (NS_SUCCEEDED(rv) && !retval.IsEmpty())
+    return setFolderPref(prefname, retval, folderflag);
+
+  // here I think we need to create a uri for the folder on the
+  // default server for this identity.
+  nsCOMPtr<nsIMsgAccountManager> accountManager =
+  do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  nsCOMPtr<nsIArray> servers;
+  rv = accountManager->GetServersForIdentity(this, getter_AddRefs(servers));
+  NS_ENSURE_SUCCESS(rv,rv);
+  nsCOMPtr<nsIMsgIncomingServer> server(do_QueryElementAt(servers, 0, &rv));
+  if (NS_SUCCEEDED(rv))
+  {
+    bool defaultToServer;
+    server->GetDefaultCopiesAndFoldersPrefsToServer(&defaultToServer);
+    // if we should default to special folders on the server,
+    // use the local folders server
+    if (!defaultToServer)
+    {
+      rv = accountManager->GetLocalFoldersServer(getter_AddRefs(server));
+      NS_ENSURE_SUCCESS(rv, rv);
+    }
+    nsCOMPtr<nsIMsgFolder> rootFolder;
+    // this will get the deferred to server's root folder, if "server"
+    // is deferred, e.g., using the pop3 global inbox.
+    rv = server->GetRootMsgFolder(getter_AddRefs(rootFolder));
+    NS_ENSURE_SUCCESS(rv, rv);
+    if (rootFolder)
+    {
+      rv = rootFolder->GetURI(retval);
+      NS_ENSURE_SUCCESS(rv, rv);
+      retval.Append('/');
+      retval.Append(folderName);
+      return setFolderPref(prefname, retval, folderflag);
+    }
+  }
+  // if there are no servers for this identity, return generic failure.
+  return NS_ERROR_FAILURE;
+}
+
+nsresult
+nsMsgIdentity::setFolderPref(const char *prefname, const nsACString& value, uint32_t folderflag)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsCString oldpref;
+  nsresult rv;
+  nsCOMPtr<nsIRDFResource> res;
+  nsCOMPtr<nsIMsgFolder> folder;
+  nsCOMPtr<nsIRDFService> rdf(do_GetService(kRDFServiceCID, &rv));
+
+  if (folderflag == nsMsgFolderFlags::SentMail)
+  {
+    // Clear the temporary return receipt filter so that the new filter
+    // rule can be recreated (by ConfigureTemporaryFilters()).
+    nsCOMPtr<nsIMsgAccountManager> accountManager =
+      do_GetService(NS_MSGACCOUNTMANAGER_CONTRACTID, &rv);
+    NS_ENSURE_SUCCESS(rv,rv);
+
+    nsCOMPtr<nsIArray> servers;
+    rv = accountManager->GetServersForIdentity(this, getter_AddRefs(servers));
+    NS_ENSURE_SUCCESS(rv,rv);
+    uint32_t cnt = 0;
+    servers->GetLength(&cnt);
+    if (cnt > 0)
+    {
+      nsCOMPtr<nsIMsgIncomingServer> server(do_QueryElementAt(servers, 0, &rv));
+      if (NS_SUCCEEDED(rv))
+        server->ClearTemporaryReturnReceiptsFilter(); // okay to fail; no need to check for return code
+    }
+  }
+
+  // get the old folder, and clear the special folder flag on it
+  rv = mPrefBranch->GetCharPref(prefname, getter_Copies(oldpref));
+  if (NS_SUCCEEDED(rv) && !oldpref.IsEmpty())
+  {
+    rv = rdf->GetResource(oldpref, getter_AddRefs(res));
+    if (NS_SUCCEEDED(rv) && res)
+    {
+      folder = do_QueryInterface(res, &rv);
+      if (NS_SUCCEEDED(rv))
+        rv = folder->ClearFlag(folderflag);
+    }
+  }
+
+  // set the new folder, and set the special folder flags on it
+  rv = SetCharAttribute(prefname, value);
+  if (NS_SUCCEEDED(rv) && !value.IsEmpty())
+  {
+    rv = rdf->GetResource(value, getter_AddRefs(res));
+    if (NS_SUCCEEDED(rv) && res)
+    {
+      folder = do_QueryInterface(res, &rv);
+      if (NS_SUCCEEDED(rv))
+        rv = folder->SetFlag(folderflag);
+    }
+  }
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgIdentity::SetUnicharAttribute(const char *aName, const nsAString& val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  if (!val.IsEmpty()) {
+    nsresult rv;
+    nsCOMPtr<nsISupportsString> supportsString(
+        do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv));
+    if (NS_SUCCEEDED(rv))
+      rv = supportsString->SetData(val);
+    if (NS_SUCCEEDED(rv))
+      rv = mPrefBranch->SetComplexValue(aName,
+                                        NS_GET_IID(nsISupportsString),
+                                        supportsString);
+    return rv;
+  }
+
+  mPrefBranch->ClearUserPref(aName);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgIdentity::GetUnicharAttribute(const char *aName, nsAString& val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsCOMPtr<nsISupportsString> supportsString;
+  if (NS_FAILED(mPrefBranch->GetComplexValue(aName,
+                                             NS_GET_IID(nsISupportsString),
+                                             getter_AddRefs(supportsString))))
+    mDefPrefBranch->GetComplexValue(aName,
+                                    NS_GET_IID(nsISupportsString),
+                                    getter_AddRefs(supportsString));
+
+  if (supportsString)
+    supportsString->GetData(val);
+  else
+    val.Truncate();
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgIdentity::SetCharAttribute(const char *aName, const nsACString& val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  if (!val.IsEmpty())
+    return mPrefBranch->SetCharPref(aName, nsCString(val).get());
+
+  mPrefBranch->ClearUserPref(aName);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgIdentity::GetCharAttribute(const char *aName, nsACString& val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  nsCString tmpVal;
+  if (NS_FAILED(mPrefBranch->GetCharPref(aName, getter_Copies(tmpVal))))
+    mDefPrefBranch->GetCharPref(aName, getter_Copies(tmpVal));
+  val = tmpVal;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgIdentity::SetBoolAttribute(const char *aName, bool val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  return mPrefBranch->SetBoolPref(aName, val);
+}
+
+NS_IMETHODIMP nsMsgIdentity::GetBoolAttribute(const char *aName, bool *val)
+{
+  NS_ENSURE_ARG_POINTER(val);
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  *val = false;
+
+  if (NS_FAILED(mPrefBranch->GetBoolPref(aName, val)))
+    mDefPrefBranch->GetBoolPref(aName, val);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgIdentity::SetIntAttribute(const char *aName, int32_t val)
+{
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  return mPrefBranch->SetIntPref(aName, val);
+}
+
+NS_IMETHODIMP nsMsgIdentity::GetIntAttribute(const char *aName, int32_t *val)
+{
+  NS_ENSURE_ARG_POINTER(val);
+
+  if (!mPrefBranch)
+    return NS_ERROR_NOT_INITIALIZED;
+
+  *val = 0;
+
+  if (NS_FAILED(mPrefBranch->GetIntPref(aName, val)))
+    mDefPrefBranch->GetIntPref(aName, val);
+
+  return NS_OK;
+}
+
+#define COPY_IDENTITY_FILE_VALUE(SRC_ID,MACRO_GETTER,MACRO_SETTER)   \
+  {  \
+    nsresult macro_rv;  \
+    nsCOMPtr <nsIFile>macro_spec;   \
+          macro_rv = SRC_ID->MACRO_GETTER(getter_AddRefs(macro_spec)); \
+          if (NS_SUCCEEDED(macro_rv)) \
+            this->MACRO_SETTER(macro_spec);     \
+  }
+
+#define COPY_IDENTITY_INT_VALUE(SRC_ID,MACRO_GETTER,MACRO_SETTER)   \
+  {  \
+        nsresult macro_rv;  \
+          int32_t macro_oldInt;  \
+          macro_rv = SRC_ID->MACRO_GETTER(&macro_oldInt);  \
+          if (NS_SUCCEEDED(macro_rv)) \
+            this->MACRO_SETTER(macro_oldInt);     \
+  }
+
+#define COPY_IDENTITY_BOOL_VALUE(SRC_ID,MACRO_GETTER,MACRO_SETTER)   \
+  {  \
+        nsresult macro_rv;  \
+          bool macro_oldBool;  \
+          macro_rv = SRC_ID->MACRO_GETTER(&macro_oldBool);  \
+          if (NS_SUCCEEDED(macro_rv)) \
+            this->MACRO_SETTER(macro_oldBool);     \
+  }
+
+#define COPY_IDENTITY_STR_VALUE(SRC_ID,MACRO_GETTER,MACRO_SETTER)   \
+  {  \
+          nsCString macro_oldStr;  \
+        nsresult macro_rv;  \
+          macro_rv = SRC_ID->MACRO_GETTER(macro_oldStr);  \
+            if (NS_SUCCEEDED(macro_rv)) { \
+                    this->MACRO_SETTER(macro_oldStr);  \
+            } \
+  }
+
+#define COPY_IDENTITY_WSTR_VALUE(SRC_ID,MACRO_GETTER,MACRO_SETTER)   \
+  {  \
+          nsString macro_oldStr;  \
+        nsresult macro_rv;  \
+          macro_rv = SRC_ID->MACRO_GETTER(macro_oldStr); \
+          if (NS_SUCCEEDED(macro_rv)) { \
+                  this->MACRO_SETTER(macro_oldStr);  \
+              }  \
+  }
+
+NS_IMETHODIMP
+nsMsgIdentity::Copy(nsIMsgIdentity *identity)
+{
+    NS_ENSURE_ARG_POINTER(identity);
+
+    COPY_IDENTITY_BOOL_VALUE(identity,GetComposeHtml,SetComposeHtml)
+    COPY_IDENTITY_STR_VALUE(identity,GetEmail,SetEmail)
+    COPY_IDENTITY_WSTR_VALUE(identity,GetLabel,SetLabel)
+    COPY_IDENTITY_STR_VALUE(identity,GetReplyTo,SetReplyTo)
+    COPY_IDENTITY_WSTR_VALUE(identity,GetFullName,SetFullName)
+    COPY_IDENTITY_WSTR_VALUE(identity,GetOrganization,SetOrganization)
+    COPY_IDENTITY_STR_VALUE(identity,GetDraftFolder,SetDraftFolder)
+    COPY_IDENTITY_STR_VALUE(identity,GetArchiveFolder,SetArchiveFolder)
+    COPY_IDENTITY_STR_VALUE(identity,GetFccFolder,SetFccFolder)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetFccReplyFollowsParent,
+                             SetFccReplyFollowsParent)
+    COPY_IDENTITY_STR_VALUE(identity,GetStationeryFolder,SetStationeryFolder)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetArchiveEnabled,SetArchiveEnabled)
+    COPY_IDENTITY_INT_VALUE(identity,GetArchiveGranularity,
+                            SetArchiveGranularity)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetArchiveKeepFolderStructure,
+                             SetArchiveKeepFolderStructure)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetAttachSignature,SetAttachSignature)
+    COPY_IDENTITY_FILE_VALUE(identity,GetSignature,SetSignature)
+    COPY_IDENTITY_WSTR_VALUE(identity,GetHtmlSigText,SetHtmlSigText)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetHtmlSigFormat,SetHtmlSigFormat)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetAutoQuote,SetAutoQuote)
+    COPY_IDENTITY_INT_VALUE(identity,GetReplyOnTop,SetReplyOnTop)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetSigBottom,SetSigBottom)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetSigOnForward,SetSigOnForward)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetSigOnReply,SetSigOnReply)
+    COPY_IDENTITY_INT_VALUE(identity,GetSignatureDate,SetSignatureDate)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetAttachVCard,SetAttachVCard)
+    COPY_IDENTITY_STR_VALUE(identity,GetEscapedVCard,SetEscapedVCard)
+    COPY_IDENTITY_STR_VALUE(identity,GetSmtpServerKey,SetSmtpServerKey)
+    COPY_IDENTITY_BOOL_VALUE(identity,GetSuppressSigSep,SetSuppressSigSep)
+    return NS_OK;
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::GetRequestReturnReceipt(bool *aVal)
+{
+  NS_ENSURE_ARG_POINTER(aVal);
+
+  bool useCustomPrefs = false;
+  nsresult rv = GetBoolAttribute("use_custom_prefs", &useCustomPrefs);
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (useCustomPrefs)
+    return GetBoolAttribute("request_return_receipt_on", aVal);
+
+  nsCOMPtr<nsIPrefBranch> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return prefs->GetBoolPref("mail.receipt.request_return_receipt_on", aVal);
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::GetReceiptHeaderType(int32_t *aType)
+{
+  NS_ENSURE_ARG_POINTER(aType);
+
+  bool useCustomPrefs = false;
+  nsresult rv = GetBoolAttribute("use_custom_prefs", &useCustomPrefs);
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (useCustomPrefs)
+    return GetIntAttribute("request_receipt_header_type", aType);
+
+  nsCOMPtr<nsIPrefBranch> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return prefs->GetIntPref("mail.receipt.request_header_type", aType);
+}
+
+NS_IMETHODIMP
+nsMsgIdentity::GetRequestDSN(bool *aVal)
+{
+  NS_ENSURE_ARG_POINTER(aVal);
+
+  bool useCustomPrefs = false;
+  nsresult rv = GetBoolAttribute("dsn_use_custom_prefs", &useCustomPrefs);
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (useCustomPrefs)
+    return GetBoolAttribute("dsn_always_request_on", aVal);
+
+  nsCOMPtr<nsIPrefBranch> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return prefs->GetBoolPref("mail.dsn.always_request_on", aVal);
+}

mailnews/base/util/nsMsgIdentity.h

@@ -0,0 +1,97 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgIdentity_h___
+#define nsMsgIdentity_h___
+
+#include "nsIMsgIdentity.h"
+#include "nsIPrefBranch.h"
+#include "msgCore.h"
+#include "nsCOMPtr.h"
+#include "nsStringGlue.h"
+
+class NS_MSG_BASE nsMsgIdentity final : public nsIMsgIdentity
+{
+public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIMSGIDENTITY
+  
+private:
+  ~nsMsgIdentity() {}
+  nsCString mKey;
+  nsCOMPtr<nsIPrefBranch> mPrefBranch;
+  nsCOMPtr<nsIPrefBranch> mDefPrefBranch;
+
+protected:
+  nsresult getFolderPref(const char *pref, nsCString&, const char *, uint32_t);
+  nsresult setFolderPref(const char *pref, const nsACString&, uint32_t);
+};
+
+
+#define NS_IMPL_IDPREF_STR(_postfix, _prefname)       \
+NS_IMETHODIMP                                         \
+nsMsgIdentity::Get##_postfix(nsACString& retval)      \
+{                                                     \
+  return GetCharAttribute(_prefname, retval);         \
+}                                                     \
+NS_IMETHODIMP                                         \
+nsMsgIdentity::Set##_postfix(const nsACString& value) \
+{                                                     \
+  return SetCharAttribute(_prefname, value);          \
+}
+
+#define NS_IMPL_IDPREF_WSTR(_postfix, _prefname)     \
+NS_IMETHODIMP                                        \
+nsMsgIdentity::Get##_postfix(nsAString& retval)      \
+{                                                    \
+  return GetUnicharAttribute(_prefname, retval);     \
+}                                                    \
+NS_IMETHODIMP                                        \
+nsMsgIdentity::Set##_postfix(const nsAString& value) \
+{                                                    \
+  return SetUnicharAttribute(_prefname, value);      \
+}
+
+#define NS_IMPL_IDPREF_BOOL(_postfix, _prefname)     \
+NS_IMETHODIMP                                        \
+nsMsgIdentity::Get##_postfix(bool *retval)         \
+{                                                    \
+  return GetBoolAttribute(_prefname, retval);        \
+}                                                    \
+NS_IMETHODIMP                                        \
+nsMsgIdentity::Set##_postfix(bool value)           \
+{                                                    \
+  return mPrefBranch->SetBoolPref(_prefname, value); \
+}
+
+#define NS_IMPL_IDPREF_INT(_postfix, _prefname)     \
+NS_IMETHODIMP                                       \
+nsMsgIdentity::Get##_postfix(int32_t *retval)       \
+{                                                   \
+  return GetIntAttribute(_prefname, retval);        \
+}                                                   \
+NS_IMETHODIMP                                       \
+nsMsgIdentity::Set##_postfix(int32_t value)         \
+{                                                   \
+  return mPrefBranch->SetIntPref(_prefname, value); \
+}
+
+#define NS_IMPL_FOLDERPREF_STR(_postfix, _prefname, _foldername, _flag)  \
+NS_IMETHODIMP                                               \
+nsMsgIdentity::Get##_postfix(nsACString& retval)            \
+{                                                           \
+  nsresult rv;                                              \
+  nsCString folderPref;                                     \
+  rv = getFolderPref(_prefname, folderPref, _foldername, _flag); \
+  retval = folderPref;                                      \
+  return rv;                                                \
+}                                                           \
+NS_IMETHODIMP                                               \
+nsMsgIdentity::Set##_postfix(const nsACString& value)       \
+{                                                           \
+  return setFolderPref(_prefname, value, _flag); \
+}
+
+#endif /* nsMsgIdentity_h___ */

mailnews/base/util/nsMsgIncomingServer.h

@@ -0,0 +1,103 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgIncomingServer_h__
+#define nsMsgIncomingServer_h__
+
+#include "nsIMsgIncomingServer.h"
+#include "nsIPrefBranch.h"
+#include "nsIMsgFilterList.h"
+#include "msgCore.h"
+#include "nsIMsgFolder.h"
+#include "nsIFile.h"
+#include "nsCOMPtr.h"
+#include "nsCOMArray.h"
+#include "nsIPop3IncomingServer.h"
+#include "nsWeakReference.h"
+#include "nsIMsgDatabase.h"
+#include "nsISpamSettings.h"
+#include "nsIMsgFilterPlugin.h"
+#include "nsDataHashtable.h"
+#include "nsIMsgPluggableStore.h"
+
+class nsIMsgFolderCache;
+class nsIMsgProtocolInfo;
+
+/*
+ * base class for nsIMsgIncomingServer - derive your class from here
+ * if you want to get some free implementation
+ *
+ * this particular implementation is not meant to be used directly.
+ */
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+class NS_MSG_BASE nsMsgIncomingServer : public nsIMsgIncomingServer,
+                                        public nsSupportsWeakReference
+{
+ public:
+  nsMsgIncomingServer();
+
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIMSGINCOMINGSERVER
+
+protected:
+  virtual ~nsMsgIncomingServer();
+  nsCString m_serverKey;
+
+  // Sets m_password, if password found. Can return NS_ERROR_ABORT if the 
+  // user cancels the master password dialog.
+  nsresult GetPasswordWithoutUI();
+
+  nsresult ConfigureTemporaryReturnReceiptsFilter(nsIMsgFilterList *filterList);
+  nsresult ConfigureTemporaryServerSpamFilters(nsIMsgFilterList *filterList);
+
+  nsCOMPtr <nsIMsgFolder> m_rootFolder;
+  nsCOMPtr <nsIMsgDownloadSettings> m_downloadSettings;
+
+  // For local servers, where we put messages. For imap/pop3, where we store
+  // offline messages.
+  nsCOMPtr <nsIMsgPluggableStore> m_msgStore;
+
+  /// Helper routine to create local folder on disk if it doesn't exist
+  /// under the account's rootFolder.
+  nsresult CreateLocalFolder(const nsAString& folderName);
+
+  static nsresult GetDeferredServers(nsIMsgIncomingServer *destServer, nsCOMArray<nsIPop3IncomingServer>& aServers);
+
+  nsresult CreateRootFolder();
+  virtual nsresult CreateRootFolderFromUri(const nsCString &serverUri,
+                                           nsIMsgFolder **rootFolder) = 0;
+
+  nsresult InternalSetHostName(const nsACString& aHostname, const char * prefName);
+
+  nsCOMPtr <nsIFile> mFilterFile;
+  nsCOMPtr <nsIMsgFilterList> mFilterList;
+  nsCOMPtr <nsIMsgFilterList> mEditableFilterList;
+  nsCOMPtr<nsIPrefBranch> mPrefBranch;
+  nsCOMPtr<nsIPrefBranch> mDefPrefBranch;
+
+  // these allow us to handle duplicate incoming messages, e.g. delete them.
+  nsDataHashtable<nsCStringHashKey,int32_t> m_downloadedHdrs;
+  int32_t  m_numMsgsDownloaded;
+
+private:
+  uint32_t m_biffState;
+  bool m_serverBusy;
+  nsCOMPtr <nsISpamSettings> mSpamSettings;
+  nsCOMPtr<nsIMsgFilterPlugin> mFilterPlugin;  // XXX should be a list
+
+protected:
+  nsCString m_password;
+  bool m_canHaveFilters;
+  bool m_displayStartupPage;
+  bool mPerformingBiff;
+};
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_HIDDEN
+
+#endif // nsMsgIncomingServer_h__

mailnews/base/util/nsMsgKeyArray.cpp

@@ -0,0 +1,77 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsMsgKeyArray.h"
+#include "nsMemory.h"
+
+NS_IMPL_ISUPPORTS(nsMsgKeyArray, nsIMsgKeyArray)
+
+nsMsgKeyArray::nsMsgKeyArray()
+{
+#ifdef DEBUG
+  m_sorted = false;
+#endif
+}
+
+nsMsgKeyArray::~nsMsgKeyArray()
+{
+}
+
+NS_IMETHODIMP nsMsgKeyArray::Sort()
+{
+#ifdef DEBUG
+  m_sorted = true;
+#endif
+  m_keys.Sort();
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::GetKeyAt(int32_t aIndex, nsMsgKey *aKey)
+{
+  NS_ENSURE_ARG_POINTER(aKey);
+  *aKey = m_keys[aIndex];
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::GetLength(uint32_t *aLength)
+{
+  NS_ENSURE_ARG_POINTER(aLength);
+  *aLength = m_keys.Length();
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::SetCapacity(uint32_t aCapacity)
+{
+  m_keys.SetCapacity(aCapacity);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::AppendElement(nsMsgKey aKey)
+{
+#ifdef DEBUG
+  NS_ASSERTION(!m_sorted || m_keys.Length() == 0 ||
+               aKey > m_keys[m_keys.Length() - 1],
+               "Inserting a new key at wrong position in a sorted key list!");
+#endif
+  m_keys.AppendElement(aKey);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::InsertElementSorted(nsMsgKey aKey)
+{
+  // Ths function should be removed after interfaces are not frozen for TB38.
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP nsMsgKeyArray::GetArray(uint32_t *aCount, nsMsgKey **aKeys)
+{
+  NS_ENSURE_ARG_POINTER(aCount);
+  NS_ENSURE_ARG_POINTER(aKeys);
+  *aCount = m_keys.Length();
+  *aKeys =
+    (nsMsgKey *) nsMemory::Clone(m_keys.begin(),
+                                 m_keys.Length() * sizeof(nsMsgKey));
+  return (*aKeys) ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
+}

mailnews/base/util/nsMsgKeyArray.h

@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgKeyArray_h__
+#define nsMsgKeyArray_h__
+
+#include "nsIMsgKeyArray.h"
+#include "nsTArray.h"
+
+/*
+ * This class is a thin wrapper around an nsTArray<nsMsgKey>
+ */
+class nsMsgKeyArray : public nsIMsgKeyArray
+{
+public:
+  nsMsgKeyArray();
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGKEYARRAY
+
+  nsTArray<nsMsgKey> m_keys;
+
+private:
+  virtual ~nsMsgKeyArray();
+
+#ifdef DEBUG
+  bool m_sorted;
+#endif
+};
+
+#endif

mailnews/base/util/nsMsgKeySet.h

@@ -0,0 +1,108 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMsgKeySet_H_
+#define _nsMsgKeySet_H_
+
+#include "msgCore.h"
+#include "nsTArray.h"
+
+// nsMsgKeySet represents a set of articles.  Typically, it is the set of
+// read articles from a .newsrc file, but it can be used for other purposes
+// too.
+
+#if 0
+// If a MSG_NewsHost* is supplied to the creation routine, then that
+// MSG_NewsHost will be notified whenever a change is made to set.
+class MSG_NewsHost;
+#endif
+
+class NS_MSG_BASE nsMsgKeySet {
+public:
+  // Creates an empty set.
+  static nsMsgKeySet* Create(/* MSG_NewsHost* host = NULL*/);
+
+  // Creates a set from the list of numbers, as might be found in a
+  // newsrc file.
+  static nsMsgKeySet* Create(const char* str/* , MSG_NewsHost* host = NULL*/);
+  ~nsMsgKeySet();
+  
+  // FirstNonMember() returns the lowest non-member of the set that is
+  // greater than 0.
+  int32_t FirstNonMember();
+
+  // Output() converts to a string representation suitable for writing to a
+  // .newsrc file.  
+  nsresult Output(char **outputStr);    
+
+  // IsMember() returns whether the given article is a member of this set.
+  bool IsMember(int32_t art);
+  
+  // Add() adds the given article to the set.  (Returns 1 if a change was
+  // made, 0 if it was already there, and negative on error.)
+  int Add(int32_t art);
+  
+  // Remove() removes the given article from the set. 
+  int Remove(int32_t art);
+  
+  // AddRange() adds the (inclusive) given range of articles to the set.
+  int AddRange(int32_t first, int32_t last);
+  
+  // CountMissingInRange() takes an inclusive range of articles and returns
+  // the number of articles in that range which are not in the set.
+  int32_t CountMissingInRange(int32_t start, int32_t end);
+  
+  // FirstMissingRange() takes an inclusive range and finds the first range
+  // of articles that are not in the set.  If none, return zeros. 
+  int FirstMissingRange(int32_t min, int32_t max, int32_t* first, int32_t* last);
+  
+
+  // LastMissingRange() takes an inclusive range and finds the last range
+  // of articles that are not in the set.  If none, return zeros. 
+  int LastMissingRange(int32_t min, int32_t max, int32_t* first, int32_t* last);
+  
+  int32_t GetLastMember();
+  int32_t GetFirstMember();
+  void  SetLastMember(int32_t highWaterMark);
+  // For debugging only...
+  int32_t getLength() {return m_length;}
+
+/**
+ * Fill the passed in aArray with the keys in the message key set.
+ */
+  nsresult ToMsgKeyArray(nsTArray<nsMsgKey> &aArray);
+
+#ifdef DEBUG
+  static void RunTests();
+#endif
+
+protected:
+  nsMsgKeySet(/* MSG_NewsHost* host */);
+  nsMsgKeySet(const char* /* , MSG_NewsHost* host */);
+  bool Grow();
+  bool Optimize();
+  
+#ifdef DEBUG
+  static void test_decoder(const char*);
+  static void test_adder();
+  static void test_ranges();
+  static void test_member(bool with_cache);
+#endif
+  
+  int32_t *m_data;          /* the numbers composing the `chunks' */
+  int32_t m_data_size;        /* size of that malloc'ed block */
+  int32_t m_length;        /* active area */
+  
+  int32_t m_cached_value;      /* a potential set member, or -1 if unset*/
+  int32_t m_cached_value_index;    /* the index into `data' at which a search
+                     to determine whether `cached_value' was
+                     a member of the set ended. */
+#ifdef NEWSRC_DOES_HOST_STUFF
+  MSG_NewsHost* m_host;
+#endif
+};
+
+
+#endif /* _nsMsgKeySet_H_ */

mailnews/base/util/nsMsgLineBuffer.cpp

@@ -0,0 +1,441 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "prlog.h"
+#include "prmem.h"
+#include "nsMsgLineBuffer.h"
+#include "nsAlgorithm.h"
+#include "nsMsgUtils.h"
+#include "nsIInputStream.h" // used by nsMsgLineStreamBuffer
+#include <algorithm>
+
+nsByteArray::nsByteArray()
+{
+  MOZ_COUNT_CTOR(nsByteArray);
+  m_buffer = NULL;
+  m_bufferSize = 0;
+  m_bufferPos = 0;
+}
+
+nsByteArray::~nsByteArray()
+{
+  MOZ_COUNT_DTOR(nsByteArray);
+  PR_FREEIF(m_buffer);
+}
+
+nsresult nsByteArray::GrowBuffer(uint32_t desired_size, uint32_t quantum)
+{
+  if (m_bufferSize < desired_size)
+  {
+    char *new_buf;
+    uint32_t increment = desired_size - m_bufferSize;
+    if (increment < quantum) /* always grow by a minimum of N bytes */
+      increment = quantum;
+    
+    
+    new_buf = (m_buffer
+      ? (char *) PR_REALLOC (m_buffer, (m_bufferSize + increment))
+      : (char *) PR_MALLOC (m_bufferSize + increment));
+    if (! new_buf)
+      return NS_ERROR_OUT_OF_MEMORY;
+    m_buffer = new_buf;
+    m_bufferSize += increment;
+  }
+  return NS_OK;
+}
+
+nsresult nsByteArray::AppendString(const char *string)
+{
+  uint32_t strLength = (string) ? PL_strlen(string) : 0;
+  return AppendBuffer(string, strLength);
+  
+}
+
+nsresult nsByteArray::AppendBuffer(const char *buffer, uint32_t length)
+{
+  nsresult ret = NS_OK;
+  if (m_bufferPos + length > m_bufferSize)
+    ret = GrowBuffer(m_bufferPos + length, 1024);
+  if (NS_SUCCEEDED(ret))
+  {
+    memcpy(m_buffer + m_bufferPos, buffer, length);
+    m_bufferPos += length;
+  }
+  return ret;
+}
+
+nsMsgLineBuffer::nsMsgLineBuffer(nsMsgLineBufferHandler *handler, bool convertNewlinesP)
+{
+  MOZ_COUNT_CTOR(nsMsgLineBuffer);
+  m_handler = handler;
+  m_convertNewlinesP = convertNewlinesP;
+  m_lookingForCRLF = true;
+}
+
+nsMsgLineBuffer::~nsMsgLineBuffer()
+{
+  MOZ_COUNT_DTOR(nsMsgLineBuffer);
+}
+
+void
+nsMsgLineBuffer::SetLookingForCRLF(bool b)
+{
+  m_lookingForCRLF = b;
+}
+
+nsresult nsMsgLineBuffer::BufferInput(const char *net_buffer, int32_t net_buffer_size)
+{
+    nsresult status = NS_OK;
+    if (m_bufferPos > 0 && m_buffer && m_buffer[m_bufferPos - 1] == '\r' &&
+        net_buffer_size > 0 && net_buffer[0] != '\n') {
+        /* The last buffer ended with a CR.  The new buffer does not start
+           with a LF.  This old buffer should be shipped out and discarded. */
+        PR_ASSERT(m_bufferSize > m_bufferPos);
+        if (m_bufferSize <= m_bufferPos)
+          return NS_ERROR_UNEXPECTED;
+        if (NS_FAILED(ConvertAndSendBuffer()))
+           return NS_ERROR_FAILURE;
+        m_bufferPos = 0;
+    }
+    while (net_buffer_size > 0)
+    {
+        const char *net_buffer_end = net_buffer + net_buffer_size;
+        const char *newline = 0;
+        const char *s;
+
+        for (s = net_buffer; s < net_buffer_end; s++)
+        {
+          if (m_lookingForCRLF) {
+            /* Move forward in the buffer until the first newline.
+               Stop when we see CRLF, CR, or LF, or the end of the buffer.
+               *But*, if we see a lone CR at the *very end* of the buffer,
+               treat this as if we had reached the end of the buffer without
+               seeing a line terminator.  This is to catch the case of the
+               buffers splitting a CRLF pair, as in "FOO\r\nBAR\r" "\nBAZ\r\n".
+            */
+            if (*s == '\r' || *s == '\n') {
+              newline = s;
+              if (newline[0] == '\r') {
+                if (s == net_buffer_end - 1) {
+                  /* CR at end - wait for the next character. */
+                  newline = 0;
+                  break;
+                }
+                else if (newline[1] == '\n') {
+                  /* CRLF seen; swallow both. */
+                  newline++;
+                }
+              }
+              newline++;
+              break;
+            }
+          }
+          else {
+            /* if not looking for a CRLF, stop at CR or LF.  (for example, when parsing the newsrc file).  this fixes #9896, where we'd lose the last line of anything we'd parse that used CR as the line break. */
+            if (*s == '\r' || *s == '\n') {
+              newline = s;
+              newline++;
+              break;
+            }
+          }
+        }
+        
+        /* Ensure room in the net_buffer and append some or all of the current
+           chunk of data to it. */
+        {
+            const char *end = (newline ? newline : net_buffer_end);
+            uint32_t desired_size = (end - net_buffer) + m_bufferPos + 1;
+            
+            if (desired_size >= m_bufferSize)
+            {
+                status = GrowBuffer (desired_size, 1024);
+                if (NS_FAILED(status))
+                    return status;
+            }
+            memcpy (m_buffer + m_bufferPos, net_buffer, (end - net_buffer));
+            m_bufferPos += (end - net_buffer);
+        }
+        
+        /* Now m_buffer contains either a complete line, or as complete
+           a line as we have read so far.
+           
+           If we have a line, process it, and then remove it from `m_buffer'.
+           Then go around the loop again, until we drain the incoming data.
+           */
+        if (!newline)
+            return NS_OK;
+
+        if (NS_FAILED(ConvertAndSendBuffer()))
+          return NS_ERROR_FAILURE;
+        
+        net_buffer_size -= (newline - net_buffer);
+        net_buffer = newline;
+        m_bufferPos = 0;
+    }
+    return NS_OK;
+}
+
+nsresult nsMsgLineBuffer::HandleLine(const char *line, uint32_t line_length)
+{
+  NS_ASSERTION(false, "must override this method if you don't provide a handler");
+  return NS_OK;
+}
+
+nsresult nsMsgLineBuffer::ConvertAndSendBuffer()
+{
+    /* Convert the line terminator to the native form.
+     */
+
+    char *buf = m_buffer;
+    int32_t length = m_bufferPos;
+
+    char* newline;
+    
+    PR_ASSERT(buf && length > 0);
+    if (!buf || length <= 0) 
+        return NS_ERROR_FAILURE;
+    newline = buf + length;
+    
+    PR_ASSERT(newline[-1] == '\r' || newline[-1] == '\n');
+    if (newline[-1] != '\r' && newline[-1] != '\n')
+        return NS_ERROR_FAILURE;
+    
+    if (m_convertNewlinesP)
+    {
+#if (MSG_LINEBREAK_LEN == 1)
+      if ((newline - buf) >= 2 &&
+           newline[-2] == '\r' &&
+           newline[-1] == '\n')
+      {
+        /* CRLF -> CR or LF */
+        buf [length - 2] = MSG_LINEBREAK[0];
+        length--;
+      }
+      else if (newline > buf + 1 &&
+             newline[-1] != MSG_LINEBREAK[0])
+      {
+        /* CR -> LF or LF -> CR */
+        buf [length - 1] = MSG_LINEBREAK[0];
+      }
+#else
+      if (((newline - buf) >= 2 && newline[-2] != '\r') ||
+               ((newline - buf) >= 1 && newline[-1] != '\n'))
+      {
+        /* LF -> CRLF or CR -> CRLF */
+        length++;
+        buf[length - 2] = MSG_LINEBREAK[0];
+        buf[length - 1] = MSG_LINEBREAK[1];
+      }
+#endif
+    }    
+    return (m_handler) ? m_handler->HandleLine(buf, length) : HandleLine(buf, length);
+}
+
+// If there's still some data (non CRLF terminated) flush it out
+nsresult nsMsgLineBuffer::FlushLastLine()
+{
+  char *buf = m_buffer + m_bufferPos;
+  int32_t length = m_bufferPos - 1;
+  if (length > 0)
+    return (m_handler) ? m_handler->HandleLine(buf, length) : HandleLine(buf, length);
+  else
+    return NS_OK;
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// This is a utility class used to efficiently extract lines from an input stream by buffering
+// read but unprocessed stream data in a buffer. 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+nsMsgLineStreamBuffer::nsMsgLineStreamBuffer(uint32_t aBufferSize, bool aAllocateNewLines, bool aEatCRLFs, char aLineToken) 
+           : m_eatCRLFs(aEatCRLFs), m_allocateNewLines(aAllocateNewLines), m_lineToken(aLineToken)
+{
+  NS_PRECONDITION(aBufferSize > 0, "invalid buffer size!!!");
+  m_dataBuffer = nullptr;
+  m_startPos = 0;
+    m_numBytesInBuffer = 0;
+
+  // used to buffer incoming data by ReadNextLineFromInput
+  if (aBufferSize > 0)
+  {
+    m_dataBuffer = (char *) PR_CALLOC(sizeof(char) * aBufferSize);
+  }
+
+  m_dataBufferSize = aBufferSize;
+}
+
+nsMsgLineStreamBuffer::~nsMsgLineStreamBuffer()
+{
+  PR_FREEIF(m_dataBuffer); // release our buffer...
+}
+
+
+nsresult nsMsgLineStreamBuffer::GrowBuffer(int32_t desiredSize)
+{
+  char* newBuffer = (char *) PR_REALLOC(m_dataBuffer, desiredSize);
+  NS_ENSURE_TRUE(newBuffer, NS_ERROR_OUT_OF_MEMORY);
+  m_dataBuffer = newBuffer;
+  m_dataBufferSize = desiredSize;
+  return NS_OK;
+}
+
+void nsMsgLineStreamBuffer::ClearBuffer()
+{
+  m_startPos = 0;
+  m_numBytesInBuffer = 0;
+}
+
+// aInputStream - the input stream we want to read a line from
+// aPauseForMoreData is returned as true if the stream does not yet contain a line and we must wait for more
+// data to come into the stream.
+// Note to people wishing to modify this function: Be *VERY CAREFUL* this is a critical function used by all of
+// our mail protocols including imap, nntp, and pop. If you screw it up, you could break a lot of stuff.....
+
+char * nsMsgLineStreamBuffer::ReadNextLine(nsIInputStream * aInputStream, uint32_t &aNumBytesInLine, bool &aPauseForMoreData, nsresult *prv, bool addLineTerminator)
+{
+  // try to extract a line from m_inputBuffer. If we don't have an entire line, 
+  // then read more bytes out from the stream. If the stream is empty then wait
+  // on the monitor for more data to come in.
+  
+  NS_PRECONDITION(m_dataBuffer && m_dataBufferSize > 0, "invalid input arguments for read next line from input");
+
+  if (prv)
+    *prv = NS_OK;
+  // initialize out values
+  aPauseForMoreData = false;
+  aNumBytesInLine = 0;
+  char * endOfLine = nullptr;
+  char * startOfLine = m_dataBuffer+m_startPos;
+  
+  if (m_numBytesInBuffer > 0) // any data in our internal buffer?
+    endOfLine = PL_strchr(startOfLine, m_lineToken); // see if we already have a line ending...
+  
+  // it's possible that we got here before the first time we receive data from the server
+  // so aInputStream will be nullptr...
+  if (!endOfLine && aInputStream) // get some more data from the server
+  {
+    nsresult rv;
+    uint64_t numBytesInStream = 0;
+    uint32_t numBytesCopied = 0;
+    bool nonBlockingStream;
+    aInputStream->IsNonBlocking(&nonBlockingStream);
+    rv = aInputStream->Available(&numBytesInStream);
+    if (NS_FAILED(rv))
+    {
+      if (prv)
+        *prv = rv;
+      aNumBytesInLine = -1;
+      return nullptr;
+    }
+    if (!nonBlockingStream && numBytesInStream == 0) // if no data available,
+      numBytesInStream = m_dataBufferSize / 2; // ask for half the data buffer size.
+
+    // if the number of bytes we want to read from the stream, is greater than the number
+    // of bytes left in our buffer, then we need to shift the start pos and its contents
+    // down to the beginning of m_dataBuffer...
+    uint32_t numFreeBytesInBuffer = m_dataBufferSize - m_startPos - m_numBytesInBuffer;
+    if (numBytesInStream >= numFreeBytesInBuffer)
+    {
+      if (m_startPos)
+      {
+        memmove(m_dataBuffer, startOfLine, m_numBytesInBuffer);
+        // make sure the end of the buffer is terminated
+        m_dataBuffer[m_numBytesInBuffer] = '\0';
+        m_startPos = 0;
+        startOfLine = m_dataBuffer;
+        numFreeBytesInBuffer = m_dataBufferSize - m_numBytesInBuffer;
+        //printf("moving data in read line around because buffer filling up\n");
+      }
+      // If we didn't make enough space (or any), grow the buffer
+      if (numBytesInStream >= numFreeBytesInBuffer)
+      {
+        int64_t growBy = (numBytesInStream - numFreeBytesInBuffer) * 2 + 1;
+        // GrowBuffer cannot handles over 4GB size
+        if (m_dataBufferSize + growBy > PR_UINT32_MAX)
+          return nullptr;
+        // try growing buffer by twice as much as we need.
+        nsresult rv = GrowBuffer(m_dataBufferSize + growBy);
+        // if we can't grow the buffer, we have to bail.
+        if (NS_FAILED(rv))
+          return nullptr;
+        startOfLine = m_dataBuffer;
+        numFreeBytesInBuffer += growBy;
+      }
+      NS_ASSERTION(m_startPos == 0, "m_startPos should be 0 .....\n");
+    }
+    
+    uint32_t numBytesToCopy = std::min(uint64_t(numFreeBytesInBuffer - 1) /* leave one for a null terminator */, numBytesInStream);
+    if (numBytesToCopy > 0)
+    {
+      // read the data into the end of our data buffer
+      char *startOfNewData = startOfLine + m_numBytesInBuffer;
+      rv = aInputStream->Read(startOfNewData, numBytesToCopy, &numBytesCopied);
+      if (prv)
+        *prv = rv;
+      uint32_t i;
+      for (i = 0; i < numBytesCopied; i++)  // replace nulls with spaces
+      {
+        if (!startOfNewData[i])
+          startOfNewData[i] = ' ';
+      }
+      m_numBytesInBuffer += numBytesCopied;
+      m_dataBuffer[m_startPos + m_numBytesInBuffer] = '\0';
+
+      // okay, now that we've tried to read in more data from the stream,
+      // look for another end of line character in the new data
+      endOfLine = PL_strchr(startOfNewData, m_lineToken);
+    }
+  }
+  
+  // okay, now check again for endOfLine.
+  if (endOfLine)
+  {
+    if (!m_eatCRLFs)
+      endOfLine += 1; // count for LF or CR
+    
+    aNumBytesInLine = endOfLine - startOfLine;
+    
+    if (m_eatCRLFs && aNumBytesInLine > 0 && startOfLine[aNumBytesInLine-1] == '\r') // Remove the CR in a CRLF sequence
+      aNumBytesInLine--;
+    
+    // PR_CALLOC zeros out the allocated line
+    char* newLine = (char*) PR_CALLOC(aNumBytesInLine + (addLineTerminator ? MSG_LINEBREAK_LEN : 0) + 1);
+    if (!newLine)
+    {
+      aNumBytesInLine = 0;
+      aPauseForMoreData = true;
+      return nullptr;
+    }
+    
+    memcpy(newLine, startOfLine, aNumBytesInLine); // copy the string into the new line buffer
+    if (addLineTerminator)
+    {
+      memcpy(newLine + aNumBytesInLine, MSG_LINEBREAK, MSG_LINEBREAK_LEN);
+      aNumBytesInLine += MSG_LINEBREAK_LEN;
+    }
+    
+    if (m_eatCRLFs)
+      endOfLine += 1; // advance past LF or CR if we haven't already done so...
+    
+    // now we need to update the data buffer to go past the line we just read out. 
+    m_numBytesInBuffer -= (endOfLine - startOfLine);
+    if (m_numBytesInBuffer)
+      m_startPos = endOfLine - m_dataBuffer;
+    else
+      m_startPos = 0;
+    
+    return newLine;
+  }
+  
+  aPauseForMoreData = true;
+  return nullptr; // if we somehow got here. we don't have another line in the buffer yet...need to wait for more data...
+}
+
+bool nsMsgLineStreamBuffer::NextLineAvailable()
+{
+  return (m_numBytesInBuffer > 0 && PL_strchr(m_dataBuffer+m_startPos, m_lineToken));
+}
+

mailnews/base/util/nsMsgLineBuffer.h

@@ -0,0 +1,107 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef _nsMsgLineBuffer_H
+#define _nsMsgLineBuffer_H
+
+#include "msgCore.h"    // precompiled header...
+
+// I can't believe I have to have this stupid class, but I can't find
+// anything suitable (nsStrImpl might be, when it's done). nsIByteBuffer
+// would do, if I had a stream for input, which I don't.
+
+class NS_MSG_BASE nsByteArray 
+{
+public:
+  nsByteArray();
+  virtual ~nsByteArray();
+  uint32_t  GetSize() {return m_bufferSize;}
+  uint32_t  GetBufferPos() {return m_bufferPos;}
+  nsresult  GrowBuffer(uint32_t desired_size, uint32_t quantum = 1024);
+  nsresult  AppendString(const char *string);
+  nsresult  AppendBuffer(const char *buffer, uint32_t length);
+  void    ResetWritePos() {m_bufferPos = 0;}
+  char    *GetBuffer() {return m_buffer;}
+protected:
+  char    *m_buffer;
+  uint32_t  m_bufferSize;
+  uint32_t  m_bufferPos;  // write Pos in m_buffer - where the next byte should go.
+};
+
+
+class NS_MSG_BASE nsMsgLineBufferHandler : public nsByteArray
+{
+public:
+  virtual nsresult HandleLine(const char *line, uint32_t line_length) = 0;
+};
+
+class NS_MSG_BASE nsMsgLineBuffer : public nsMsgLineBufferHandler
+{
+public:
+  nsMsgLineBuffer(nsMsgLineBufferHandler *handler, bool convertNewlinesP);
+  
+  virtual    ~nsMsgLineBuffer();
+  nsresult    BufferInput(const char *net_buffer, int32_t net_buffer_size);
+  // Not sure why anyone cares, by NNTPHost seems to want to know the buf pos.
+  uint32_t    GetBufferPos() {return m_bufferPos;}
+  
+  virtual nsresult HandleLine(const char *line, uint32_t line_length);
+  // flush last line, though it won't be CRLF terminated.
+  virtual nsresult FlushLastLine();
+protected:
+  nsMsgLineBuffer(bool convertNewlinesP);
+  
+  nsresult ConvertAndSendBuffer();
+  void SetLookingForCRLF(bool b);
+  
+  nsMsgLineBufferHandler *m_handler;
+  bool        m_convertNewlinesP;
+  bool        m_lookingForCRLF; 
+};
+
+// I'm adding this utility class here for lack of a better place. This utility class is similar to nsMsgLineBuffer
+// except it works from an input stream. It is geared towards efficiently parsing new lines out of a stream by storing
+// read but unprocessed bytes in a buffer. I envision the primary use of this to be our mail protocols such as imap, news and
+// pop which need to process line by line data being returned in the form of a proxied stream from the server.
+
+class nsIInputStream;
+
+class NS_MSG_BASE nsMsgLineStreamBuffer
+{
+public:
+  // aBufferSize -- size of the buffer you want us to use for buffering stream data
+  // aEndOfLinetoken -- The delimiter string to be used for determining the end of line. This 
+  //              allows us to parse platform specific end of line endings by making it
+  //            a parameter.
+  // aAllocateNewLines -- true if you want calls to ReadNextLine to allocate new memory for the line. 
+  //            if false, the char * returned is just a ptr into the buffer. Subsequent calls to
+  //            ReadNextLine will alter the data so your ptr only has a life time of a per call.
+  // aEatCRLFs  -- true if you don't want to see the CRLFs on the lines returned by ReadNextLine. 
+  //         false if you do want to see them.
+  // aLineToken -- Specify the line token to look for, by default is LF ('\n') which cover as well CRLF. If
+  //            lines are terminated with a CR only, you need to set aLineToken to CR ('\r')
+  nsMsgLineStreamBuffer(uint32_t aBufferSize, bool aAllocateNewLines, 
+                        bool aEatCRLFs = true, char aLineToken = '\n'); // specify the size of the buffer you want the class to use....
+  virtual ~nsMsgLineStreamBuffer();
+  
+  // Caller must free the line returned using PR_Free
+  // aEndOfLinetoken -- delimiter used to denote the end of a line.
+  // aNumBytesInLine -- The number of bytes in the line returned
+  // aPauseForMoreData -- There is not enough data in the stream to make a line at this time...
+  char * ReadNextLine(nsIInputStream * aInputStream, uint32_t &anumBytesInLine, bool &aPauseForMoreData, nsresult *rv = nullptr, bool addLineTerminator = false);
+  nsresult GrowBuffer(int32_t desiredSize);
+  void ClearBuffer();
+  bool NextLineAvailable();
+protected:
+  bool m_eatCRLFs;
+  bool m_allocateNewLines;
+  char * m_dataBuffer;
+  uint32_t m_dataBufferSize;
+  uint32_t m_startPos;
+  uint32_t m_numBytesInBuffer;
+  char m_lineToken;
+};
+
+
+#endif

mailnews/base/util/nsMsgMailNewsUrl.h

@@ -0,0 +1,87 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgMailNewsUrl_h___
+#define nsMsgMailNewsUrl_h___
+
+#include "nscore.h"
+#include "nsISupports.h"
+#include "nsIUrlListener.h"
+#include "nsTObserverArray.h"
+#include "nsIMsgWindow.h"
+#include "nsIMsgStatusFeedback.h"
+#include "nsCOMPtr.h"
+#include "nsCOMArray.h"
+#include "nsIMimeHeaders.h"
+#include "nsIMsgMailNewsUrl.h"
+#include "nsIURL.h"
+#include "nsIURIWithPrincipal.h"
+#include "nsILoadGroup.h"
+#include "nsIMsgSearchSession.h"
+#include "nsICacheEntry.h"
+#include "nsICacheSession.h"
+#include "nsIMimeMiscStatus.h"
+#include "nsWeakReference.h"
+#include "nsStringGlue.h"
+
+///////////////////////////////////////////////////////////////////////////////////
+// Okay, I found that all of the mail and news url interfaces needed to support
+// several common interfaces (in addition to those provided through nsIURI). 
+// So I decided to group them all in this implementation so we don't have to
+// duplicate the code.
+//
+//////////////////////////////////////////////////////////////////////////////////
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+class NS_MSG_BASE nsMsgMailNewsUrl : public nsIMsgMailNewsUrl,
+                                     public nsIURIWithPrincipal
+{
+public:
+    nsMsgMailNewsUrl();
+
+    NS_DECL_THREADSAFE_ISUPPORTS
+    NS_DECL_NSIMSGMAILNEWSURL
+    NS_DECL_NSIURI
+    NS_DECL_NSIURIWITHQUERY
+    NS_DECL_NSIURL
+    NS_DECL_NSIURIWITHPRINCIPAL
+
+protected:
+  virtual ~nsMsgMailNewsUrl();
+
+  nsCOMPtr<nsIURL> m_baseURL;
+  nsCOMPtr<nsIPrincipal> m_principal;
+  nsWeakPtr m_statusFeedbackWeak;
+  nsWeakPtr m_msgWindowWeak;
+  nsWeakPtr m_loadGroupWeak;
+  nsCOMPtr<nsIMimeHeaders> mMimeHeaders;
+  nsCOMPtr<nsIMsgSearchSession> m_searchSession;
+  nsCOMPtr<nsICacheEntry> m_memCacheEntry;
+  nsCOMPtr<nsIMsgHeaderSink> mMsgHeaderSink;
+  char *m_errorMessage;
+  int64_t mMaxProgress;
+  bool m_runningUrl;
+  bool m_updatingFolder;
+  bool m_msgIsInLocalCache;
+  bool m_suppressErrorMsgs;
+  bool m_isPrincipalURL;
+
+  // the following field is really a bit of a hack to make 
+  // open attachments work. The external applications code sometimes tries to figure out the right
+  // handler to use by looking at the file extension of the url we are trying to load. Unfortunately,
+  // the attachment file name really isn't part of the url string....so we'll store it here...and if 
+  // the url we are running is an attachment url, we'll set it here. Then when the helper apps code
+  // asks us for it, we'll return the right value.
+  nsCString mAttachmentFileName;
+
+  nsTObserverArray<nsCOMPtr<nsIUrlListener> > mUrlListeners;
+};
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_HIDDEN
+
+#endif /* nsMsgMailNewsUrl_h___ */

mailnews/base/util/nsMsgProtocol.h

@@ -0,0 +1,239 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgProtocol_h__
+#define nsMsgProtocol_h__
+
+#include "mozilla/Attributes.h"
+#include "nsIStreamListener.h"
+#include "nsIInputStream.h"
+#include "nsIOutputStream.h"
+#include "nsIChannel.h"
+#include "nsIURL.h"
+#include "nsIThread.h"
+#include "nsILoadGroup.h"
+#include "nsIFile.h"
+#include "nsCOMPtr.h"
+#include "nsIInterfaceRequestor.h"
+#include "nsIInterfaceRequestorUtils.h"
+#include "nsIProgressEventSink.h"
+#include "nsITransport.h"
+#include "nsIAsyncOutputStream.h"
+#include "nsIAuthModule.h"
+#include "nsStringGlue.h"
+#include "nsWeakReference.h"
+
+class nsIMsgWindow;
+class nsIPrompt;
+class nsIMsgMailNewsUrl;
+class nsMsgFilePostHelper;
+class nsIProxyInfo;
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+// This is a helper class used to encapsulate code shared between all of the
+// mailnews protocol objects (imap, news, pop, smtp, etc.) In particular,
+// it unifies the core networking code for the protocols. My hope is that
+// this will make unification with Necko easier as we'll only have to change
+// this class and not all of the mailnews protocols.
+class NS_MSG_BASE nsMsgProtocol : public nsIStreamListener
+                                , public nsIChannel
+                                , public nsITransportEventSink
+{
+public:
+  nsMsgProtocol(nsIURI * aURL);
+
+  NS_DECL_THREADSAFE_ISUPPORTS
+  // nsIChannel support
+  NS_DECL_NSICHANNEL
+  NS_DECL_NSIREQUEST
+  
+  NS_DECL_NSISTREAMLISTENER
+  NS_DECL_NSIREQUESTOBSERVER
+  NS_DECL_NSITRANSPORTEVENTSINK
+
+  // LoadUrl -- A protocol typically overrides this function, sets up any local state for the url and
+  // then calls the base class which opens the socket if it needs opened. If the socket is 
+  // already opened then we just call ProcessProtocolState to start the churning process.
+  // aConsumer is the consumer for the url. It can be null if this argument is not appropriate
+  virtual nsresult LoadUrl(nsIURI * aURL, nsISupports * aConsumer = nullptr);
+
+  virtual nsresult SetUrl(nsIURI * aURL); // sometimes we want to set the url before we load it
+  void ShowAlertMessage(nsIMsgMailNewsUrl *aMsgUrl, nsresult aStatus);
+
+  // Flag manipulators
+  virtual bool TestFlag  (uint32_t flag) {return flag & m_flags;}
+  virtual void   SetFlag   (uint32_t flag) { m_flags |= flag; }
+  virtual void   ClearFlag (uint32_t flag) { m_flags &= ~flag; }
+
+protected:
+  virtual ~nsMsgProtocol();
+
+  // methods for opening and closing a socket with core netlib....
+  // mscott -okay this is lame. I should break this up into a file protocol and a socket based
+  // protocool class instead of cheating and putting both methods here...
+
+  // open a connection with a specific host and port
+  // aHostName must be UTF-8 encoded.
+  virtual nsresult OpenNetworkSocketWithInfo(const char * aHostName,
+                                             int32_t aGetPort,
+                                             const char *connectionType,
+                                             nsIProxyInfo *aProxyInfo,
+                                             nsIInterfaceRequestor* callbacks);
+  // helper routine
+  nsresult GetFileFromURL(nsIURI * aURL, nsIFile **aResult);
+  virtual nsresult OpenFileSocket(nsIURI * aURL, uint32_t aStartPosition, int32_t aReadCount); // used to open a file socket connection
+
+  nsresult GetTopmostMsgWindow(nsIMsgWindow **aWindow);
+
+  virtual const char* GetType() {return nullptr;}
+  nsresult GetQoSBits(uint8_t *aQoSBits);
+
+  // a Protocol typically overrides this method. They free any of their own connection state and then
+  // they call up into the base class to free the generic connection objects
+  virtual nsresult CloseSocket(); 
+
+  virtual nsresult SetupTransportState(); // private method used by OpenNetworkSocket and OpenFileSocket
+
+  // ProcessProtocolState - This is the function that gets churned by calls to OnDataAvailable. 
+  // As data arrives on the socket, OnDataAvailable calls ProcessProtocolState.
+  
+  virtual nsresult ProcessProtocolState(nsIURI * url, nsIInputStream * inputStream, 
+                                        uint64_t sourceOffset, uint32_t length) = 0;
+
+  // SendData -- Writes the data contained in dataBuffer into the current output stream. 
+  // It also informs the transport layer that this data is now available for transmission.
+  // Returns a positive number for success, 0 for failure (not all the bytes were written to the
+  // stream, etc). 
+    // aSuppressLogging is a hint that sensitive data is being sent and should not be logged
+  virtual nsresult SendData(const char * dataBuffer, bool aSuppressLogging = false);
+
+  virtual nsresult PostMessage(nsIURI* url, nsIFile* aPostFile);
+
+  virtual nsresult InitFromURI(nsIURI *aUrl);
+
+  nsresult DoNtlmStep1(const char *username, const char *password, nsCString &response);
+  nsresult DoNtlmStep2(nsCString &commandResponse, nsCString &response);
+
+  nsresult DoGSSAPIStep1(const char *service, const char *username, nsCString &response);
+  nsresult DoGSSAPIStep2(nsCString &commandResponse, nsCString &response);
+  // Ouput stream for writing commands to the socket  
+  nsCOMPtr<nsIOutputStream>   m_outputStream;   // this will be obtained from the transport interface
+  nsCOMPtr<nsIInputStream>    m_inputStream;
+
+  // Ouput stream for writing commands to the socket
+  nsCOMPtr<nsITransport>  m_transport; 
+  nsCOMPtr<nsIRequest>    m_request;
+
+  bool          m_socketIsOpen; // mscott: we should look into keeping this state in the nsSocketTransport...
+                                  // I'm using it to make sure I open the socket the first time a URL is loaded into the connection
+  uint32_t      m_flags; // used to store flag information
+  //uint32_t  m_startPosition;
+  int32_t       m_readCount;
+
+  nsCOMPtr<nsIFile> m_tempMsgFile;  // we currently have a hack where displaying a msg involves writing it to a temp file first
+
+  // auth module for access to NTLM functions
+  nsCOMPtr<nsIAuthModule> m_authModule;
+
+  // the following is a catch all for nsIChannel related data
+  nsCOMPtr<nsIURI>            m_originalUrl;  // the original url
+  nsCOMPtr<nsIURI>            m_url;          // the running url
+  nsCOMPtr<nsIStreamListener> m_channelListener;
+  nsCOMPtr<nsISupports>        m_channelContext;
+  nsCOMPtr<nsILoadGroup>      m_loadGroup;
+  nsLoadFlags                 mLoadFlags;
+  nsCOMPtr<nsIProgressEventSink> mProgressEventSink;
+  nsCOMPtr<nsIInterfaceRequestor> mCallbacks;
+  nsCOMPtr<nsISupports>       mOwner;
+  nsCString                   mContentType;
+  nsCString                   mCharset;
+  int64_t                     mContentLength;
+  nsCOMPtr<nsILoadInfo>       m_loadInfo;
+
+  nsCString m_lastPasswordSent; // used to prefill the password prompt
+
+  // private helper routine used by subclasses to quickly get a reference to the correct prompt dialog
+  // for a mailnews url. 
+  nsresult GetPromptDialogFromUrl(nsIMsgMailNewsUrl * aMsgUrl, nsIPrompt ** aPromptDialog);
+
+  // if a url isn't going to result in any content then we want to suppress calls to
+  // OnStartRequest, OnDataAvailable and OnStopRequest
+  bool mSuppressListenerNotifications;
+};
+
+
+// This is is a subclass of nsMsgProtocol extends the parent class with AsyncWrite support. Protocols like smtp
+// and news want to leverage aysnc write. We don't want everyone who inherits from nsMsgProtocol to have to
+// pick up the extra overhead.
+class NS_MSG_BASE nsMsgAsyncWriteProtocol : public nsMsgProtocol
+                                          , public nsSupportsWeakReference
+{
+public:
+  NS_DECL_ISUPPORTS_INHERITED
+
+  NS_IMETHOD Cancel(nsresult status) override;
+
+  nsMsgAsyncWriteProtocol(nsIURI * aURL);
+  
+  // temporary over ride...
+  virtual nsresult PostMessage(nsIURI* url, nsIFile *postFile) override;
+  
+  // over ride the following methods from the base class
+  virtual nsresult SetupTransportState() override;
+  virtual nsresult SendData(const char * dataBuffer, bool aSuppressLogging = false) override;
+  nsCString mAsyncBuffer;
+
+  // if we suspended the asynch write while waiting for more data to write then this will be TRUE
+  bool mSuspendedWrite;
+  nsCOMPtr<nsIRequest>     m_WriteRequest;
+  nsCOMPtr<nsIAsyncOutputStream>    mAsyncOutStream;
+  nsCOMPtr<nsIOutputStreamCallback> mProvider;
+  nsCOMPtr<nsIThread>               mProviderThread;
+
+  // because we are reading the post data in asychronously, it's possible that we aren't sending it 
+  // out fast enough and the reading gets blocked. The following set of state variables are used to 
+  // track this.
+  bool    mSuspendedRead;
+  bool    mInsertPeriodRequired; // do we need to insert a '.' as part of the unblocking process
+   
+  nsresult ProcessIncomingPostData(nsIInputStream *inStr, uint32_t count);
+  nsresult UnblockPostReader();
+  nsresult UpdateSuspendedReadBytes(uint32_t aNewBytes, bool aAddToPostPeriodByteCount);
+  nsresult PostDataFinished(); // this is so we'll send out a closing '.' and release any state related to the post
+
+
+  // these two routines are used to pause and resume our loading of the file containing the contents
+  // we are trying to post. We call these routines when we aren't sending the bits out fast enough
+  // to keep up with the file read. 
+  nsresult SuspendPostFileRead();
+  nsresult ResumePostFileRead(); 
+  nsresult UpdateSuspendedReadBytes(uint32_t aNewBytes); 
+  void UpdateProgress(uint32_t aNewBytes);
+  nsMsgFilePostHelper * mFilePostHelper; // needs to be a weak reference
+protected:
+  virtual ~nsMsgAsyncWriteProtocol();
+
+  // the streams for the pipe used to queue up data for the async write calls to the server.
+  // we actually re-use the same mOutStream variable in our parent class for the output
+  // stream to the socket channel. So no need for a new variable here.
+  nsCOMPtr<nsIInputStream>  mInStream;    
+  nsCOMPtr<nsIInputStream>  mPostDataStream;
+  uint32_t                  mSuspendedReadBytes;   // remaining # of bytes we need to read before   
+                                                   // the input stream becomes unblocked
+  uint32_t                  mSuspendedReadBytesPostPeriod; // # of bytes which need processed after we insert a '.' before 
+                                                           // the input stream becomes unblocked.
+  int64_t  mFilePostSize; // used for determining progress on posting files.
+  uint32_t  mNumBytesPosted; // used for deterimining progress on posting files 
+  bool      mGenerateProgressNotifications; // set during a post operation after we've started sending the post data...
+
+  virtual nsresult CloseSocket() override; 
+};
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_HIDDEN
+
+#endif /* nsMsgProtocol_h__ */

mailnews/base/util/nsMsgReadStateTxn.cpp

@@ -0,0 +1,66 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsMsgReadStateTxn.h"
+
+#include "nsIMutableArray.h"
+#include "nsIMsgHdr.h"
+#include "nsComponentManagerUtils.h"
+
+
+nsMsgReadStateTxn::nsMsgReadStateTxn()
+{
+}
+
+nsMsgReadStateTxn::~nsMsgReadStateTxn()
+{
+}
+
+nsresult
+nsMsgReadStateTxn::Init(nsIMsgFolder *aParentFolder,
+                        uint32_t aNumKeys,
+                        nsMsgKey *aMsgKeyArray)
+{
+  NS_ENSURE_ARG_POINTER(aParentFolder);
+
+  mParentFolder = aParentFolder;
+  mMarkedMessages.AppendElements(aMsgKeyArray, aNumKeys);
+
+  return nsMsgTxn::Init();
+}
+
+NS_IMETHODIMP 
+nsMsgReadStateTxn::UndoTransaction()
+{
+  return MarkMessages(false);
+}
+
+NS_IMETHODIMP 
+nsMsgReadStateTxn::RedoTransaction()
+{
+  return MarkMessages(true);
+}
+
+NS_IMETHODIMP
+nsMsgReadStateTxn::MarkMessages(bool aAsRead)
+{
+  nsresult rv;
+  nsCOMPtr<nsIMutableArray> messageArray = 
+    do_CreateInstance(NS_ARRAY_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  uint32_t length = mMarkedMessages.Length();
+  for (uint32_t i = 0; i < length; i++) {
+    nsCOMPtr<nsIMsgDBHdr> curMsgHdr;
+    rv = mParentFolder->GetMessageHeader(mMarkedMessages[i], 
+                                         getter_AddRefs(curMsgHdr));
+    if (NS_SUCCEEDED(rv) && curMsgHdr) {
+      messageArray->AppendElement(curMsgHdr, false);
+    }
+  }
+
+  return mParentFolder->MarkMessagesRead(messageArray, aAsRead);
+}
+

mailnews/base/util/nsMsgReadStateTxn.h

@@ -0,0 +1,48 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgBaseUndoTxn_h_
+#define nsMsgBaseUndoTxn_h_
+
+#include "mozilla/Attributes.h"
+#include "nsMsgTxn.h"
+#include "nsTArray.h"
+#include "nsCOMPtr.h"
+#include "MailNewsTypes.h"
+#include "nsIMsgFolder.h"
+
+
+#define NS_MSGREADSTATETXN_IID \
+{ /* 121FCE4A-3EA1-455C-8161-839E1557D0CF */ \
+  0x121FCE4A, 0x3EA1, 0x455C, \
+  { 0x81, 0x61, 0x83, 0x9E, 0x15, 0x57, 0xD0, 0xCF } \
+}
+
+
+//------------------------------------------------------------------------------
+// A mark-all transaction handler. Helper for redo/undo of message read states.
+//------------------------------------------------------------------------------
+class NS_MSG_BASE nsMsgReadStateTxn : public nsMsgTxn
+{
+public:
+  nsMsgReadStateTxn();
+  virtual ~nsMsgReadStateTxn();
+
+  nsresult Init(nsIMsgFolder *aParentFolder,
+                uint32_t aNumKeys,
+                nsMsgKey *aMsgKeyArray);
+  NS_IMETHOD UndoTransaction() override;
+  NS_IMETHOD RedoTransaction() override;
+
+protected:
+  NS_IMETHOD MarkMessages(bool aAsRead);
+
+private:
+  nsCOMPtr<nsIMsgFolder> mParentFolder;
+  nsTArray<nsMsgKey>     mMarkedMessages;
+};
+
+#endif  // nsMsgBaseUndoTxn_h_
+

mailnews/base/util/nsMsgTxn.cpp

@@ -0,0 +1,294 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsMsgTxn.h"
+#include "nsIMsgHdr.h"
+#include "nsIMsgDatabase.h"
+#include "nsCOMArray.h"
+#include "nsArrayEnumerator.h"
+#include "nsComponentManagerUtils.h"
+#include "nsVariant.h"
+#include "nsIProperty.h"
+#include "nsMsgMessageFlags.h"
+#include "nsIMsgFolder.h"
+
+NS_IMPL_ADDREF(nsMsgTxn)
+NS_IMPL_RELEASE(nsMsgTxn)
+NS_INTERFACE_MAP_BEGIN(nsMsgTxn)
+  NS_INTERFACE_MAP_ENTRY(nsIWritablePropertyBag)
+  NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsIPropertyBag, nsIWritablePropertyBag)
+  NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIWritablePropertyBag)
+  NS_INTERFACE_MAP_ENTRY(nsITransaction)
+  NS_INTERFACE_MAP_ENTRY(nsIPropertyBag2)
+  NS_INTERFACE_MAP_ENTRY(nsIWritablePropertyBag2)
+NS_INTERFACE_MAP_END
+
+nsMsgTxn::nsMsgTxn() 
+{
+  m_txnType = 0;
+}
+
+nsMsgTxn::~nsMsgTxn()
+{
+}
+
+nsresult nsMsgTxn::Init()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::HasKey(const nsAString& name, bool *aResult)
+{
+  *aResult = mPropertyHash.Get(name, nullptr);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::Get(const nsAString& name, nsIVariant* *_retval)
+{
+  mPropertyHash.Get(name, _retval);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::GetProperty(const nsAString& name, nsIVariant* * _retval)
+{
+  return mPropertyHash.Get(name, _retval) ? NS_OK : NS_ERROR_FAILURE;
+}
+
+NS_IMETHODIMP nsMsgTxn::SetProperty(const nsAString& name, nsIVariant *value)
+{
+  NS_ENSURE_ARG_POINTER(value);
+  mPropertyHash.Put(name, value);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::DeleteProperty(const nsAString& name)
+{
+  if (!mPropertyHash.Get(name, nullptr))
+    return NS_ERROR_FAILURE;
+
+  mPropertyHash.Remove(name);
+  return mPropertyHash.Get(name, nullptr) ? NS_ERROR_FAILURE : NS_OK;
+}
+
+//
+// nsMailSimpleProperty class and impl; used for GetEnumerator
+// This is same as nsSimpleProperty but for external API use.
+//
+
+class nsMailSimpleProperty final : public nsIProperty 
+{
+public:
+  nsMailSimpleProperty(const nsAString& aName, nsIVariant* aValue)
+      : mName(aName), mValue(aValue)
+  {
+  }
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIPROPERTY
+protected:
+  ~nsMailSimpleProperty() {}
+
+  nsString mName;
+  nsCOMPtr<nsIVariant> mValue;
+};
+
+NS_IMPL_ISUPPORTS(nsMailSimpleProperty, nsIProperty)
+
+NS_IMETHODIMP nsMailSimpleProperty::GetName(nsAString& aName)
+{
+  aName.Assign(mName);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMailSimpleProperty::GetValue(nsIVariant* *aValue)
+{
+  NS_IF_ADDREF(*aValue = mValue);
+  return NS_OK;
+}
+
+// end nsMailSimpleProperty
+
+NS_IMETHODIMP nsMsgTxn::GetEnumerator(nsISimpleEnumerator* *_retval)
+{
+  nsCOMArray<nsIProperty> propertyArray;
+  for (auto iter = mPropertyHash.Iter(); !iter.Done(); iter.Next()) {
+    nsMailSimpleProperty *sprop = new nsMailSimpleProperty(iter.Key(),
+                                                           iter.Data());
+    propertyArray.AppendObject(sprop);
+  }
+  return NS_NewArrayEnumerator(_retval, propertyArray);
+}
+
+#define IMPL_GETSETPROPERTY_AS(Name, Type) \
+NS_IMETHODIMP \
+nsMsgTxn::GetPropertyAs ## Name (const nsAString & prop, Type *_retval) \
+{ \
+    nsIVariant* v = mPropertyHash.GetWeak(prop); \
+    if (!v) \
+        return NS_ERROR_NOT_AVAILABLE; \
+    return v->GetAs ## Name(_retval); \
+} \
+\
+NS_IMETHODIMP \
+nsMsgTxn::SetPropertyAs ## Name (const nsAString & prop, Type value) \
+{ \
+    nsCOMPtr<nsIWritableVariant> var = new nsVariant(); \
+    var->SetAs ## Name(value); \
+    return SetProperty(prop, var); \
+}
+
+IMPL_GETSETPROPERTY_AS(Int32, int32_t)
+IMPL_GETSETPROPERTY_AS(Uint32, uint32_t)
+IMPL_GETSETPROPERTY_AS(Int64, int64_t)
+IMPL_GETSETPROPERTY_AS(Uint64, uint64_t)
+IMPL_GETSETPROPERTY_AS(Double, double)
+IMPL_GETSETPROPERTY_AS(Bool, bool)
+
+NS_IMETHODIMP nsMsgTxn::GetPropertyAsAString(const nsAString & prop, 
+                                             nsAString & _retval)
+{
+  nsIVariant* v = mPropertyHash.GetWeak(prop);
+  if (!v)
+    return NS_ERROR_NOT_AVAILABLE;
+  return v->GetAsAString(_retval);
+}
+
+NS_IMETHODIMP nsMsgTxn::GetPropertyAsACString(const nsAString & prop, 
+                                              nsACString & _retval)
+{
+  nsIVariant* v = mPropertyHash.GetWeak(prop);
+  if (!v)
+    return NS_ERROR_NOT_AVAILABLE;
+  return v->GetAsACString(_retval);
+}
+
+NS_IMETHODIMP nsMsgTxn::GetPropertyAsAUTF8String(const nsAString & prop, 
+                                                 nsACString & _retval)
+{
+  nsIVariant* v = mPropertyHash.GetWeak(prop);
+  if (!v)
+    return NS_ERROR_NOT_AVAILABLE;
+  return v->GetAsAUTF8String(_retval);
+}
+
+NS_IMETHODIMP nsMsgTxn::GetPropertyAsInterface(const nsAString & prop,
+                                               const nsIID & aIID,
+                                               void** _retval)
+{
+  nsIVariant* v = mPropertyHash.GetWeak(prop);
+  if (!v)
+    return NS_ERROR_NOT_AVAILABLE;
+  nsCOMPtr<nsISupports> val;
+  nsresult rv = v->GetAsISupports(getter_AddRefs(val));
+  if (NS_FAILED(rv))
+    return rv;
+  if (!val) {
+    // We have a value, but it's null
+    *_retval = nullptr;
+    return NS_OK;
+  }
+  return val->QueryInterface(aIID, _retval);
+}
+
+NS_IMETHODIMP nsMsgTxn::SetPropertyAsAString(const nsAString & prop, 
+                                             const nsAString & value)
+{
+  nsCOMPtr<nsIWritableVariant> var = new nsVariant();
+  var->SetAsAString(value);
+  return SetProperty(prop, var);
+}
+
+NS_IMETHODIMP nsMsgTxn::SetPropertyAsACString(const nsAString & prop, 
+                                              const nsACString & value)
+{
+  nsCOMPtr<nsIWritableVariant> var = new nsVariant();
+  var->SetAsACString(value);
+  return SetProperty(prop, var);
+}
+
+NS_IMETHODIMP nsMsgTxn::SetPropertyAsAUTF8String(const nsAString & prop, 
+                                                 const nsACString & value)
+{
+  nsCOMPtr<nsIWritableVariant> var = new nsVariant();
+  var->SetAsAUTF8String(value);
+  return SetProperty(prop, var);
+}
+
+NS_IMETHODIMP nsMsgTxn::SetPropertyAsInterface(const nsAString & prop, 
+                                               nsISupports* value)
+{
+  nsCOMPtr<nsIWritableVariant> var = new nsVariant();
+  var->SetAsISupports(value);
+  return SetProperty(prop, var);
+}
+
+/////////////////////// Transaction Stuff //////////////////
+NS_IMETHODIMP nsMsgTxn::DoTransaction(void)
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::GetIsTransient(bool *aIsTransient)
+{
+  if (nullptr!=aIsTransient)
+    *aIsTransient = false;
+  else
+    return NS_ERROR_NULL_POINTER;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgTxn::Merge(nsITransaction *aTransaction, bool *aDidMerge)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+
+nsresult nsMsgTxn::GetMsgWindow(nsIMsgWindow **msgWindow)
+{
+    if (!msgWindow || !m_msgWindow)
+        return NS_ERROR_NULL_POINTER;
+    *msgWindow = m_msgWindow;
+    NS_ADDREF (*msgWindow);
+    return NS_OK;
+}
+
+nsresult nsMsgTxn::SetMsgWindow(nsIMsgWindow *msgWindow)
+{
+    m_msgWindow = msgWindow;
+    return NS_OK;
+}
+
+
+nsresult
+nsMsgTxn::SetTransactionType(uint32_t txnType)
+{
+  return SetPropertyAsUint32(NS_LITERAL_STRING("type"), txnType);
+}
+
+/*none of the callers pass null aFolder, 
+  we always initialize aResult (before we pass in) for the case where the key is not in the db*/
+nsresult 
+nsMsgTxn::CheckForToggleDelete(nsIMsgFolder *aFolder, const nsMsgKey &aMsgKey, bool *aResult)
+{
+  NS_ENSURE_ARG(aResult);
+  nsCOMPtr<nsIMsgDBHdr> message;
+  nsCOMPtr<nsIMsgDatabase> db;
+  nsresult rv = aFolder->GetMsgDatabase(getter_AddRefs(db));
+  if (db)
+  {
+    bool containsKey;
+    rv = db->ContainsKey(aMsgKey, &containsKey);
+    if (NS_FAILED(rv) || !containsKey)   // the message has been deleted from db, so we cannot do toggle here
+      return NS_OK;
+    rv = db->GetMsgHdrForKey(aMsgKey, getter_AddRefs(message));
+    uint32_t flags;
+    if (NS_SUCCEEDED(rv) && message)
+    {
+      message->GetFlags(&flags);
+      *aResult = (flags & nsMsgMessageFlags::IMAPDeleted) != 0;
+    }
+  }
+  return rv;
+}

mailnews/base/util/nsMsgTxn.h

@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgTxn_h__
+#define nsMsgTxn_h__
+
+#include "mozilla/Attributes.h"
+#include "nsITransaction.h"
+#include "msgCore.h"
+#include "nsCOMPtr.h"
+#include "nsIMsgWindow.h"
+#include "nsInterfaceHashtable.h"
+#include "MailNewsTypes2.h"
+#include "nsIVariant.h"
+#include "nsIWritablePropertyBag.h"
+#include "nsIWritablePropertyBag2.h"
+
+#define NS_MESSAGETRANSACTION_IID \
+{ /* da621b30-1efc-11d3-abe4-00805f8ac968 */ \
+    0xda621b30, 0x1efc, 0x11d3, \
+  { 0xab, 0xe4, 0x00, 0x80, 0x5f, 0x8a, 0xc9, 0x68 } }
+/**
+ * base class for all message undo/redo transactions.
+ */
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+class NS_MSG_BASE nsMsgTxn : public nsITransaction, 
+                             public nsIWritablePropertyBag,
+                             public nsIWritablePropertyBag2
+{
+public:
+    nsMsgTxn();
+
+    nsresult Init();
+
+    NS_IMETHOD DoTransaction(void) override;
+
+    NS_IMETHOD UndoTransaction(void) override = 0;
+
+    NS_IMETHOD RedoTransaction(void) override = 0;
+    
+    NS_IMETHOD GetIsTransient(bool *aIsTransient) override;
+
+    NS_IMETHOD Merge(nsITransaction *aTransaction, bool *aDidMerge) override;
+
+    nsresult GetMsgWindow(nsIMsgWindow **msgWindow);
+    nsresult SetMsgWindow(nsIMsgWindow *msgWindow);
+    nsresult SetTransactionType(uint32_t txnType);
+ 
+    NS_DECL_THREADSAFE_ISUPPORTS
+    NS_DECL_NSIPROPERTYBAG
+    NS_DECL_NSIPROPERTYBAG2
+    NS_DECL_NSIWRITABLEPROPERTYBAG
+    NS_DECL_NSIWRITABLEPROPERTYBAG2
+
+protected:
+    virtual ~nsMsgTxn();
+
+    // a hash table of string -> nsIVariant
+    nsInterfaceHashtable<nsStringHashKey, nsIVariant> mPropertyHash;
+    nsCOMPtr<nsIMsgWindow> m_msgWindow;
+    uint32_t m_txnType;
+    nsresult CheckForToggleDelete(nsIMsgFolder *aFolder, const nsMsgKey &aMsgKey, bool *aResult);
+};
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_HIDDEN
+
+#endif

mailnews/base/util/nsMsgUtils.h

@@ -0,0 +1,589 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _NSMSGUTILS_H
+#define _NSMSGUTILS_H
+
+#include "nsIURL.h"
+#include "nsStringGlue.h"
+#include "msgCore.h"
+#include "nsCOMPtr.h"
+#include "MailNewsTypes2.h"
+#include "nsTArray.h"
+#include "nsInterfaceRequestorAgg.h"
+#include "nsILoadGroup.h"
+// Disable deprecation warnings generated by nsISupportsArray and associated
+// classes.
+#if defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning (disable : 4996)
+#endif
+#include "nsISupportsArray.h"
+#include "nsIAtom.h"
+#include "nsINetUtil.h"
+#include "nsIRequest.h"
+#include "nsILoadInfo.h"
+#include "nsServiceManagerUtils.h"
+#include "nsUnicharUtils.h"
+#include "nsIFile.h"
+
+class nsIChannel;
+class nsIFile;
+class nsIPrefBranch;
+class nsIMsgFolder;
+class nsIMsgMessageService;
+class nsIUrlListener;
+class nsIOutputStream;
+class nsIInputStream;
+class nsIMsgDatabase;
+class nsIMutableArray;
+class nsIProxyInfo;
+class nsIMsgWindow;
+class nsISupportsArray;
+class nsIStreamListener;
+
+#define FILE_IO_BUFFER_SIZE (16*1024)
+#define MSGS_URL    "chrome://messenger/locale/messenger.properties"
+
+//These are utility functions that can used throughout the mailnews code
+
+NS_MSG_BASE nsresult GetMessageServiceContractIDForURI(const char *uri, nsCString &contractID);
+
+NS_MSG_BASE nsresult GetMessageServiceFromURI(const nsACString& uri, nsIMsgMessageService **aMessageService);
+
+NS_MSG_BASE nsresult GetMsgDBHdrFromURI(const char *uri, nsIMsgDBHdr **msgHdr);
+
+NS_MSG_BASE nsresult CreateStartupUrl(const char *uri, nsIURI** aUrl);
+
+NS_MSG_BASE nsresult NS_MsgGetPriorityFromString(
+                       const char * const priority,
+                       nsMsgPriorityValue & outPriority);
+
+NS_MSG_BASE nsresult NS_MsgGetPriorityValueString(
+                       const nsMsgPriorityValue p,
+                       nsACString & outValueString);
+
+NS_MSG_BASE nsresult NS_MsgGetUntranslatedPriorityName(
+                       const nsMsgPriorityValue p,
+                       nsACString & outName);
+
+NS_MSG_BASE nsresult NS_MsgHashIfNecessary(nsAutoString &name);
+NS_MSG_BASE nsresult NS_MsgHashIfNecessary(nsAutoCString &name);
+
+NS_MSG_BASE nsresult FormatFileSize(int64_t size, bool useKB, nsAString &formattedSize);
+
+
+/**
+ * given a folder uri, return the path to folder in the user profile directory.
+ *
+ * @param aFolderURI uri of folder we want the path to, without the scheme
+ * @param[out] aPathString result path string
+ * @param aScheme scheme of the uri
+ * @param[optional] aIsNewsFolder is this a news folder?
+ */
+NS_MSG_BASE nsresult 
+NS_MsgCreatePathStringFromFolderURI(const char *aFolderURI,
+                                    nsCString& aPathString,
+                                    const nsCString &aScheme,
+                                    bool aIsNewsFolder=false);
+
+/**
+ * Given a string and a length, removes any "Re:" strings from the front.
+ * It also deals with that dumbass "Re[2]:" thing that some losing mailers do.
+ *
+ * If mailnews.localizedRe is set, it will also remove localized "Re:" strings.
+ *
+ * @return true if it made a change (in which case the caller should look to
+ *         modifiedSubject for the result) and false otherwise (in which
+ *         case the caller should look at subject for the result)
+ */
+NS_MSG_BASE bool NS_MsgStripRE(const nsCString& subject, nsCString& modifiedSubject);
+
+NS_MSG_BASE char * NS_MsgSACopy(char **destination, const char *source);
+
+NS_MSG_BASE char * NS_MsgSACat(char **destination, const char *source);
+
+NS_MSG_BASE nsresult NS_MsgEscapeEncodeURLPath(const nsAString& aStr,
+                                               nsCString& aResult);
+
+NS_MSG_BASE nsresult NS_MsgDecodeUnescapeURLPath(const nsACString& aPath,
+                                                 nsAString& aResult);
+
+NS_MSG_BASE bool WeAreOffline();
+
+// Check if a folder with aFolderUri exists
+NS_MSG_BASE nsresult GetExistingFolder(const nsCString& aFolderURI, nsIMsgFolder **aFolder);
+
+// Escape lines starting with "From ", ">From ", etc. in a buffer.
+NS_MSG_BASE nsresult EscapeFromSpaceLine(nsIOutputStream *ouputStream, char *start, const char *end);
+NS_MSG_BASE bool IsAFromSpaceLine(char *start, const char *end);
+
+NS_MSG_BASE nsresult NS_GetPersistentFile(const char *relPrefName,
+                                          const char *absPrefName,
+                                          const char *dirServiceProp, // Can be NULL
+                                          bool& gotRelPref,
+                                          nsIFile **aFile,
+                                          nsIPrefBranch *prefBranch = nullptr);
+
+NS_MSG_BASE nsresult NS_SetPersistentFile(const char *relPrefName,
+                                          const char *absPrefName,
+                                          nsIFile *aFile,
+                                          nsIPrefBranch *prefBranch = nullptr);
+
+NS_MSG_BASE nsresult IsRFC822HeaderFieldName(const char *aHdr, bool *aResult);
+
+NS_MSG_BASE nsresult NS_GetUnicharPreferenceWithDefault(nsIPrefBranch *prefBranch,   //can be null, if so uses the root branch
+                                                        const char *prefName,
+                                                        const nsAString& defValue,
+                                                        nsAString& prefValue);
+
+NS_MSG_BASE nsresult NS_GetLocalizedUnicharPreferenceWithDefault(nsIPrefBranch *prefBranch,   //can be null, if so uses the root branch
+                                                                 const char *prefName,
+                                                                 const nsAString& defValue,
+                                                                 nsAString& prefValue);
+
+NS_MSG_BASE nsresult NS_GetLocalizedUnicharPreference(nsIPrefBranch *prefBranch,   //can be null, if so uses the root branch
+                                                      const char *prefName,
+                                                      nsAString& prefValue);
+
+  /**
+   * this needs a listener, because we might have to create the folder
+   * on the server, and that is asynchronous
+   */
+NS_MSG_BASE nsresult GetOrCreateFolder(const nsACString & aURI, nsIUrlListener *aListener);
+
+// Returns true if the nsIURI is a message under an RSS account
+NS_MSG_BASE nsresult IsRSSArticle(nsIURI * aMsgURI, bool *aIsRSSArticle);
+
+// digest needs to be a pointer to a 16 byte buffer
+#define DIGEST_LENGTH 16
+
+NS_MSG_BASE nsresult MSGCramMD5(const char *text, int32_t text_len, const char *key, int32_t key_len, unsigned char *digest);
+NS_MSG_BASE nsresult MSGApopMD5(const char *text, int32_t text_len, const char *password, int32_t password_len, unsigned char *digest);
+
+// helper functions to convert a 64bits PRTime into a 32bits value (compatible time_t) and vice versa.
+NS_MSG_BASE void PRTime2Seconds(PRTime prTime, uint32_t *seconds);
+NS_MSG_BASE void PRTime2Seconds(PRTime prTime, int32_t *seconds);
+NS_MSG_BASE void Seconds2PRTime(uint32_t seconds, PRTime *prTime);
+// helper function to generate current date+time as a string
+NS_MSG_BASE void MsgGenerateNowStr(nsACString &nowStr);
+
+// Appends the correct summary file extension onto the supplied fileLocation
+// and returns it in summaryLocation.
+NS_MSG_BASE nsresult GetSummaryFileLocation(nsIFile* fileLocation,
+                                            nsIFile** summaryLocation);
+
+// Gets a special directory and appends the supplied file name onto it.
+NS_MSG_BASE nsresult GetSpecialDirectoryWithFileName(const char* specialDirName,
+                                                     const char* fileName,
+                                                     nsIFile** result);
+
+// cleanup temp files with the given filename and extension, including
+// the consecutive -NNNN ones that we can find. If there are holes, e.g.,
+// <filename>-1-10,12.<extension> exist, but <filename>-11.<extension> does not
+// we'll clean up 1-10. If the leaks are common, I think the gaps will tend to
+// be filled.
+NS_MSG_BASE nsresult MsgCleanupTempFiles(const char *fileName, const char *extension);
+
+NS_MSG_BASE nsresult MsgGetFileStream(nsIFile *file, nsIOutputStream **fileStream);
+
+NS_MSG_BASE nsresult MsgReopenFileStream(nsIFile *file, nsIInputStream *fileStream);
+
+// Automatically creates an output stream with a suitable buffer
+NS_MSG_BASE nsresult MsgNewBufferedFileOutputStream(nsIOutputStream **aResult, nsIFile *aFile, int32_t aIOFlags = -1, int32_t aPerm = -1);
+
+// Automatically creates an output stream with a suitable buffer, but write to a temporary file first, then rename to aFile
+NS_MSG_BASE nsresult MsgNewSafeBufferedFileOutputStream(nsIOutputStream **aResult, nsIFile *aFile, int32_t aIOFlags = -1, int32_t aPerm = -1);
+
+// fills in the position of the passed in keyword in the passed in keyword list
+// and returns false if the keyword isn't present
+NS_MSG_BASE bool MsgFindKeyword(const nsCString &keyword, nsCString &keywords, int32_t *aStartOfKeyword, int32_t *aLength);
+
+NS_MSG_BASE bool MsgHostDomainIsTrusted(nsCString &host, nsCString &trustedMailDomains);
+
+// gets an nsIFile from a UTF-8 file:// path
+NS_MSG_BASE nsresult MsgGetLocalFileFromURI(const nsACString &aUTF8Path, nsIFile **aFile);
+
+NS_MSG_BASE void MsgStripQuotedPrintable (unsigned char *src);
+
+/*
+ * Utility function copied from nsReadableUtils
+ */
+NS_MSG_BASE bool MsgIsUTF8(const nsACString& aString);
+
+/*
+ * Utility functions that call functions from nsINetUtil
+ */
+
+NS_MSG_BASE nsresult MsgEscapeString(const nsACString &aStr,
+                                     uint32_t aType, nsACString &aResult);
+
+NS_MSG_BASE nsresult MsgUnescapeString(const nsACString &aStr, 
+                                       uint32_t aFlags, nsACString &aResult);
+
+NS_MSG_BASE nsresult MsgEscapeURL(const nsACString &aStr, uint32_t aFlags,
+                                  nsACString &aResult);
+
+// Converts an nsTArray of nsMsgKeys plus a database, to an array of nsIMsgDBHdrs.
+NS_MSG_BASE nsresult MsgGetHeadersFromKeys(nsIMsgDatabase *aDB,
+                                           const nsTArray<nsMsgKey> &aKeys,
+                                           nsIMutableArray *aHeaders);
+// Converts an array of nsMsgKeys plus a database, to an array of nsIMsgDBHdrs.
+NS_MSG_BASE nsresult MsgGetHdrsFromKeys(nsIMsgDatabase *aDB,
+                                        nsMsgKey *aKeys,
+                                        uint32_t aNumKeys,
+                                        nsIMutableArray **aHeaders);
+
+NS_MSG_BASE nsresult MsgExamineForProxy(nsIChannel *channel,
+                                        nsIProxyInfo **proxyInfo);
+
+NS_MSG_BASE int32_t MsgFindCharInSet(const nsCString &aString,
+                                     const char* aChars, uint32_t aOffset = 0);
+NS_MSG_BASE int32_t MsgFindCharInSet(const nsString &aString,
+                                     const char* aChars, uint32_t aOffset = 0);
+
+
+// advances bufferOffset to the beginning of the next line, if we don't
+// get to maxBufferOffset first. Returns false if we didn't get to the
+// next line.
+NS_MSG_BASE bool MsgAdvanceToNextLine(const char *buffer, uint32_t &bufferOffset,
+                                   uint32_t maxBufferOffset);
+
+/**
+ * Alerts the user that the login to the server failed. Asks whether the
+ * connection should: retry, cancel, or request a new password.
+ *
+ * @param aMsgWindow The message window associated with this action (cannot
+ *                   be null).
+ * @param aHostname  The hostname of the server for which the login failed.
+ * @param aResult    The button pressed. 0 for retry, 1 for cancel,
+ *                   2 for enter a new password.
+ * @return           NS_OK for success, NS_ERROR_* if there was a failure in
+ *                   creating the dialog.
+ */
+NS_MSG_BASE nsresult MsgPromptLoginFailed(nsIMsgWindow *aMsgWindow,
+                                          const nsCString &aHostname,
+                                          int32_t *aResult);
+
+/**
+ * Calculate a PRTime value used to determine if a date is XX
+ * days ago. This is used by various retention setting algorithms.
+ */
+NS_MSG_BASE PRTime MsgConvertAgeInDaysToCutoffDate(int32_t ageInDays);
+
+/**
+ * Converts the passed in term list to its string representation.
+ *
+ * @param      aTermList    Array of nsIMsgSearchTerms
+ * @param[out] aOutString   result representation of search terms.
+ *
+ */
+NS_MSG_BASE nsresult MsgTermListToString(nsISupportsArray *aTermList, nsCString &aOutString);
+
+NS_MSG_BASE nsresult
+MsgStreamMsgHeaders(nsIInputStream *aInputStream, nsIStreamListener *aConsumer);
+
+/**
+ * convert string to uint64_t
+ *
+ * @param str conveted string
+ * @returns   uint64_t vaule for success, 0 for parse failure
+ */
+NS_MSG_BASE uint64_t ParseUint64Str(const char *str);
+
+/**
+ * Detect charset of file
+ *
+ * @param      aFile    The target of nsIFile
+ * @param[out] aCharset The charset string
+ */
+NS_MSG_BASE nsresult MsgDetectCharsetFromFile(nsIFile *aFile, nsACString &aCharset);
+
+/*
+ * Converts a buffer to plain text. Some conversions may
+ * or may not work with certain end charsets which is why we
+ * need that as an argument to the function. If charset is
+ * unknown or deemed of no importance NULL could be passed.
+ * @param[in/out] aConBuf        Variable with the text to convert
+ * @param         formatFlowed   Use format flowed?
+ * @param         delsp          Use delsp=yes when flowed
+ * @param         formatOutput   Reformat the output?
+ & @param         disallowBreaks Disallow breaks when formatting
+ */
+NS_MSG_BASE nsresult
+ConvertBufToPlainText(nsString &aConBuf, bool formatFlowed, bool delsp,
+                                         bool formatOutput, bool disallowBreaks);
+
+/**
+ * The following definitons exist for compatibility between the internal and
+ * external APIs. Where possible they just forward to the existing API.
+ */
+
+#ifdef MOZILLA_INTERNAL_API
+#include "nsEscape.h"
+
+/**
+ * The internal API expects nsCaseInsensitiveC?StringComparator() and true.
+ * Redefine CaseInsensitiveCompare so that Find works.
+ */
+#define CaseInsensitiveCompare true
+/**
+ * The following methods are not exposed to the external API, but when we're
+ * using the internal API we can simply redirect the calls appropriately.
+ */
+#define MsgLowerCaseEqualsLiteral(str, l) \
+        (str).LowerCaseEqualsLiteral(l)
+#define MsgRFindChar(str, ch, len) \
+        (str).RFindChar(ch, len)
+#define MsgCompressWhitespace(str) \
+        (str).CompressWhitespace()
+#define MsgEscapeHTML(str) \
+        nsEscapeHTML(str)
+#define MsgEscapeHTML2(buffer, len) \
+        nsEscapeHTML2(buffer, len)
+#define MsgReplaceSubstring(str, what, replacement) \
+        (str).ReplaceSubstring(what, replacement)
+#define MsgIsUTF8(str) \
+        IsUTF8(str)
+#define MsgNewInterfaceRequestorAggregation(aFirst, aSecond, aResult) \
+        NS_NewInterfaceRequestorAggregation(aFirst, aSecond, aResult)
+#define MsgNewNotificationCallbacksAggregation(aCallbacks, aLoadGroup, aResult) \
+        NS_NewNotificationCallbacksAggregation(aCallbacks, aLoadGroup, aResult)
+#define MsgGetAtom(aString) \
+        NS_Atomize(aString)
+#define MsgNewAtom(aString) \
+        NS_Atomize(aString)
+#define MsgReplaceChar(aString, aNeedle, aReplacement) \
+        (aString).ReplaceChar(aNeedle, aReplacement)
+#define MsgFind(str, what, ignore_case, offset) \
+        (str).Find(what, ignore_case, offset)
+#define MsgCountChar(aString, aChar) \
+        (aString).CountChar(aChar)
+
+#else
+
+/**
+ * The external API expects CaseInsensitiveCompare. Redefine
+ * nsCaseInsensitiveC?StringComparator() so that Equals works.
+ */
+#define nsCaseInsensitiveCStringComparator() \
+        CaseInsensitiveCompare
+#define nsCaseInsensitiveStringComparator() \
+        CaseInsensitiveCompare
+/// The external API does not provide kNotFound.
+#define kNotFound -1
+/**
+ * The external API does not provide the following methods. While we can
+ * reasonably easily define them in terms of existing methods, we only want
+ * to do this when using the external API.
+ */
+#define AppendASCII \
+        AppendLiteral
+#define AppendUTF16toUTF8(source, dest) \
+        (dest).Append(NS_ConvertUTF16toUTF8(source))
+#define AppendUTF8toUTF16(source, dest) \
+        (dest).Append(NS_ConvertUTF8toUTF16(source))
+#define AppendASCIItoUTF16(source, dest) \
+        (dest).Append(NS_ConvertASCIItoUTF16(source))
+#define Compare(str1, str2, comp) \
+        (str1).Compare(str2, comp)
+#define CaseInsensitiveFindInReadable(what, str) \
+        ((str).Find(what, CaseInsensitiveCompare) != kNotFound)
+#define LossyAppendUTF16toASCII(source, dest) \
+        (dest).Append(NS_LossyConvertUTF16toASCII(source))
+#define Last() \
+        EndReading()[-1]
+#define SetCharAt(ch, index) \
+        Replace(index, 1, ch)
+#define NS_NewISupportsArray(result) \
+        CallCreateInstance(NS_SUPPORTSARRAY_CONTRACTID, static_cast<nsISupportsArray**>(result))
+/**
+ * The internal and external methods expect the parameters in a different order.
+ * The internal API also always expects a flag rather than a comparator.
+ */
+inline int32_t MsgFind(nsAString &str, const char *what, bool ignore_case, uint32_t offset)
+{
+  return str.Find(what, offset, ignore_case);
+}
+
+inline int32_t MsgFind(nsACString &str, const char *what, bool ignore_case, int32_t offset)
+{
+  /* See Find_ComputeSearchRange from nsStringObsolete.cpp */
+  if (offset < 0) {
+    offset = 0;
+  }
+  if (ignore_case)
+    return str.Find(nsDependentCString(what), offset, CaseInsensitiveCompare);
+  return str.Find(nsDependentCString(what), offset);
+}
+
+inline int32_t MsgFind(nsACString &str, const nsACString &what, bool ignore_case, int32_t offset)
+{
+  /* See Find_ComputeSearchRange from nsStringObsolete.cpp */
+  if (offset < 0) {
+    offset = 0;
+  }
+  if (ignore_case)
+    return str.Find(what, offset, CaseInsensitiveCompare);
+  return str.Find(what, offset);
+}
+
+/**
+ * The following methods are not exposed to the external API so we define
+ * equivalent versions here.
+ */
+/// Equivalent of LowerCaseEqualsLiteral(literal)
+#define MsgLowerCaseEqualsLiteral(str, literal) \
+        (str).Equals(literal, CaseInsensitiveCompare)
+/// Equivalent of RFindChar(ch, len)
+#define MsgRFindChar(str, ch, len) \
+        StringHead(str, len).RFindChar(ch)
+/// Equivalent of aString.CompressWhitespace()
+NS_MSG_BASE void MsgCompressWhitespace(nsCString& aString);
+/// Equivalent of nsEscapeHTML(aString)
+NS_MSG_BASE char *MsgEscapeHTML(const char *aString);
+/// Equivalent of nsEscapeHTML2(aBuffer, aLen)
+NS_MSG_BASE char16_t *MsgEscapeHTML2(const char16_t *aBuffer, int32_t aLen);
+// Existing replacement for IsUTF8
+NS_MSG_BASE bool MsgIsUTF8(const nsACString& aString);
+/// Equivalent of NS_Atomize(aUTF8String)
+NS_MSG_BASE already_AddRefed<nsIAtom> MsgNewAtom(const char* aString);
+/// Equivalent of NS_Atomize(aUTF8String)
+inline already_AddRefed<nsIAtom> MsgGetAtom(const char* aUTF8String)
+{
+  return MsgNewAtom(aUTF8String);
+}
+/// Equivalent of ns(C)String::ReplaceSubstring(what, replacement)
+NS_MSG_BASE void MsgReplaceSubstring(nsAString &str, const nsAString &what, const nsAString &replacement);
+NS_MSG_BASE void MsgReplaceSubstring(nsACString &str, const char *what, const char *replacement);
+/// Equivalent of ns(C)String::ReplaceChar(what, replacement)
+NS_MSG_BASE void MsgReplaceChar(nsString& str, const char *set, const char16_t replacement);
+NS_MSG_BASE void MsgReplaceChar(nsCString& str, const char needle, const char replacement);
+// Equivalent of NS_NewInterfaceRequestorAggregation(aFirst, aSecond, aResult)
+NS_MSG_BASE nsresult MsgNewInterfaceRequestorAggregation(nsIInterfaceRequestor *aFirst,
+                                                         nsIInterfaceRequestor *aSecond,
+                                                         nsIInterfaceRequestor **aResult);
+
+/**
+ * This function is based on NS_NewNotificationCallbacksAggregation from
+ * nsNetUtil.h
+ *
+ * This function returns a nsIInterfaceRequestor instance that returns the
+ * same result as NS_QueryNotificationCallbacks when queried.
+ */
+inline nsresult
+MsgNewNotificationCallbacksAggregation(nsIInterfaceRequestor  *callbacks,
+                                       nsILoadGroup           *loadGroup,
+                                       nsIInterfaceRequestor **result)
+{
+    nsCOMPtr<nsIInterfaceRequestor> cbs;
+    if (loadGroup)
+        loadGroup->GetNotificationCallbacks(getter_AddRefs(cbs));
+    return MsgNewInterfaceRequestorAggregation(callbacks, cbs, result);
+}
+
+/**
+ * Count occurences of specified character in string.
+ *
+ */
+inline
+uint32_t MsgCountChar(nsACString &aString, char16_t aChar) {
+  const char *begin, *end;
+  uint32_t num_chars = 0;
+  aString.BeginReading(&begin, &end);
+  for (const char *current = begin; current < end; ++current) {
+      if (*current == aChar)
+        ++num_chars;
+  }
+  return num_chars;
+}
+
+inline
+uint32_t MsgCountChar(nsAString &aString, char16_t aChar) {
+  const char16_t *begin, *end;
+  uint32_t num_chars = 0;
+  aString.BeginReading(&begin, &end);
+  for (const char16_t *current = begin; current < end; ++current) {
+      if (*current == aChar)
+        ++num_chars;
+  }
+  return num_chars;
+}
+
+#endif
+
+/**
+ * Converts a hex string into an integer.
+ * Processes up to aNumChars characters or the first non-hex char.
+ * It is not an error if less than aNumChars valid hex digits are found.
+ */
+NS_MSG_BASE uint64_t MsgUnhex(const char *aHexString, size_t aNumChars);
+
+/**
+ * Checks if a string is a valid hex literal containing at least aNumChars digits.
+ */
+NS_MSG_BASE bool MsgIsHex(const char *aHexString, size_t aNumChars);
+
+/**
+ * Convert an uint32_t to a nsMsgKey.
+ * Currently they are mostly the same but we need to preserve the notion that
+ * nsMsgKey is an opaque value that can't be treated as a generic integer
+ * (except when storing it into the database). It enables type safety checks and
+ * may prevent coding errors.
+ */
+NS_MSG_BASE nsMsgKey msgKeyFromInt(uint32_t aValue);
+
+NS_MSG_BASE nsMsgKey msgKeyFromInt(uint64_t aValue);
+
+/**
+ * Helper function to extract query part from URL spec.
+ */
+nsAutoCString MsgExtractQueryPart(nsAutoCString spec, const char* queryToExtract);
+
+/**
+ * Helper macro for defining getter/setters. Ported from nsISupportsObsolete.h
+ */
+#define NS_IMPL_GETSET(clazz, attr, type, member) \
+  NS_IMETHODIMP clazz::Get##attr(type *result) \
+  { \
+    NS_ENSURE_ARG_POINTER(result); \
+    *result = member; \
+    return NS_OK; \
+  } \
+  NS_IMETHODIMP clazz::Set##attr(type aValue) \
+  { \
+    member = aValue; \
+    return NS_OK; \
+  }
+
+#endif
+
+ /**
+ * Macro and helper function for reporting an error, warning or
+ * informational message to the Error Console
+ *
+ * This will require the inclusion of the following files in the source file
+ * #include "nsIScriptError.h"
+ * #include "nsIConsoleService.h"
+ *
+ */
+
+NS_MSG_BASE
+void MsgLogToConsole4(const nsAString &aErrorText, const nsAString &aFilename,
+                      uint32_t aLine, uint32_t flags);
+
+// Macro with filename and line number
+#define MSG_LOG_TO_CONSOLE(_text, _flag) MsgLogToConsole4(NS_LITERAL_STRING(_text), NS_LITERAL_STRING(__FILE__), __LINE__, _flag)
+#define MSG_LOG_ERR_TO_CONSOLE(_text) MSG_LOG_TO_CONSOLE(_text, nsIScriptError::errorFlag)
+#define MSG_LOG_WARN_TO_CONSOLE(_text) MSG_LOG_TO_CONSOLE(_text, nsIScriptError::warningFlag)
+#define MSG_LOG_INFO_TO_CONSOLE(_text) MSG_LOG_TO_CONSOLE(_text, nsIScriptError::infoFlag)
+
+// Helper macros to cope with shoddy I/O error reporting (or lack thereof)
+#define MSG_NS_ERROR(_txt) do { NS_ERROR(_txt); MSG_LOG_ERR_TO_CONSOLE(_txt); } while(0)
+#define MSG_NS_WARNING(_txt) do { NS_WARNING(_txt); MSG_LOG_WARN_TO_CONSOLE(_txt); } while (0)
+#define MSG_NS_WARN_IF_FALSE(_val, _txt) do { if (!(_val)) { NS_WARNING(_txt); MSG_LOG_WARN_TO_CONSOLE(_txt); } } while (0)
+#define MSG_NS_INFO(_txt) do { MSG_LOCAL_INFO_TO_CONSOLE(_txt); \
+  fprintf(stderr,"(info) %s (%s:%d)\n", _txt, __FILE__, __LINE__); } while(0)

mailnews/base/util/nsStopwatch.cpp

@@ -0,0 +1,183 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsStopwatch.h"
+
+#include <stdio.h>
+#include <time.h>
+#if defined(XP_UNIX)
+#include <unistd.h>
+#include <sys/times.h>
+#include <sys/time.h>
+#include <errno.h>
+#elif defined(XP_WIN)
+#include "windows.h"
+#endif // elif defined(XP_WIN)
+
+#include "nsMemory.h"
+/*
+ * This basis for the logic in this file comes from (will used to come from):
+ *  (mozilla/)modules/libutil/public/stopwatch.cpp.
+ *  
+ * It was no longer used in the mozilla tree, and is being migrated to
+ * comm-central where we actually have a need for it.  ("Being" in the sense
+ * that it will not be removed immediately from mozilla-central.)
+ * 
+ * Simplification and general clean-up has been performed and the fix for
+ * bug 96669 has been integrated.
+ */
+
+NS_IMPL_ISUPPORTS(nsStopwatch, nsIStopwatch)
+
+#if defined(XP_UNIX)
+/** the number of ticks per second */
+static double gTicks = 0;
+#define MICRO_SECONDS_TO_SECONDS_MULT static_cast<double>(1.0e-6)
+#elif defined(WIN32)
+#ifdef DEBUG
+#ifdef MOZILLA_INTERNAL_API
+#include "nsPrintfCString.h"
+#endif
+#endif
+// 1 tick per 100ns = 10 per us = 10 * 1,000 per ms = 10 * 1,000 * 1,000 per sec.
+#define WIN32_TICK_RESOLUTION static_cast<double>(1.0e-7)
+// subtract off to get to the unix epoch
+#define UNIX_EPOCH_IN_FILE_TIME 116444736000000000L
+#endif // elif defined(WIN32)
+
+nsStopwatch::nsStopwatch()
+ : fTotalRealTimeSecs(0.0)
+ , fTotalCpuTimeSecs(0.0)
+ , fRunning(false)
+{
+#if defined(XP_UNIX)
+  // idempotent in the event of a race under all coherency models
+  if (!gTicks)
+  {
+    // we need to clear errno because sysconf's spec says it leaves it the same
+    //  on success and only sets it on failure.
+    errno = 0;
+    gTicks = (clock_t)sysconf(_SC_CLK_TCK);
+    // in event of failure, pick an arbitrary value so we don't divide by zero.
+    if (errno)
+      gTicks = 1000000L;
+  }
+#endif
+}
+
+nsStopwatch::~nsStopwatch()
+{
+}
+
+NS_IMETHODIMP nsStopwatch::Start()
+{
+  fTotalRealTimeSecs = 0.0;
+  fTotalCpuTimeSecs = 0.0;
+  return Resume();
+}
+
+NS_IMETHODIMP nsStopwatch::Stop()
+{
+  fStopRealTimeSecs = GetRealTime();
+  fStopCpuTimeSecs  = GetCPUTime();
+  if (fRunning)
+  {
+    fTotalCpuTimeSecs  += fStopCpuTimeSecs  - fStartCpuTimeSecs;
+    fTotalRealTimeSecs += fStopRealTimeSecs - fStartRealTimeSecs;
+  }
+  fRunning = false;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsStopwatch::Resume()
+{
+  if (!fRunning)
+  {
+    fStartRealTimeSecs = GetRealTime();
+    fStartCpuTimeSecs  = GetCPUTime();
+  }
+  fRunning = true;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsStopwatch::GetCpuTimeSeconds(double *result)
+{
+  NS_ENSURE_ARG_POINTER(result);
+  *result = fTotalCpuTimeSecs;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsStopwatch::GetRealTimeSeconds(double *result)
+{
+  NS_ENSURE_ARG_POINTER(result);
+  *result = fTotalRealTimeSecs;
+  return NS_OK;
+}
+
+double nsStopwatch::GetRealTime()
+{
+#if defined(XP_UNIX)
+  struct timeval t;
+  gettimeofday(&t, NULL);
+  return t.tv_sec + t.tv_usec * MICRO_SECONDS_TO_SECONDS_MULT;
+#elif defined(WIN32)
+  union     {FILETIME ftFileTime;
+             __int64  ftInt64;
+            } ftRealTime; // time the process has spent in kernel mode
+  SYSTEMTIME st;
+  GetSystemTime(&st);
+  SystemTimeToFileTime(&st, &ftRealTime.ftFileTime);
+  return (ftRealTime.ftInt64 - UNIX_EPOCH_IN_FILE_TIME) * WIN32_TICK_RESOLUTION;
+#else
+#error "nsStopwatch not supported on this platform."
+#endif
+}
+
+double nsStopwatch::GetCPUTime()
+{
+#if defined(XP_UNIX)
+  struct tms cpt;
+  times(&cpt);
+  return (double)(cpt.tms_utime+cpt.tms_stime) / gTicks;
+#elif defined(WIN32)
+  FILETIME    ftCreate,       // when the process was created
+              ftExit;         // when the process exited
+
+  union     {FILETIME ftFileTime;
+             __int64  ftInt64;
+            } ftKernel; // time the process has spent in kernel mode
+
+  union     {FILETIME ftFileTime;
+             __int64  ftInt64;
+            } ftUser;   // time the process has spent in user mode
+
+  HANDLE hProcess = GetCurrentProcess();
+#ifdef DEBUG
+  BOOL ret =
+#endif
+    GetProcessTimes(hProcess, &ftCreate, &ftExit,
+                              &ftKernel.ftFileTime, &ftUser.ftFileTime);
+#ifdef DEBUG
+#ifdef MOZILLA_INTERNAL_API
+  if (!ret)
+    NS_ERROR(nsPrintfCString("GetProcessTimes() failed, error=0x%lx.", GetLastError()).get());
+#else
+  if (!ret) {
+    // nsPrintfCString() is unavailable to report GetLastError().
+    NS_ERROR("GetProcessTimes() failed.");
+  }
+#endif
+#endif
+
+  /*
+   * Process times are returned in a 64-bit structure, as the number of
+   * 100 nanosecond ticks since 1 January 1601.  User mode and kernel mode
+   * times for this process are in separate 64-bit structures.
+   * Add them and convert the result to seconds.
+   */
+  return (ftKernel.ftInt64 + ftUser.ftInt64) * WIN32_TICK_RESOLUTION;
+#else
+#error "nsStopwatch not supported on this platform."
+#endif
+}

mailnews/base/util/nsStopwatch.h

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsStopwatch_h_
+#define _nsStopwatch_h_
+
+#include "nsIStopwatch.h"
+
+#include "msgCore.h"
+
+#define NS_STOPWATCH_CID \
+{0x6ef7eafd, 0x72d0, 0x4c56, {0x94, 0x09, 0x67, 0xe1, 0x6d, 0x0f, 0x25, 0x5b}}
+
+#define NS_STOPWATCH_CONTRACTID "@mozilla.org/stopwatch;1"
+
+#undef  IMETHOD_VISIBILITY
+#define IMETHOD_VISIBILITY NS_VISIBILITY_DEFAULT
+
+class NS_MSG_BASE nsStopwatch : public nsIStopwatch
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSISTOPWATCH
+
+  nsStopwatch();
+private:
+  virtual ~nsStopwatch();
+
+  /// Wall-clock start time in seconds since unix epoch.
+  double fStartRealTimeSecs;
+  /// Wall-clock stop time in seconds since unix epoch.
+  double fStopRealTimeSecs;
+  /// CPU-clock start time in seconds (of CPU time used since app start)
+  double fStartCpuTimeSecs;
+  /// CPU-clock stop time in seconds (of CPU time used since app start)
+  double fStopCpuTimeSecs;
+  /// Total wall-clock time elapsed in seconds. 
+  double fTotalRealTimeSecs;
+  /// Total CPU time elapsed in seconds.
+  double fTotalCpuTimeSecs;
+
+  /// Is the timer running?
+  bool fRunning;
+  
+  static double GetRealTime();
+  static double GetCPUTime();
+};
+
+#endif // _nsStopwatch_h_

mailnews/base/util/templateUtils.js

@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var EXPORTED_SYMBOLS = ["PluralStringFormatter", "makeFriendlyDateAgo"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/PluralForm.jsm");
+Cu.import("resource:///modules/StringBundle.js");
+
+function PluralStringFormatter(aBundleURI) {
+  this._bundle = new StringBundle(aBundleURI);
+}
+
+PluralStringFormatter.prototype = {
+  get: function(aStringName, aReplacements, aPluralCount) {
+    let str = this._bundle.get(aStringName);
+    if (aPluralCount !== undefined)
+      str = PluralForm.get(aPluralCount, str);
+    if (aReplacements !== undefined) {
+      for (let i = 0; i < aReplacements.length; i++)
+        str = str.replace("#" + (i+1), aReplacements[i]);
+    }
+    return str;
+  },
+};
+
+
+var gTemplateUtilsStrings = new PluralStringFormatter(
+  "chrome://messenger/locale/templateUtils.properties"
+);
+
+/**
+ * Helper function to generate a localized "friendly" representation of
+ * time relative to the present.  If the time input is "today", it returns
+ * a string corresponding to just the time.  If it's yesterday, it returns
+ * "yesterday" (localized).  If it's in the last week, it returns the day
+ * of the week. If it's before that, it returns the date.
+ *
+ * @param time
+ *        the time (better be in the past!)
+ * @return The string with a "human-friendly" representation of that time
+ *        relative to now.
+ */
+function makeFriendlyDateAgo(time)
+{
+  let dts = Cc["@mozilla.org/intl/scriptabledateformat;1"]
+              .getService(Ci.nsIScriptableDateFormat);
+
+  // Figure out when today begins
+  let now = new Date();
+  let today = new Date(now.getFullYear(), now.getMonth(),
+                       now.getDate());
+
+  // Get the end time to display
+  let end = time;
+
+  // Figure out if the end time is from today, yesterday,
+  // this week, etc.
+  let dateTime;
+  let kDayInMsecs = 24 * 60 * 60 * 1000;
+  let k6DaysInMsecs = 6 * kDayInMsecs;
+  if (end >= today) {
+    // activity finished after today started, show the time
+    dateTime = dts.FormatTime("", dts.timeFormatNoSeconds,
+                                  end.getHours(), end.getMinutes(),0);
+  } else if (today - end < kDayInMsecs) {
+    // activity finished after yesterday started, show yesterday
+    dateTime = gTemplateUtilsStrings.get("yesterday");
+  } else if (today - end < k6DaysInMsecs) {
+    // activity finished after last week started, show day of week
+    dateTime = end.toLocaleFormat("%A");
+  } else if (now.getFullYear() == end.getFullYear()) {
+    // activity must have been from some time ago.. show month/day
+    let month = end.toLocaleFormat("%B");
+    // Remove leading 0 by converting the date string to a number
+    let date = Number(end.toLocaleFormat("%d"));
+    dateTime = gTemplateUtilsStrings.get("monthDate", [month, date]);
+  } else {
+    // not this year, so show full date format
+    dateTime = dts.FormatDate("", dts.dateFormatShort,
+                              end.getFullYear(), end.getMonth() + 1,
+                              end.getDate());
+  }
+  return dateTime;
+}

mailnews/base/util/traceHelper.js

@@ -0,0 +1,113 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['DebugTraceHelper'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var SPACES = "                                                   ";
+var BRIGHT_COLORS = {
+  red: "\x1b[1;31m",
+  green: "\x1b[1;32m",
+  yellow: "\x1b[1;33m",
+  blue: "\x1b[1;34m",
+  magenta: "\x1b[1;35m",
+  cyan: "\x1b[1;36m",
+  white: "\x1b[1;37m",
+};
+var DARK_COLORS = {
+  red: "\x1b[0;31m",
+  green: "\x1b[0;32m",
+  yellow: "\x1b[0;33m",
+  blue: "\x1b[0;34m",
+  magenta: "\x1b[0;35m",
+  cyan: "\x1b[0;36m",
+  white: "\x1b[0;37m",
+};
+var STOP_COLORS = "\x1b[0m";
+
+
+/**
+ * Example usages:
+ *
+ * Components.utils.import("resource:///modules/traceHelper.js");
+ * var debugContext = {color: "cyan"};
+ * DebugTraceHelper.tracify(FolderDisplayWidget.prototype,
+ *                          "FolderDisplayWidget", /.+/, debugContext);
+ * DebugTraceHelper.tracify(MessageDisplayWidget.prototype,
+ *                          "MessageDisplayWidget", /.+/, debugContext);
+ * DebugTraceHelper.tracify(StandaloneFolderDisplayWidget.prototype,
+ *                          "StandaloneFolderDisplayWidget", /.+/, debugContext);
+ * DebugTraceHelper.tracify(StandaloneMessageDisplayWidget.prototype,
+ *                          "StandaloneMessageDisplayWidget", /.+/, debugContext);
+ * DebugTraceHelper.tracify(DBViewWrapper.prototype,
+ *                          "DBViewWrapper", /.+/, {color: "green"});
+ * DebugTraceHelper.tracify(JSTreeSelection.prototype,
+ *                          "JSTreeSelection", /.+/, {color: "yellow"});
+ */
+var DebugTraceHelper = {
+  tracify: function(aObj, aDesc, aPat, aContext, aSettings) {
+    aContext.depth = 0;
+    let color = aSettings.color || "cyan";
+    aSettings.introCode = BRIGHT_COLORS[color];
+    aSettings.outroCode = DARK_COLORS[color];
+    for (let key in aObj) {
+      if (aPat.test(key)) {
+        // ignore properties!
+        if (aObj.__lookupGetter__(key) || aObj.__lookupSetter__(key))
+          continue;
+        // ignore non-functions!
+        if (typeof(aObj[key]) != "function")
+          continue;
+        let name = key;
+        let prev = aObj[name];
+        aObj[name] = function() {
+          let argstr = "";
+          for (let i = 0; i < arguments.length; i++) {
+            let arg = arguments[i];
+            if (arg == null)
+              argstr += " null";
+            else if (typeof(arg) == "function")
+              argstr += " function "+ arg.name;
+            else
+              argstr += " " + arguments[i].toString();
+          }
+
+          let indent = SPACES.substr(0, aContext.depth++ * 2);
+          dump(indent + "--> " + aSettings.introCode + aDesc + "::" + name +
+               ":" + argstr +
+               STOP_COLORS + "\n");
+          let ret;
+          try {
+            ret = prev.apply(this, arguments);
+          }
+          catch (ex) {
+            if (ex.stack) {
+              dump(BRIGHT_COLORS.red + "Exception: " + ex + "\n  " +
+                   ex.stack.replace("\n", "\n  ") + STOP_COLORS + "\n");
+            }
+            else {
+              dump(BRIGHT_COLORS.red + "Exception: " + ex.fileName + ":" +
+                   ex.lineNumber + ": " + ex + STOP_COLORS + "\n");
+            }
+            aContext.depth--;
+            dump(indent + "<-- " + aSettings.outroCode + aDesc + "::" + name +
+                 STOP_COLORS + "\n");
+            throw ex;
+          }
+          aContext.depth--;
+          dump(indent + "<-- " + aSettings.outroCode + aDesc + "::" + name +
+               ": " + (ret != null ? ret.toString() : "null") +
+               STOP_COLORS + "\n");
+          return ret;
+        };
+      }
+    }
+  }
+};