Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

mailnews/db/gloda/components/glautocomp.js

@@ -0,0 +1,544 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource:///modules/errUtils.js");
+
+var Gloda = null;
+var GlodaUtils = null;
+var MultiSuffixTree = null;
+var TagNoun = null;
+var FreeTagNoun = null;
+
+function ResultRowFullText(aItem, words, typeForStyle) {
+  this.item = aItem;
+  this.words = words;
+  this.typeForStyle = "gloda-fulltext-" + typeForStyle;
+}
+ResultRowFullText.prototype = {
+  multi: false,
+  fullText: true
+};
+
+function ResultRowSingle(aItem, aCriteriaType, aCriteria, aExplicitNounID) {
+  this.nounID = aExplicitNounID || aItem.NOUN_ID;
+  this.nounDef = Gloda._nounIDToDef[this.nounID];
+  this.criteriaType = aCriteriaType;
+  this.criteria = aCriteria;
+  this.item = aItem;
+  this.typeForStyle = "gloda-single-" + this.nounDef.name;
+}
+ResultRowSingle.prototype = {
+  multi: false,
+  fullText: false
+};
+
+function ResultRowMulti(aNounID, aCriteriaType, aCriteria, aQuery) {
+  this.nounID = aNounID;
+  this.nounDef = Gloda._nounIDToDef[aNounID];
+  this.criteriaType = aCriteriaType;
+  this.criteria = aCriteria;
+  this.collection = aQuery.getCollection(this);
+  this.collection.becomeExplicit();
+  this.renderer = null;
+}
+ResultRowMulti.prototype = {
+  multi: true,
+  typeForStyle: "gloda-multi",
+  fullText: false,
+  onItemsAdded: function(aItems) {
+    if (this.renderer) {
+      for (let [iItem, item] of aItems.entries()) {
+        this.renderer.renderItem(item);
+      }
+    }
+  },
+  onItemsModified: function(aItems) {
+  },
+  onItemsRemoved: function(aItems) {
+  },
+  onQueryCompleted: function() {
+  }
+};
+
+function nsAutoCompleteGlodaResult(aListener, aCompleter, aString) {
+  this.listener = aListener;
+  this.completer = aCompleter;
+  this.searchString = aString;
+  this._results = [];
+  this._pendingCount = 0;
+  this._problem = false;
+  // Track whether we have reported anything to the complete controller so
+  //  that we know not to send notifications to it during calls to addRows
+  //  prior to that point.
+  this._initiallyReported = false;
+
+  this.wrappedJSObject = this;
+}
+nsAutoCompleteGlodaResult.prototype = {
+  getObjectAt: function(aIndex) {
+    return this._results[aIndex] || null;
+  },
+  markPending: function ACGR_markPending(aCompleter) {
+    this._pendingCount++;
+  },
+  markCompleted: function ACGR_markCompleted(aCompleter) {
+    if (--this._pendingCount == 0 && this.active) {
+      this.listener.onSearchResult(this.completer, this);
+    }
+  },
+  announceYourself: function ACGR_announceYourself() {
+    this._initiallyReported = true;
+    this.listener.onSearchResult(this.completer, this);
+  },
+  addRows: function ACGR_addRows(aRows) {
+    if (!aRows.length)
+      return;
+    this._results.push.apply(this._results, aRows);
+    if (this._initiallyReported && this.active) {
+      this.listener.onSearchResult(this.completer, this);
+    }
+  },
+  // ==== nsIAutoCompleteResult
+  searchString: null,
+  get searchResult() {
+    if (this._problem)
+      return Ci.nsIAutoCompleteResult.RESULT_FAILURE;
+    if (this._results.length)
+      return (!this._pendingCount) ? Ci.nsIAutoCompleteResult.RESULT_SUCCESS
+                          : Ci.nsIAutoCompleteResult.RESULT_SUCCESS_ONGOING;
+    else
+      return (!this._pendingCount) ? Ci.nsIAutoCompleteResult.RESULT_NOMATCH
+                          : Ci.nsIAutoCompleteResult.RESULT_NOMATCH_ONGOING;
+  },
+  active: false,
+  defaultIndex: -1,
+  errorDescription: null,
+  get matchCount() {
+    return (this._results === null) ? 0 : this._results.length;
+  },
+  // this is the lower text, (shows the url in firefox)
+  // we try and show the contact's name here.
+  getValueAt: function(aIndex) {
+    let thing = this._results[aIndex];
+    return thing.name || thing.value || thing.subject || null;
+  },
+  getLabelAt: function(aIndex) {
+    return this.getValueAt(aIndex);
+  },
+  // rich uses this to be the "title".  it is the upper text
+  // we try and show the identity here.
+  getCommentAt: function(aIndex) {
+    let thing = this._results[aIndex];
+    if (thing.value) // identity
+      return thing.contact.name;
+    else
+      return thing.name || thing.subject;
+  },
+  // rich uses this to be the "type"
+  getStyleAt: function(aIndex) {
+    let row = this._results[aIndex];
+    return row.typeForStyle;
+  },
+  // rich uses this to be the icon
+  getImageAt: function(aIndex) {
+    let thing = this._results[aIndex];
+    if (!thing.value)
+      return null;
+
+    return ""; // we don't want to use gravatars as is.
+    /*
+    let md5hash = GlodaUtils.md5HashString(thing.value);
+    let gravURL = "http://www.gravatar.com/avatar/" + md5hash +
+                                "?d=identicon&s=32&r=g";
+    return gravURL;
+    */
+  },
+  getFinalCompleteValueAt: function(aIndex) {
+    return this.getValueAt(aIndex);
+  },
+  removeValueAt: function() {},
+
+  _stop: function() {
+  }
+};
+
+var MAX_POPULAR_CONTACTS = 200;
+
+/**
+ * Complete contacts/identities based on name/email.  Instant phase is based on
+ *  a suffix-tree built of popular contacts/identities.  Delayed phase relies
+ *  on a LIKE search of all known contacts.
+ */
+function ContactIdentityCompleter() {
+  // get all the contacts
+  let contactQuery = Gloda.newQuery(Gloda.NOUN_CONTACT);
+  contactQuery.orderBy("-popularity").limit(MAX_POPULAR_CONTACTS);
+  this.contactCollection = contactQuery.getCollection(this, null);
+  this.contactCollection.becomeExplicit();
+}
+ContactIdentityCompleter.prototype = {
+  _popularitySorter: function(a, b){ return b.popularity - a.popularity; },
+  complete: function ContactIdentityCompleter_complete(aResult, aString) {
+    if (aString.length < 3) {
+      // In CJK, first name or last name is sometime used as 1 character only.
+      // So we allow autocompleted search even if 1 character.
+      //
+      // [U+3041 - U+9FFF ... Full-width Katakana, Hiragana
+      //                      and CJK Ideograph
+      // [U+AC00 - U+D7FF ... Hangul
+      // [U+F900 - U+FFDC ... CJK compatibility ideograph
+      if (!aString.match(/[\u3041-\u9fff\uac00-\ud7ff\uf900-\uffdc]/))
+        return false;
+    }
+
+    let matches;
+    if (this.suffixTree) {
+      matches = this.suffixTree.findMatches(aString.toLowerCase());
+    }
+    else
+      matches = [];
+
+    // let's filter out duplicates due to identity/contact double-hits by
+    //  establishing a map based on the contact id for these guys.
+    // let's also favor identities as we do it, because that gets us the
+    //  most accurate gravat, potentially
+    let contactToThing = {};
+    for (let iMatch = 0; iMatch < matches.length; iMatch++) {
+      let thing = matches[iMatch];
+      if (thing.NOUN_ID == Gloda.NOUN_CONTACT && !(thing.id in contactToThing))
+        contactToThing[thing.id] = thing;
+      else if (thing.NOUN_ID == Gloda.NOUN_IDENTITY)
+        contactToThing[thing.contactID] = thing;
+    }
+    // and since we can now map from contacts down to identities, map contacts
+    //  to the first identity for them that we find...
+    matches = Object.keys(contactToThing).map(id => contactToThing[id]).
+      map(val => val.NOUN_ID == Gloda.NOUN_IDENTITY ? val : val.identities[0]);
+
+    let rows = matches.
+      map(match => new ResultRowSingle(match, "text", aResult.searchString));
+    aResult.addRows(rows);
+
+    // - match against database contacts / identities
+    let pending = {contactToThing: contactToThing, pendingCount: 2};
+
+    let contactQuery = Gloda.newQuery(Gloda.NOUN_CONTACT);
+    contactQuery.nameLike(contactQuery.WILDCARD, aString,
+	contactQuery.WILDCARD);
+    pending.contactColl = contactQuery.getCollection(this, aResult);
+    pending.contactColl.becomeExplicit();
+
+    let identityQuery = Gloda.newQuery(Gloda.NOUN_IDENTITY);
+    identityQuery.kind("email").valueLike(identityQuery.WILDCARD, aString,
+        identityQuery.WILDCARD);
+    pending.identityColl = identityQuery.getCollection(this, aResult);
+    pending.identityColl.becomeExplicit();
+
+    aResult._contactCompleterPending = pending;
+
+    return true;
+  },
+  onItemsAdded: function(aItems, aCollection) {
+  },
+  onItemsModified: function(aItems, aCollection) {
+  },
+  onItemsRemoved: function(aItems, aCollection) {
+  },
+  onQueryCompleted: function(aCollection) {
+    // handle the initial setup case...
+    if (aCollection.data == null) {
+      // cheat and explicitly add our own contact...
+      if (Gloda.myContact &&
+          !(Gloda.myContact.id in this.contactCollection._idMap))
+        this.contactCollection._onItemsAdded([Gloda.myContact]);
+
+      // the set of identities owned by the contacts is automatically loaded as part
+      //  of the contact loading...
+      // (but only if we actually have any contacts)
+      this.identityCollection =
+        this.contactCollection.subCollections[Gloda.NOUN_IDENTITY];
+
+      let contactNames = this.contactCollection.items.
+        map(c => c.name.replace(" ", "").toLowerCase() || "x");
+      // if we had no contacts, we will have no identity collection!
+      let identityMails;
+      if (this.identityCollection)
+        identityMails = this.identityCollection.items.
+          map(i => i.value.toLowerCase());
+
+      // The suffix tree takes two parallel lists; the first contains strings
+      //  while the second contains objects that correspond to those strings.
+      // In the degenerate case where identityCollection does not exist, it will
+      //  be undefined.  Calling concat with an argument of undefined simply
+      //  duplicates the list we called concat on, and is thus harmless.  Our
+      //  use of && on identityCollection allows its undefined value to be
+      //  passed through to concat.  identityMails will likewise be undefined.
+      this.suffixTree = new MultiSuffixTree(contactNames.concat(identityMails),
+        this.contactCollection.items.concat(this.identityCollection &&
+          this.identityCollection.items));
+
+      return;
+    }
+
+    // handle the completion case
+    let result = aCollection.data;
+    let pending = result._contactCompleterPending;
+
+    if (--pending.pendingCount == 0) {
+      let possibleDudes = [];
+
+      let contactToThing = pending.contactToThing;
+
+      let items;
+
+      // check identities first because they are better than contacts in terms
+      //  of display
+      items = pending.identityColl.items;
+      for (let iIdentity = 0; iIdentity < items.length; iIdentity++){
+        let identity = items[iIdentity];
+        if (!(identity.contactID in contactToThing)) {
+          contactToThing[identity.contactID] = identity;
+          possibleDudes.push(identity);
+          // augment the identity with its contact's popularity
+          identity.popularity = identity.contact.popularity;
+        }
+      }
+      items = pending.contactColl.items;
+      for (let iContact = 0; iContact < items.length; iContact++) {
+        let contact = items[iContact];
+        if (!(contact.id in contactToThing)) {
+          contactToThing[contact.id] = contact;
+          possibleDudes.push(contact.identities[0]);
+        }
+      }
+
+      // sort in order of descending popularity
+      possibleDudes.sort(this._popularitySorter);
+      let rows = possibleDudes.
+        map(dude => new ResultRowSingle(dude, "text", result.searchString));
+      result.addRows(rows);
+      result.markCompleted(this);
+
+      // the collections no longer care about the result, make it clear.
+      delete pending.identityColl.data;
+      delete pending.contactColl.data;
+      // the result object no longer needs us or our data
+      delete result._contactCompleterPending;
+    }
+  }
+};
+
+/**
+ * Complete tags that are used on contacts.
+ */
+function ContactTagCompleter() {
+  FreeTagNoun.populateKnownFreeTags();
+  this._buildSuffixTree();
+  FreeTagNoun.addListener(this);
+}
+ContactTagCompleter.prototype = {
+  _buildSuffixTree: function() {
+    let tagNames = [], tags = [];
+    for (let [tagName, tag] in Iterator(FreeTagNoun.knownFreeTags)) {
+      tagNames.push(tagName.toLowerCase());
+      tags.push(tag);
+    }
+    this._suffixTree = new MultiSuffixTree(tagNames, tags);
+    this._suffixTreeDirty = false;
+  },
+  onFreeTagAdded: function(aTag) {
+    this._suffixTreeDirty = true;
+  },
+  complete: function ContactTagCompleter_complete(aResult, aString) {
+    // now is not the best time to do this; have onFreeTagAdded use a timer.
+    if (this._suffixTreeDirty)
+      this._buildSuffixTree();
+
+    if (aString.length < 2)
+      return false; // no async mechanism that will add new rows
+
+    let tags = this._suffixTree.findMatches(aString.toLowerCase());
+    let rows = [];
+    for (let tag of tags) {
+      let query = Gloda.newQuery(Gloda.NOUN_CONTACT);
+      query.freeTags(tag);
+      let resRow = new ResultRowMulti(Gloda.NOUN_CONTACT, "tag", tag.name,
+                                      query);
+      rows.push(resRow);
+    }
+    aResult.addRows(rows);
+
+    return false; // no async mechanism that will add new rows
+  }
+};
+
+/**
+ * Complete tags that are used on messages
+ */
+function MessageTagCompleter() {
+  this._buildSuffixTree();
+}
+MessageTagCompleter.prototype = {
+  _buildSuffixTree: function MessageTagCompleter__buildSufficeTree() {
+    let tagNames = [], tags = [];
+    let tagArray = TagNoun.getAllTags();
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      tagNames.push(tag.tag.toLowerCase());
+      tags.push(tag);
+    }
+    this._suffixTree = new MultiSuffixTree(tagNames, tags);
+    this._suffixTreeDirty = false;
+  },
+  complete: function MessageTagCompleter_complete(aResult, aString) {
+    if (aString.length < 2)
+      return false;
+
+    let tags = this._suffixTree.findMatches(aString.toLowerCase());
+    let rows = [];
+    for (let tag of tags) {
+      let resRow = new ResultRowSingle(tag, "tag", tag.tag, TagNoun.id);
+      rows.push(resRow);
+    }
+    aResult.addRows(rows);
+
+    return false; // no async mechanism that will add new rows
+  }
+};
+
+/**
+ * Complete with helpful hints about full-text search
+ */
+function FullTextCompleter() {
+}
+FullTextCompleter.prototype = {
+  complete: function FullTextCompleter_complete(aResult, aSearchString) {
+    if (aSearchString.length < 4)
+      return false;
+    // We use code very similar to that in msg_search.js, except that we
+    // need to detect when we found phrases, as well as strip commas.
+    aSearchString = aSearchString.trim();
+    let terms = [];
+    let phraseFound = false;
+    while (aSearchString) {
+      let term = "";
+      if (aSearchString.startsWith('"')) {
+        let endIndex = aSearchString.indexOf(aSearchString[0], 1);
+        // eat the quote if it has no friend
+        if (endIndex == -1) {
+          aSearchString = aSearchString.substring(1);
+          continue;
+        }
+        phraseFound = true;
+        term = aSearchString.substring(1, endIndex).trim();
+        if (term)
+          terms.push(term);
+        aSearchString = aSearchString.substring(endIndex + 1);
+        continue;
+      }
+
+      let spaceIndex = aSearchString.indexOf(" ");
+      if (spaceIndex == -1) {
+        terms.push(aSearchString.replace(/,/g, ""));
+        break;
+      }
+
+      term = aSearchString.substring(0, spaceIndex).replace(/,/g, "");
+      if (term)
+        terms.push(term);
+      aSearchString = aSearchString.substring(spaceIndex+1);
+    }
+
+    if (terms.length == 1 && !phraseFound)
+      aResult.addRows([new ResultRowFullText(aSearchString, terms, "single")]);
+    else
+      aResult.addRows([new ResultRowFullText(aSearchString, terms, "all")]);
+
+    return false; // no async mechanism that will add new rows
+  }
+};
+
+var LOG;
+
+function nsAutoCompleteGloda() {
+  this.wrappedJSObject = this;
+  try {
+    // set up our awesome globals!
+    if (Gloda === null) {
+      let loadNS = {};
+      Cu.import("resource:///modules/gloda/public.js", loadNS);
+      Gloda = loadNS.Gloda;
+
+      Cu.import("resource:///modules/gloda/utils.js", loadNS);
+      GlodaUtils = loadNS.GlodaUtils;
+      Cu.import("resource:///modules/gloda/suffixtree.js", loadNS);
+      MultiSuffixTree = loadNS.MultiSuffixTree;
+      Cu.import("resource:///modules/gloda/noun_tag.js", loadNS);
+      TagNoun = loadNS.TagNoun;
+      Cu.import("resource:///modules/gloda/noun_freetag.js", loadNS);
+      FreeTagNoun = loadNS.FreeTagNoun;
+
+      Cu.import("resource:///modules/gloda/log4moz.js", loadNS);
+      LOG = loadNS["Log4Moz"].repository.getLogger("gloda.autocomp");
+    }
+
+    this.completers = [];
+    this.curResult = null;
+
+    this.completers.push(new FullTextCompleter());        // not async.
+    this.completers.push(new ContactIdentityCompleter()); // potentially async.
+    this.completers.push(new ContactTagCompleter());      // not async.
+    this.completers.push(new MessageTagCompleter());      // not async.
+  } catch (e) {
+    logException(e);
+  }
+}
+
+nsAutoCompleteGloda.prototype = {
+  classID: Components.ID("{3bbe4d77-3f70-4252-9500-bc00c26f476d}"),
+  QueryInterface: XPCOMUtils.generateQI([
+      Components.interfaces.nsIAutoCompleteSearch]),
+
+  startSearch: function(aString, aParam, aResult, aListener) {
+    try {
+      let result = new nsAutoCompleteGlodaResult(aListener, this, aString);
+      // save this for hacky access to the search.  I somewhat suspect we simply
+      //  should not be using the formal autocomplete mechanism at all.
+      // Used in glodacomplete.xml.
+      this.curResult = result;
+
+      // Guard against late async results being sent.
+      this.curResult.active = true;
+
+      if (aParam == "global") {
+        for (let completer of this.completers) {
+          // they will return true if they have something pending.
+          if (completer.complete(result, aString))
+            result.markPending(completer);
+        }
+      //} else {
+      //   It'd be nice to do autocomplete in the quicksearch modes based
+      //   on the specific values for that mode in the current view.
+      //   But we don't do that yet.
+      }
+
+      result.announceYourself();
+    } catch (e) {
+      logException(e);
+    }
+  },
+
+  stopSearch: function() {
+    this.curResult.active = false;
+  }
+};
+
+var components = [nsAutoCompleteGloda];
+var NSGetFactory = XPCOMUtils.generateNSGetFactory(components);

mailnews/db/gloda/components/gloda.manifest

@@ -0,0 +1,5 @@
+component {3bbe4d77-3f70-4252-9500-bc00c26f476d} glautocomp.js
+contract @mozilla.org/autocomplete/search;1?name=gloda {3bbe4d77-3f70-4252-9500-bc00c26f476d}
+component {8cddbbbc-7ced-46b0-a936-8cddd1928c24} jsmimeemitter.js
+contract @mozilla.org/gloda/jsmimeemitter;1 {8cddbbbc-7ced-46b0-a936-8cddd1928c24}
+category mime-emitter @mozilla.org/messenger/mimeemitter;1?type=application/x-js-mime-message @mozilla.org/gloda/jsmimeemitter;1

mailnews/db/gloda/components/jsmimeemitter.js

@@ -0,0 +1,493 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var kStateUnknown = 0;
+var kStateInHeaders = 1;
+var kStateInBody = 2;
+var kStateInAttachment = 3;
+
+/**
+ * When the saneBodySize flag is active, limit body parts to at most this many
+ *  bytes.  See |MsgHdrToMimeMessage| for more information on the flag.
+ *
+ * The choice of 20k was made on the very scientific basis of running a query
+ *  against my indexed e-mail and finding the point where these things taper
+ *  off.  I chose 20 because things had tapered off pretty firmly by 16, so
+ *  20 gave it some space and it was also the end of a mini-plateau.
+ */
+var MAX_SANE_BODY_PART_SIZE = 20 * 1024;
+
+/**
+ * Custom nsIMimeEmitter to build a sub-optimal javascript representation of a
+ *  MIME message.  The intent is that a better mechanism than is evolved to
+ *  provide a javascript-accessible representation of the message.
+ *
+ * Processing occurs in two passes.  During the first pass, libmime is parsing
+ *  the stream it is receiving, and generating header and body events for all
+ *  MimeMessage instances it encounters.  This provides us with the knowledge
+ *  of each nested message in addition to the top level message, their headers
+ *  and sort-of their bodies.  The sort-of is that we may get more than
+ *  would normally be displayed in cases involving multipart/alternatives.
+ * We have augmented libmime to have a notify_nested_options parameter which
+ *  is enabled when we are the consumer.  This option causes MimeMultipart to
+ *  always emit a content-type header (via addHeaderField), defaulting to
+ *  text/plain when an explicit value is not present.  Additionally,
+ *  addHeaderField is called with a custom "x-jsemitter-part-path" header with
+ *  the value being the part path (ex: 1.2.2).  Having the part path greatly
+ *  simplifies our life for building the part hierarchy.
+ * During the second pass, the libmime object model is traversed, generating
+ *  attachment notifications for all leaf nodes.  From our perspective, this
+ *  means file attachments and embedded messages (message/rfc822).  We use this
+ *  pass to create the attachment objects proper, which we then substitute into
+ *  the part tree we have already built.
+ */
+function MimeMessageEmitter() {
+  this._mimeMsg = {};
+  Cu.import("resource:///modules/gloda/mimemsg.js", this._mimeMsg);
+  this._utils = {};
+  Cu.import("resource:///modules/gloda/utils.js", this._utils);
+
+  this._url = null;
+  this._partRE = this._utils.GlodaUtils.PART_RE;
+
+  this._outputListener = null;
+
+  this._curPart = null;
+  this._curAttachment = null;
+  this._partMap = {};
+  this._bogusPartTranslation = {};
+
+  this._state = kStateUnknown;
+
+  this._writeBody = false;
+}
+
+var deathToNewlines = /\n/g;
+
+MimeMessageEmitter.prototype = {
+  classID: Components.ID("{8cddbbbc-7ced-46b0-a936-8cddd1928c24}"),
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIMimeEmitter]),
+
+  initialize: function mime_emitter_initialize(aUrl, aChannel, aFormat) {
+    this._url = aUrl;
+    this._curPart = new this._mimeMsg.MimeMessage();
+    // the partName is intentionally ""!  not a place-holder!
+    this._curPart.partName = "";
+    this._curAttachment = "";
+    this._partMap[""] = this._curPart;
+
+    // pull options across...
+    let options = this._mimeMsg.MsgHdrToMimeMessage.OPTION_TUNNEL;
+    this._saneBodySize = (options && ("saneBodySize" in options)) ?
+                           options.saneBodySize : false;
+
+    this._mimeMsg.MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aUrl.spec] =
+      this._curPart;
+  },
+
+  complete: function mime_emitter_complete() {
+    this._url = null;
+
+    this._outputListener = null;
+
+    this._curPart = null;
+    this._curAttachment = null;
+    this._partMap = null;
+    this._bogusPartTranslation = null;
+  },
+
+  setPipe: function mime_emitter_setPipe(aInputStream, aOutputStream) {
+    // we do not care about these
+  },
+  set outputListener(aListener) {
+    this._outputListener = aListener;
+  },
+  get outputListener() {
+    return this._outputListener;
+  },
+
+  _stripParams: function mime_emitter__stripParams(aValue) {
+    let indexSemi = aValue.indexOf(";");
+    if (indexSemi >= 0)
+        aValue = aValue.substring(0, indexSemi);
+    return aValue;
+  },
+
+  _beginPayload: function mime_emitter__beginPayload(aContentType) {
+    let contentTypeNoParams = this._stripParams(aContentType).toLowerCase();
+    if (contentTypeNoParams == "text/plain" ||
+        contentTypeNoParams == "text/html" ||
+        contentTypeNoParams == "text/enriched") {
+      this._curPart = new this._mimeMsg.MimeBody(contentTypeNoParams);
+      this._writeBody = true;
+    }
+    else if (contentTypeNoParams == "message/rfc822") {
+      // startHeader will take care of this
+      this._curPart = new this._mimeMsg.MimeMessage();
+      // do not fall through into the content-type setting case; this
+      //  content-type needs to get clobbered by the actual content-type of
+      //  the enclosed message.
+      this._writeBody = false;
+      return;
+    }
+    // this is going to fall-down with TNEF encapsulation and such, we really
+    //  need to just be consuming the object model.
+    else if (contentTypeNoParams.startsWith("multipart/")) {
+      this._curPart = new this._mimeMsg.MimeContainer(contentTypeNoParams);
+      this._writeBody = false;
+    }
+    else {
+      this._curPart = new this._mimeMsg.MimeUnknown(contentTypeNoParams);
+      this._writeBody = false;
+    }
+    // put the full content-type in the headers and normalize out any newlines
+    this._curPart.headers["content-type"] =
+      [aContentType.replace(deathToNewlines, "")];
+  },
+
+  // ----- Header Routines
+  /**
+   * StartHeader provides the base case for our processing.  It is the first
+   *  notification we receive when processing begins on the outer rfc822
+   *  message.  We do not receive an x-jsemitter-part-path notification for the
+   *  message, but the aIsRootMailHeader tells us everything we need to know.
+   *  (Or it would if we hadn't already set everything up in initialize.)
+   *
+   * When dealing with nested RFC822 messages, we will receive the
+   *  addHeaderFields for the content-type and the x-jsemitter-part-path
+   *  prior to the startHeader call.  This is because the MIME multipart
+   *  container that holds the message is the one generating the notification.
+   *  For that reason, we do not process them here, but instead in
+   *  addHeaderField and _beginPayload.
+   *
+   * We do need to track our state for addHeaderField's benefit though.
+   */
+  startHeader: function mime_emitter_startHeader(aIsRootMailHeader,
+      aIsHeaderOnly, aMsgID, aOutputCharset) {
+    this._state = kStateInHeaders;
+  },
+  /**
+   * Receives a header field name and value for the current MIME part, which
+   *  can be an rfc822/message or one of its sub-parts.
+   *
+   * The emitter architecture treats rfc822/messages as special because it was
+   *  architected around presentation.  In that case, the organizing concept
+   *  is the single top-level rfc822/message.  (It did not 'look into' nested
+   *  messages in most cases.)
+   * As a result the interface is biased towards being 'in the headers' or
+   *  'in the body', corresponding to calls to startHeader and startBody,
+   *  respectively.
+   * This information is interesting to us because the message itself is an
+   *  odd pseudo-mime-part.  Because it has only one child, its headers are,
+   *  in a way, its payload, but they also serve as the description of its
+   *  MIME child part.  This introduces a complication in that we see the
+   *  content-type for the message's "body" part before we actually see any
+   *  of the headers.  To deal with this, we punt on the construction of the
+   *  body part to the call to startBody() and predicate our logic on the
+   *  _state field.
+   */
+  addHeaderField: function mime_emitter_addHeaderField(aField, aValue) {
+    if (this._state == kStateInBody) {
+      aField = aField.toLowerCase();
+      if (aField == "content-type")
+        this._beginPayload(aValue, true);
+      else if (aField == "x-jsemitter-part-path") {
+        // This is either naming the current part, or referring to an already
+        //  existing part (in the case of multipart/related on its second pass).
+        // As such, check if the name already exists in our part map.
+        let partName = this._stripParams(aValue);
+        // if it does, then make the already-existing part at that path current
+        if (partName in this._partMap) {
+          this._curPart = this._partMap[partName];
+          this._writeBody = "body" in this._curPart;
+        }
+        // otherwise, name the part we are holding onto and place it.
+        else {
+          this._curPart.partName = partName;
+          this._placePart(this._curPart);
+        }
+      }
+      else if (aField == "x-jsemitter-encrypted" && aValue == "1") {
+        this._curPart.isEncrypted = true;
+      }
+      // There is no other field to be emitted in the body case other than the
+      //  ones we just handled.  (They were explicitly added for the js
+      //  emitter.)
+    }
+    else if (this._state == kStateInHeaders) {
+      let lowerField = aField.toLowerCase();
+      if (lowerField in this._curPart.headers)
+        this._curPart.headers[lowerField].push(aValue);
+      else
+        this._curPart.headers[lowerField] = [aValue];
+    }
+  },
+  addAllHeaders: function mime_emitter_addAllHeaders(aAllHeaders, aHeaderSize) {
+    // This is called by the parsing code after the calls to AddHeaderField (or
+    //  AddAttachmentField if the part is an attachment), and seems to serve
+    //  a specialized, quasi-redundant purpose.  (nsMimeBaseEmitter creates a
+    //  nsIMimeHeaders instance and hands it to the nsIMsgMailNewsUrl.)
+    // nop
+  },
+  writeHTMLHeaders: function mime_emitter_writeHTMLHeaders(aName) {
+    // It doesn't look like this should even be part of the interface; I think
+    //  only the nsMimeHtmlDisplayEmitter::EndHeader call calls this signature.
+    // nop
+  },
+  endHeader: function mime_emitter_endHeader(aName) {
+  },
+  updateCharacterSet: function mime_emitter_updateCharacterSet(aCharset) {
+    // we do not need to worry about this.  it turns out this notification is
+    //  exclusively for the benefit of the UI.  libmime, believe it or not,
+    //  is actually doing the right thing under the hood and handles all the
+    //  encoding issues for us.
+    // so, get ready for the only time you will ever hear this:
+    //  three cheers for libmime!
+  },
+
+  /**
+   * Place a part in its proper location; requires the parent to be present.
+   * However, we no longer require in-order addition of children.  (This is
+   *  currently a hedge against extension code doing wacky things.  Our
+   *  motivating use-case is multipart/related which actually does generate
+   *  everything in order on its first pass, but has a wacky second pass. It
+   *  does not actually trigger the out-of-order code because we have
+   *  augmented the libmime code to generate its x-jsemitter-part-path info
+   *  a second time, in which case we reuse the part we already created.)
+   *
+   * @param aPart Part to place.
+   */
+  _placePart: function(aPart) {
+    let partName = aPart.partName;
+    this._partMap[partName] = aPart;
+
+    let [storagePartName, parentName, parentPart] = this._findOrCreateParent(partName);
+    let lastDotIndex = storagePartName.lastIndexOf(".");
+    if (parentPart !== undefined) {
+      let indexInParent = parseInt(storagePartName.substring(lastDotIndex+1)) - 1;
+      // handle out-of-order notification...
+      if (indexInParent < parentPart.parts.length)
+        parentPart.parts[indexInParent] = aPart;
+      else {
+        while (indexInParent > parentPart.parts.length)
+          parentPart.parts.push(null);
+        parentPart.parts.push(aPart);
+      }
+    }
+  },
+
+  /**
+   * In case the MIME structure is wrong, (i.e. we have no parent to add the
+   *  current part to), this function recursively makes sure we create the
+   *  missing bits in the hierarchy.
+   * What happens in the case of encrypted emails (mimecryp.cpp):
+   *  1. is the message
+   *  1.1 doesn't exist
+   *  1.1.1 is the multipart/alternative that holds the text/plain and text/html
+   *  1.1.1.1 is text/plain
+   *  1.1.1.2 is text/html
+   * This function fills the missing bits.
+   */
+  _findOrCreateParent: function (aPartName) {
+    let partName = aPartName + "";
+    let parentName = partName.substring(0, partName.lastIndexOf("."));
+    let parentPart;
+    if (parentName in this._partMap) {
+      parentPart = this._partMap[parentName]
+      let lastDotIndex = partName.lastIndexOf(".");
+      let indexInParent = parseInt(partName.substring(lastDotIndex+1)) - 1;
+      if ("parts" in parentPart && indexInParent == parentPart.parts.length - 1)
+        return [partName, parentName, parentPart];
+      else
+        return this._findAnotherContainer(aPartName);
+    } else {
+      // Find the grandparent
+      let [, grandParentName, grandParentPart] = this._findOrCreateParent(parentName);
+      // Create the missing part.
+      let parentPart = new this._mimeMsg.MimeContainer("multipart/fake-container");
+      // Add it to the grandparent, remember we added it in the hierarchy.
+      grandParentPart.parts.push(parentPart);
+      this._partMap[parentName] = parentPart;
+      return [partName, parentName, parentPart];
+    }
+  },
+
+  /**
+   * In the case of UUEncoded attachments, libmime tells us about the attachment
+   *  as a child of a MimeBody. This obviously doesn't make us happy, so in case
+   *  libmime wants us to attach an attachment to something that's not a
+   *  container, we walk up the mime tree to find a suitable container to hold
+   *  the attachment.
+   * The results are cached so that they're consistent accross calls — this
+   *  ensures the call to _replacePart works fine.
+   */
+  _findAnotherContainer: function(aPartName) {
+    if (aPartName in this._bogusPartTranslation)
+      return this._bogusPartTranslation[aPartName];
+
+    let parentName = aPartName + "";
+    let parentPart;
+    while (!(parentPart && "parts" in parentPart) && parentName.length) {
+      parentName = parentName.substring(0, parentName.lastIndexOf("."));
+      parentPart = this._partMap[parentName];
+    }
+    let childIndex = parentPart.parts.length;
+    let fallbackPartName = (parentName ? parentName +"." : "")+(childIndex+1);
+    return (this._bogusPartTranslation[aPartName] = [fallbackPartName, parentName, parentPart]);
+  },
+
+  /**
+   * In the case of attachments, we need to replace an existing part with a
+   *  more representative part...
+   *
+   * @param aPart Part to place.
+   */
+  _replacePart: function(aPart) {
+    // _partMap always maps the libmime names to parts
+    let partName = aPart.partName;
+    this._partMap[partName] = aPart;
+
+    let [storagePartName, parentName, parentPart] = this._findOrCreateParent(partName);
+
+    let childNamePart = storagePartName.substring(storagePartName.lastIndexOf(".")+1);
+    let childIndex = parseInt(childNamePart) - 1;
+
+    // The attachment has been encapsulated properly in a MIME part (most of
+    // the cases). This does not hold for UUencoded-parts for instance (see
+    // test_mime_attachments_size.js for instance).
+    if (childIndex < parentPart.parts.length) {
+      let oldPart = parentPart.parts[childIndex];
+      parentPart.parts[childIndex] = aPart;
+      // copy over information from the original part
+      aPart.parts = oldPart.parts;
+      aPart.headers = oldPart.headers;
+      aPart.isEncrypted = oldPart.isEncrypted;
+    } else {
+      parentPart.parts[childIndex] = aPart;
+    }
+  },
+
+  // ----- Attachment Routines
+  // The attachment processing happens after the initial streaming phase (during
+  //  which time we receive the messages, both bodies and headers).  Our caller
+  //  traverses the libmime child object hierarchy, emitting an attachment for
+  //  each leaf object or sub-message.
+  startAttachment: function mime_emitter_startAttachment(aName, aContentType,
+      aUrl, aIsExternalAttachment) {
+    this._state = kStateInAttachment;
+
+    // we need to strip our magic flags from the URL; this regexp matches all
+    // the specific flags that the jsmimeemitter understands (we abuse the URL
+    // parameters to pass information all the way to here)
+    aUrl = aUrl.replace(/((header=filter|emitter=js|fetchCompleteMessage=(true|false)|examineEncryptedParts=(true|false)))&?/g, "");
+    // the url should contain a part= piece that tells us the part name, which
+    // we then use to figure out where to place that part if it's a real
+    // attachment.
+    let partMatch, partName;
+    if (aUrl.startsWith("http") || aUrl.startsWith("file")) {
+      // if we have a remote url, unlike non external mail part urls, it may also
+      // contain query strings starting with ?; PART_RE does not handle this.
+      partMatch = aUrl.match(/[?&]part=[^&]+$/);
+      partMatch = partMatch && partMatch[0];
+      partName = partMatch && partMatch.split("part=")[1];
+    }
+    else {
+      partMatch = this._partRE.exec(aUrl);
+      partName = partMatch && partMatch[1];
+    }
+    this._curAttachment = partName;
+
+    if (aContentType == "message/rfc822") {
+      // we want to offer extension authors a way to see attachments as the
+      // message readers sees them, which means attaching an extra url property
+      // to the part that was already created before
+      if (partName) {
+        // we disguise this MimeMessage into something that can be used as a
+        // MimeAttachment so that it is transparent for the user code
+        this._partMap[partName].url = aUrl;
+        this._partMap[partName].isExternal = aIsExternalAttachment;
+        this._partMap[partName].name = aName;
+        this._partMap[partName].isRealAttachment = true;
+      }
+    }
+    else if (partName) {
+      let part = new this._mimeMsg.MimeMessageAttachment(partName,
+          aName, aContentType, aUrl, aIsExternalAttachment);
+      // replace the existing part with the attachment...
+      this._replacePart(part);
+    }
+  },
+  addAttachmentField: function mime_emitter_addAttachmentField(aField, aValue) {
+    // What gets passed in here is X-Mozilla-PartURL with a value that
+    //  is completely identical to aUrl from the call to startAttachment.
+    //  (it's the same variable they use in each case).  As such, there is
+    //  no reason to handle that here.
+    // However, we also pass information about the size of the attachment, and
+    //  that we want to handle
+    if (aField == "X-Mozilla-PartSize" && (this._curAttachment in this._partMap))
+      this._partMap[this._curAttachment].size = parseInt(aValue);
+  },
+  endAttachment: function mime_emitter_endAttachment() {
+    // don't need to do anything here, since we don't care about the headers.
+  },
+  endAllAttachments: function mime_emitter_endAllAttachments() {
+    // nop
+  },
+
+  // ----- Body Routines
+  /**
+   * We don't get an x-jsemitter-part-path for the message body, and we ignored
+   *  our body part's content-type in addHeaderField, so this serves as our
+   *  notice to set up the part (giving it a name).
+   */
+  startBody: function mime_emitter_startBody(aIsBodyOnly, aMsgID, aOutCharset) {
+    this._state = kStateInBody;
+
+    let subPartName = (this._curPart.partName == "") ?
+                        "1" :
+                        this._curPart.partName + ".1";
+    this._beginPayload(this._curPart.get("content-type", "text/plain"));
+    this._curPart.partName = subPartName;
+    this._placePart(this._curPart);
+  },
+
+  /**
+   * Write to the body.  When saneBodySize is active, we stop adding if we are
+   *  already at the limit for this body part.
+   */
+  writeBody: function mime_emitter_writeBody(aBuf, aSize, aOutAmountWritten) {
+    if (this._writeBody &&
+        (!this._saneBodySize ||
+         this._curPart.size < MAX_SANE_BODY_PART_SIZE))
+      this._curPart.appendBody(aBuf);
+  },
+
+  endBody: function mime_emitter_endBody() {
+  },
+
+  // ----- Generic Write (confusing)
+  // (binary data writing...)
+  write: function mime_emitter_write(aBuf, aSize, aOutAmountWritten) {
+    // we don't actually ever get called because we don't have the attachment
+    //  binary payloads pass through us, but we do the following just in case
+    //  we did get called (otherwise the caller gets mad and throws exceptions).
+    aOutAmountWritten.value = aSize;
+  },
+
+  // (string writing)
+  utilityWrite: function mime_emitter_utilityWrite(aBuf) {
+    this.write(aBuf, aBuf.length, {});
+  },
+};
+
+var components = [MimeMessageEmitter];
+var NSGetFactory = XPCOMUtils.generateNSGetFactory(components);

mailnews/db/gloda/components/moz.build

@@ -0,0 +1,11 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXTRA_COMPONENTS += [
+    'glautocomp.js',
+    'gloda.manifest',
+    'jsmimeemitter.js',
+]
+

mailnews/db/gloda/content/glodacomplete.css

@@ -0,0 +1,94 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+textbox[type="glodacomplete"] {
+  -moz-binding: url("chrome://global/content/bindings/autocomplete.xml#autocomplete");
+}
+
+panel[type="glodacomplete-richlistbox"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#glodacomplete-rich-result-popup");
+}
+
+.autocomplete-richlistbox {
+  -moz-binding: url("chrome://global/content/bindings/autocomplete.xml#autocomplete-richlistbox");
+  -moz-user-focus: ignore;
+  -moz-appearance: none;
+}
+
+.autocomplete-richlistbox > scrollbox {
+  overflow-x: hidden !important;
+}
+
+.explanation, .gloda-single-identity {
+  margin-inline-start: 1em;
+  margin-top: 2px;
+  margin-bottom: 2px;
+}
+
+.ac-comment {
+  font-size: 1.1em;
+  margin-inline-start: 0;
+}
+
+.ac-url-text {
+  color: -moz-nativehyperlinktext;
+  font-size: 0.95em;
+}
+
+span.ac-emphasize-text {
+  font-weight: bold;
+}
+
+.ac-url-text[selected="true"] {
+  color: inherit !important;
+}
+
+.gloda-single-identity[selected="true"] .ac-url{
+  color: white;
+}
+
+.parameters {
+  font-style: italic;
+  margin-inline-start: 1em;
+}
+
+.autocomplete-richlistitem[type="gloda-single-tag"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-single-tag-item");
+  overflow: -moz-hidden-unscrollable;
+}
+
+.autocomplete-richlistitem[type="gloda-single-identity"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-single-identity-item");
+  -moz-box-orient: vertical;
+  overflow: -moz-hidden-unscrollable;
+}
+
+.autocomplete-richlistitem[type="gloda-fulltext-single"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-fulltext-single-item");
+  overflow: -moz-hidden-unscrollable;
+}
+
+.autocomplete-richlistitem[type="gloda-fulltext-any"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-fulltext-any-item");
+  overflow: -moz-hidden-unscrollable;
+}
+
+.autocomplete-richlistitem[type="gloda-fulltext-all"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-fulltext-all-item");
+  overflow: -moz-hidden-unscrollable;
+}
+
+richlistitem[type="gloda-contact-chunk"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-contact-chunk");
+  -moz-box-orient: vertical;
+  overflow: -moz-hidden-unscrollable;
+}
+
+.autocomplete-richlistitem[type="gloda-multi"] {
+  -moz-binding: url("chrome://gloda/content/glodacomplete.xml#gloda-multi-item");
+  -moz-box-orient: vertical;
+  overflow: -moz-hidden-unscrollable;
+}
+
+/* .autocomplete-history-dropmarker wants to be optional, but we don't care */

mailnews/db/gloda/content/glodacomplete.xml

@@ -0,0 +1,644 @@
+<?xml version="1.0"?>
+
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<bindings id="autocompleteBindings"
+          xmlns="http://www.mozilla.org/xbl"
+          xmlns:html="http://www.w3.org/1999/xhtml"
+          xmlns:xul="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
+          xmlns:xbl="http://www.mozilla.org/xbl">
+
+  <binding id="glodacomplete-rich-result-popup" extends="chrome://global/content/bindings/autocomplete.xml#autocomplete-rich-result-popup">
+    <implementation implements="nsIAutoCompletePopup">
+      <method name="_appendCurrentResult">
+        <body>
+          <![CDATA[
+          var controller = this.mInput.controller;
+
+          // Process maxRows per chunk to improve performance and user experience
+          for (let i = 0; i < this.maxRows; i++) {
+            if (this._currentIndex >= this._matchCount)
+              return;
+
+            var existingItemsCount = this.richlistbox.childNodes.length;
+            var item;
+
+            // trim the leading/trailing whitespace
+            var trimmedSearchString = controller.searchString.trim();
+
+            // Unescape the URI spec for showing as an entry in the popup
+            let url = Components.classes["@mozilla.org/intl/texttosuburi;1"].
+              getService(Components.interfaces.nsITextToSubURI).
+              unEscapeURIForUI("UTF-8", controller.getValueAt(this._currentIndex));
+
+            // Unlike our superclass, we create nodes every time because we have
+            //  heterogeneous results and we cannot rely on the XBL bindings to
+            //  to change fast enough.
+            item = document.createElementNS("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul", "richlistitem");
+
+            var glodaCompleter =  Components.
+              classes["@mozilla.org/autocomplete/search;1?name=gloda"].
+              getService(). //Components.interfaces.nsIAutoCompleteSearch)
+              wrappedJSObject;
+            var result = glodaCompleter.curResult;
+
+            // set these attributes before we set the class
+            // so that we can use them from the contructor
+            var row = result.getObjectAt(this._currentIndex);
+            var obj = row.item;
+            item.setAttribute("text", trimmedSearchString);
+            item.setAttribute("type", result.getStyleAt(this._currentIndex));
+
+            item.row = row;
+
+            // set the class at the end so we can use the attributes
+            // in the xbl constructor
+            item.className = "autocomplete-richlistitem";
+            this.richlistbox.appendChild(item);
+
+            this._currentIndex++;
+          }
+
+          // yield after each batch of items so that typing the url bar is responsive
+          setTimeout(() => this._appendCurrentResult(), 0);
+        ]]>
+        </body>
+      </method>
+      <method name="_invalidate">
+        <body>
+          <![CDATA[
+          setTimeout(() => this.adjustHeight(), 0);
+
+          // remove all child nodes because we never want to reuse them.
+          while (this.richlistbox.hasChildNodes())
+            this.richlistbox.lastChild.remove();
+
+          this._currentIndex = 0;
+          this._appendCurrentResult();
+        ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+  <!-- This is autocomplete.xml's autocomplete-richlistitem duplicated and
+       modified to include its useful helper functions, but eliminating anything
+       that assumes specific content sub-items.  Namely, url/title/etc.  -->
+  <binding id="glodacomplete-base-richlistitem" extends="chrome://global/content/bindings/richlistbox.xml#richlistitem">
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <field name="_boundaryCutoff">null</field>
+
+      <property name="boundaryCutoff" readonly="true">
+        <getter>
+          <![CDATA[
+          if (!this._boundaryCutoff) {
+            this._boundaryCutoff =
+              Services.prefs
+                      .getIntPref("toolkit.autocomplete.richBoundaryCutoff");
+          }
+          return this._boundaryCutoff;
+          ]]>
+        </getter>
+      </property>
+
+      <method name="_getBoundaryIndices">
+        <parameter name="aText"/>
+        <parameter name="aSearchTokens"/>
+        <body>
+          <![CDATA[
+          // Short circuit for empty search ([""] == "")
+          if (aSearchTokens == "")
+            return [0, aText.length];
+
+          // Find which regions of text match the search terms
+          let regions = [];
+          for (let search of aSearchTokens) {
+            let matchIndex;
+            let startIndex = 0;
+            let searchLen = search.length;
+
+            // Find all matches of the search terms, but stop early for perf
+            let lowerText = aText.toLowerCase().substr(0, this.boundaryCutoff);
+            while ((matchIndex = lowerText.indexOf(search, startIndex)) >= 0) {
+              // Start the next search from where this one finished
+              startIndex = matchIndex + searchLen;
+              regions.push([matchIndex, startIndex]);
+            }
+          }
+
+          // Sort the regions by start position then end position
+          regions = regions.sort(function(a, b) {
+            let start = a[0] - b[0];
+            return (start == 0) ? a[1] - b[1] : start;
+          });
+
+          // Generate the boundary indices from each region
+          let start = 0;
+          let end = 0;
+          let boundaries = [];
+          let len = regions.length;
+          for (let i = 0; i < len; i++) {
+            // We have a new boundary if the start of the next is past the end
+            let region = regions[i];
+            if (region[0] > end) {
+              // First index is the beginning of match
+              boundaries.push(start);
+              // Second index is the beginning of non-match
+              boundaries.push(end);
+
+              // Track the new region now that we've stored the previous one
+              start = region[0];
+            }
+
+            // Push back the end index for the current or new region
+            end = Math.max(end, region[1]);
+          }
+
+          // Add the last region
+          boundaries.push(start);
+          boundaries.push(end);
+
+          // Put on the end boundary if necessary
+          if (end < aText.length)
+            boundaries.push(aText.length);
+
+          // Skip the first item because it's always 0
+          return boundaries.slice(1);
+          ]]>
+        </body>
+      </method>
+
+      <method name="_getSearchTokens">
+        <parameter name="aSearch"/>
+        <body>
+          <![CDATA[
+          let search = aSearch.toLowerCase();
+          return search.split(/\s+/);
+          ]]>
+        </body>
+      </method>
+
+      <method name="_needsAlternateEmphasis">
+        <parameter name="aText"/>
+        <body>
+          <![CDATA[
+          for (let i = aText.length; --i >= 0; ) {
+            let charCode = aText.charCodeAt(i);
+            // Arabic, Syriac, Indic languages are likely to have ligatures
+            // that are broken when using the main emphasis styling
+            if (0x0600 <= charCode && charCode <= 0x109F)
+              return true;
+          }
+
+          return false;
+          ]]>
+        </body>
+      </method>
+
+      <method name="_setUpDescription">
+        <parameter name="aDescriptionElement"/>
+        <parameter name="aText"/>
+        <body>
+          <![CDATA[
+          // Get rid of all previous text
+          while (aDescriptionElement.hasChildNodes())
+            aDescriptionElement.lastChild.remove();
+
+          // Get the indices that separate match and non-match text
+          let search = this.getAttribute("text");
+          let tokens = this._getSearchTokens(search);
+          let indices = this._getBoundaryIndices(aText, tokens);
+
+          // If we're searching for something that needs alternate emphasis,
+          // we'll need to check the text that we match
+          let checkAlt = this._needsAlternateEmphasis(search);
+
+          let next;
+          let start = 0;
+          let len = indices.length;
+          // Even indexed boundaries are matches, so skip the 0th if it's empty
+          for (let i = indices[0] == 0 ? 1 : 0; i < len; i++) {
+            next = indices[i];
+            let text = aText.substr(start, next - start);
+            start = next;
+
+            if (i % 2 == 0) {
+              // Emphasize the text for even indices
+              let span = aDescriptionElement.appendChild(
+                document.createElementNS("http://www.w3.org/1999/xhtml", "span"));
+              span.className = checkAlt && this._needsAlternateEmphasis(text) ?
+                "ac-emphasize-alt" : "ac-emphasize-text";
+              span.textContent = text;
+            } else {
+              // Otherwise, it's plain text
+              aDescriptionElement.appendChild(document.createTextNode(text));
+            }
+          }
+          ]]>
+        </body>
+      </method>
+
+      <method name="_setUpOverflow">
+        <parameter name="aParentBox"/>
+        <parameter name="aEllipsis"/>
+        <body>
+          <![CDATA[
+          // Hide the ellipsis incase there's just enough to not underflow
+          aEllipsis.hidden = true;
+
+          // Start with the parent's width and subtract off its children
+          let tooltip = [];
+          let children = aParentBox.childNodes;
+          let widthDiff = aParentBox.boxObject.width;
+
+          for (let i = 0; i < children.length; i++) {
+            // Only consider a child if it actually takes up space
+            let childWidth = children[i].boxObject.width;
+            if (childWidth > 0) {
+              // Subtract a little less to account for subpixel rounding
+              widthDiff -= childWidth - .5;
+
+              // Add to the tooltip if it's not hidden and has text
+              let childText = children[i].textContent;
+              if (childText)
+                tooltip.push(childText);
+            }
+          }
+
+          // If the children take up more space than the parent.. overflow!
+          if (widthDiff < 0) {
+            // Re-show the ellipsis now that we know it's needed
+            aEllipsis.hidden = false;
+
+            // Separate text components with a ndash --
+            aParentBox.tooltipText = tooltip.join(" \u2013 ");
+          }
+          ]]>
+        </body>
+      </method>
+
+      <method name="_doUnderflow">
+        <parameter name="aName"/>
+        <body>
+          <![CDATA[
+          // Hide the ellipsis right when we know we're underflowing instead of
+          // waiting for the timeout to trigger the _setUpOverflow calculations
+          this[aName + "Box"].tooltipText = "";
+          this[aName + "OverflowEllipsis"].hidden = true;
+          ]]>
+        </body>
+      </method>
+
+    </implementation>
+  </binding>
+
+  <binding id="gloda-single-tag-item" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content orient="vertical">
+      <xul:description anonid="explanation" class="explanation gloda-single"/>
+    </content>
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <constructor>
+        <![CDATA[
+            this._explanation = document.getAnonymousElementByAttribute(this, "anonid", "explanation");
+
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            return "tag " + this.row.item.tag;
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+            let label = gGlodaCompleteStrings.get("glodaComplete.messagesTagged.label");
+            this._explanation.value = label.replace("#1", this.row.item.tag);
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+
+  <binding id="gloda-fulltext-single-item" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content orient="vertical">
+      <xul:description anonid="explanation" class="explanation gloda-fulltext-single"/>
+      <xul:description anonid="parameters"/>
+    </content>
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <constructor>
+        <![CDATA[
+            this._explanation = document.getAnonymousElementByAttribute(this, "anonid", "explanation");
+
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            return "full text search: " + this.row.item;
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+            let label = gGlodaCompleteStrings.get("glodaComplete.messagesMentioning.label");
+            this._explanation.value = label.replace("#1", this.row.item);
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+  <binding id="gloda-fulltext-all-item" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content orient="vertical">
+      <xul:description anonid="explanation" class="explanation"/>
+    </content>
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <constructor>
+        <![CDATA[
+            this._explanation = document.getAnonymousElementByAttribute(this, "anonid", "explanation");
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            return "full text search: " + this.row.item; // what is this for? l10n?
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+          let label = gGlodaCompleteStrings.get("glodaComplete.messagesMentioningMany.label");
+          this._explanation.value = label.replace("#1", this.row.words.join(", "));
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+  <binding id="gloda-single-identity-item" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content>
+      <xul:hbox class="gloda-single-identity">
+        <xul:image anonid="picture" class="picture"/>
+        <xul:vbox>
+          <xul:hbox>
+            <xul:hbox anonid="name-box" class="ac-title" flex="1"
+                      onunderflow="_doUnderflow('_name');">
+              <xul:description anonid="name" class="ac-normal-text ac-comment"
+                               xbl:inherits="selected"/>
+            </xul:hbox>
+            <xul:label anonid="name-overflow-ellipsis" xbl:inherits="selected"
+                       class="ac-ellipsis-after ac-comment" hidden="true"/>
+          </xul:hbox>
+          <xul:hbox>
+            <xul:hbox anonid="identity-box" class="ac-url" flex="1"
+                      onunderflow="_doUnderflow('_identity');">
+              <xul:description anonid="identity" class="ac-normal-text ac-url-text"
+                               xbl:inherits="selected"/>
+            </xul:hbox>
+            <xul:label anonid="identity-overflow-ellipsis" xbl:inherits="selected"
+                       class="ac-ellipsis-after ac-url-text" hidden="true"/>
+            <xul:image anonid="type-image" class="ac-type-icon"/>
+          </xul:hbox>
+        </xul:vbox>
+      </xul:hbox>
+    </content>
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <constructor>
+        <![CDATA[
+            let ellipsis = "\u2026";
+            try {
+              ellipsis = Services.prefs.getComplexValue("intl.ellipsis",
+                  Components.interfaces.nsIPrefLocalizedString).data;
+            } catch (ex) {
+              // Do nothing.. we already have a default
+            }
+
+            this._identityOverflowEllipsis = document.getAnonymousElementByAttribute(this, "anonid", "identity-overflow-ellipsis");
+            this._nameOverflowEllipsis = document.getAnonymousElementByAttribute(this, "anonid", "name-overflow-ellipsis");
+
+            this._identityOverflowEllipsis.value = ellipsis;
+            this._nameOverflowEllipsis.value = ellipsis;
+
+            this._typeImage = document.getAnonymousElementByAttribute(this, "anonid", "type-image");
+
+            this._identityBox = document.getAnonymousElementByAttribute(this, "anonid", "identity-box");
+            this._identity = document.getAnonymousElementByAttribute(this, "anonid", "identity");
+
+            this._nameBox = document.getAnonymousElementByAttribute(this, "anonid", "name-box");
+            this._name = document.getAnonymousElementByAttribute(this, "anonid", "name");
+            
+            this._picture = document.getAnonymousElementByAttribute(this, "anonid", "picture");
+
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            var identity = this.row.item;
+            return identity.accessibleLabel;
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+          var identity = this.row.item;
+          
+          if (identity == null)
+            return;
+          
+          // I guess we should get the picture size from CSS or something?
+          this._picture.src = identity.pictureURL(32);
+          
+          // Emphasize the matching search terms for the description
+          this._setUpDescription(this._name, identity.contact.name);
+          this._setUpDescription(this._identity, identity.value);
+
+          // Set up overflow on a timeout because the contents of the box
+          // might not have a width yet even though we just changed them
+          setTimeout(this._setUpOverflow, 0, this._nameBox, this._nameOverflowEllipsis);
+          setTimeout(this._setUpOverflow, 0, this._identityBox, this._identityOverflowEllipsis);
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+  <binding id="gloda-contact-chunk" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content orient="horizontal">
+      <xul:image anonid="picture"/>
+      <xul:vbox>
+        <xul:hbox>
+          <xul:hbox anonid="name-box" class="ac-title" flex="1"
+                    onunderflow="_doUnderflow('_name');">
+            <xul:description anonid="name" class="ac-normal-text ac-comment"
+                             xbl:inherits="selected"/>
+          </xul:hbox>
+          <xul:label anonid="name-overflow-ellipsis" xbl:inherits="selected"
+                     class="ac-ellipsis-after ac-comment" hidden="true"/>
+        </xul:hbox>
+        <xul:hbox>
+          <xul:hbox anonid="identity-box" class="ac-url" flex="1"
+                    onunderflow="_doUnderflow('_identity');">
+            <xul:description anonid="identity" class="ac-normal-text ac-url-text"
+                             xbl:inherits="selected"/>
+          </xul:hbox>
+          <xul:label anonid="identity-overflow-ellipsis" xbl:inherits="selected"
+                     class="ac-ellipsis-after ac-url-text" hidden="true"/>
+          <xul:image anonid="type-image" class="ac-type-icon"/>
+        </xul:hbox>
+      </xul:vbox>
+    </content>
+    <implementation>
+      <constructor>
+        <![CDATA[
+            let ellipsis = "\u2026";
+            try {
+              ellipsis = Services.prefs.getComplexValue("intl.ellipsis",
+                  Components.interfaces.nsIPrefLocalizedString).data;
+            } catch (ex) {
+              // Do nothing.. we already have a default
+            }
+
+            this._identityOverflowEllipsis = document.getAnonymousElementByAttribute(this, "anonid", "identity-overflow-ellipsis");
+            this._nameOverflowEllipsis = document.getAnonymousElementByAttribute(this, "anonid", "name-overflow-ellipsis");
+
+            this._identityOverflowEllipsis.value = ellipsis;
+            this._nameOverflowEllipsis.value = ellipsis;
+
+            this._typeImage = document.getAnonymousElementByAttribute(this, "anonid", "type-image");
+
+            this._identityBox = document.getAnonymousElementByAttribute(this, "anonid", "identity-box");
+            this._identity = document.getAnonymousElementByAttribute(this, "anonid", "identity");
+
+            this._nameBox = document.getAnonymousElementByAttribute(this, "anonid", "name-box");
+            this._name = document.getAnonymousElementByAttribute(this, "anonid", "name");
+            
+            this._picture = document.getAnonymousElementByAttribute(this, "anonid", "picture");
+
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            var identity = this.obj;
+            return identity.accessibleLabel;
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+          var contact = this.obj;
+          
+          if (contact == null)
+            return;
+          
+          var identity = contact.identities[0];
+          
+          // I guess we should get the picture size from CSS or something?
+          this._picture.src = identity.pictureURL(32);
+          
+          // Emphasize the matching search terms for the description
+          this._setUpDescription(this._name, contact.name);
+          this._setUpDescription(this._identity, identity.value);
+
+          // Set up overflow on a timeout because the contents of the box
+          // might not have a width yet even though we just changed them
+          setTimeout(this._setUpOverflow, 0, this._nameBox, this._nameOverflowEllipsis);
+          setTimeout(this._setUpOverflow, 0, this._identityBox, this._identityOverflowEllipsis);
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+  <binding id="gloda-multi-item" extends="chrome://gloda/content/glodacomplete.xml#glodacomplete-base-richlistitem">
+    <content orient="vertical">
+      <xul:description anonid="explanation"/>
+      <xul:hbox anonid="identity-holder" flex="1">
+      </xul:hbox>
+    </content>
+    <implementation implements="nsIDOMXULSelectControlItemElement">
+      <constructor>
+        <![CDATA[
+            this._explanation = document.getAnonymousElementByAttribute(this, "anonid", "explanation");
+            this._identityHolder = document.getAnonymousElementByAttribute(this, "anonid", "identity-holder");
+
+            this._adjustAcItem();
+          ]]>
+      </constructor>
+      
+      <property name="label" readonly="true">
+        <getter>
+          <![CDATA[
+            return this._explanation.value;
+          ]]>
+        </getter>
+      </property>
+      
+      <method name="renderItem">
+        <parameter name="aObj"/>
+        <body>
+          var node = document.createElementNS(
+            "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul",
+            "richlistitem");
+          
+          node.obj = aObj;
+          node.setAttribute("type",
+                            "gloda-" + this.row.nounDef.name + "-chunk");
+          
+          this._identityHolder.appendChild(node);
+        </body>
+      </method>
+      
+      <method name="_adjustAcItem">
+        <body>
+          <![CDATA[
+          // clear out any lingering children
+          while (this._identityHolder.hasChildNodes())
+            this._identityHolder.lastChild.remove();
+          
+          var row = this.row;
+          if (row == null)
+            return;
+          
+          this._explanation.value = row.nounDef.name + "s " +
+            row.criteriaType + "ed " + row.criteria;
+          
+          // render anyone already in there
+          for (let item of row.collection.items) {
+            this.renderItem(item);
+          }
+          // listen up, yo.
+          row.renderer = this;
+          ]]>
+        </body>
+      </method>
+    </implementation>
+  </binding>
+
+
+</bindings>

mailnews/db/gloda/content/overlay.js

@@ -0,0 +1,6 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// get the core
+Components.utils.import("resource:///modules/gloda/public.js");

mailnews/db/gloda/content/thunderbirdOverlay.xul

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+  - License, v. 2.0. If a copy of the MPL was not distributed with this
+  - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+<overlay id="gloda-overlay"
+         xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul">
+  <script type="application/javascript" src="chrome://gloda/content/overlay.js"/>
+</overlay>

mailnews/db/gloda/jar.mn

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+gloda.jar:
+% content gloda   %content/
+% overlay chrome://messenger/content/messenger.xul    chrome://gloda/content/thunderbirdOverlay.xul application={3550f703-e582-4d05-9a08-453d09bdfdc6}
+  content/overlay.js               (content/overlay.js)
+  content/thunderbirdOverlay.xul   (content/thunderbirdOverlay.xul)
+  content/glodacomplete.css        (content/glodacomplete.css)
+  content/glodacomplete.xml        (content/glodacomplete.xml)

mailnews/db/gloda/modules/collection.js

@@ -0,0 +1,772 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaCollection', 'GlodaCollectionManager'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.collection");
+
+/**
+ * @namespace Central registry and logic for all collections.
+ *
+ * The collection manager is a singleton that has the following tasks:
+ * - Let views of objects (nouns) know when their objects have changed.  For
+ *   example, an attribute has changed due to user action.
+ * - Let views of objects based on queries know when new objects match their
+ *   query, or when their existing objects no longer match due to changes.
+ * - Caching/object-identity maintenance.  It is ideal if we only ever have
+ *   one instance of an object at a time.  (More specifically, only one instance
+ *   per database row 'id'.)  The collection mechanism lets us find existing
+ *   instances to this end.  Caching can be directly integrated by being treated
+ *   as a special collection.
+ */
+var GlodaCollectionManager = {
+  _collectionsByNoun: {},
+  _cachesByNoun: {},
+
+  /**
+   * Registers the existence of a collection with the collection manager.  This
+   *  is done using a weak reference so that the collection can go away if it
+   *  wants to.
+   */
+  registerCollection: function gloda_colm_registerCollection(aCollection) {
+    let collections;
+    let nounID = aCollection.query._nounDef.id;
+    if (!(nounID in this._collectionsByNoun))
+      collections = this._collectionsByNoun[nounID] = [];
+    else {
+      // purge dead weak references while we're at it
+      collections = this._collectionsByNoun[nounID].filter(function (aRef) {
+        return aRef.get(); });
+      this._collectionsByNoun[nounID] = collections;
+    }
+    collections.push(Cu.getWeakReference(aCollection));
+  },
+
+  getCollectionsForNounID: function gloda_colm_getCollectionsForNounID(aNounID){
+    if (!(aNounID in this._collectionsByNoun))
+      return [];
+
+    // generator would be nice, but I suspect get() is too expensive to use
+    //  twice (guard/predicate and value)
+    let weakCollections = this._collectionsByNoun[aNounID];
+    let collections = [];
+    for (let iColl = 0; iColl < weakCollections.length; iColl++) {
+      let collection = weakCollections[iColl].get();
+      if (collection)
+        collections.push(collection);
+    }
+    return collections;
+  },
+
+  defineCache: function gloda_colm_defineCache(aNounDef, aCacheSize) {
+    this._cachesByNoun[aNounDef.id] = new GlodaLRUCacheCollection(aNounDef,
+                                                                   aCacheSize);
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it was't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOne: function gloda_colm_cacheLookupOne(aNounID, aID, aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aID in cache._idMap) {
+        let item = cache._idMap[aID];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aID in collection._idMap) {
+        let item = collection._idMap[aID];
+        if (cache)
+          cache.add([item]);
+        return item;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Lookup multiple nouns by ID from the cache/existing collections.
+   *
+   * @param aNounID The kind of noun identified by its ID.
+   * @param aIDMap A dictionary/map whose keys must be gloda noun ids for the
+   *     given noun type and whose values are ignored.
+   * @param aTargetMap An object to hold the noun id's (key) and noun instances
+   *     (value) for the noun instances that were found available in memory
+   *     because they were cached or in existing query collections.
+   * @param [aDoCache=true] Should we add any items to the cache that we found
+   *     in collections that were in memory but not in the cache?  You would
+   *     likely want to pass false if you are only updating in-memory
+   *     representations rather than performing a new query.
+   *
+   * @return [The number that were found, the number that were not found,
+   *          a dictionary whose keys are the ids of noun instances that
+   *          were not found.]
+   */
+  cacheLookupMany: function gloda_colm_cacheLookupMany(aNounID, aIDMap,
+      aTargetMap, aDoCache) {
+    let foundCount = 0, notFoundCount = 0, notFound = {};
+
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      for (let key in aIDMap) {
+        let cacheValue = cache._idMap[key];
+        if (cacheValue === undefined) {
+          notFoundCount++;
+          notFound[key] = null;
+        }
+        else {
+          foundCount++;
+          aTargetMap[key] = cacheValue;
+          cache.hit(cacheValue);
+        }
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let key in notFound) {
+        let collValue = collection._idMap[key];
+        if (collValue !== undefined) {
+          aTargetMap[key] = collValue;
+          delete notFound[key];
+          foundCount++;
+          notFoundCount--;
+          if (cache)
+            cache.add([collValue]);
+        }
+      }
+    }
+
+    return [foundCount, notFoundCount, notFound];
+  },
+
+  /**
+   * Friendlier version of |cacheLookupMany|; takes a list of ids and returns
+   *  an object whose keys and values are the gloda id's and instances of the
+   *  instances that were found.  We don't tell you who we didn't find.  The
+   *  assumption is this is being used for in-memory updates where we only need
+   *  to tweak what is in memory.
+   */
+  cacheLookupManyList: function gloda_colm_cacheLookupManyList(aNounID, aIds) {
+    let checkMap = {}, targetMap = {};
+    for (let id of aIds) {
+      checkMap[id] = null;
+    }
+    // do not promote found items into the cache
+    this.cacheLookupMany(aNounID, checkMap, targetMap, false);
+    return targetMap;
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it was't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOneByUniqueValue:
+      function gloda_colm_cacheLookupOneByUniqueValue(aNounID, aUniqueValue,
+                                                      aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aUniqueValue in cache._uniqueValueMap) {
+        let item = cache._uniqueValueMap[aUniqueValue];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aUniqueValue in collection._uniqueValueMap) {
+        let item = collection._uniqueValueMap[aUniqueValue];
+        if (cache)
+          cache.add([item]);
+        return item;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Checks whether the provided item with the given id is actually a duplicate
+   *  of an instance that already exists in the cache/a collection.  If it is,
+   *  the pre-existing instance is returned and counts as a cache hit.  If it
+   *  is not, the passed-in instance is added to the cache and returned.
+   */
+  cacheLoadUnifyOne: function gloda_colm_cacheLoadUnifyOne(aItem) {
+    let items = [aItem];
+    this.cacheLoadUnify(aItem.NOUN_ID, items);
+    return items[0];
+  },
+
+  /**
+   * Given a list of items, check if any of them already have duplicate,
+   *  canonical, instances in the cache or collections.  Items with pre-existing
+   *  instances are replaced by those instances in the provided list, and each
+   *  counts as a cache hit.  Items without pre-existing instances are added
+   *  to the cache and left intact.
+   */
+  cacheLoadUnify: function gloda_colm_cacheLoadUnify(aNounID, aItems,
+      aCacheIfMissing) {
+    let cache = this._cachesByNoun[aNounID];
+    if (aCacheIfMissing === undefined)
+      aCacheIfMissing = true;
+
+    // track the items we haven't yet found in a cache/collection (value) and
+    //  their index in aItems (key).  We're somewhat abusing the dictionary
+    //  metaphor with the intent of storing tuples here.  We also do it because
+    //  it allows random-access deletion theoretically without cost.  (Since
+    //  we delete during iteration, that may be wrong, but it sounds like the
+    //  semantics still work?)
+    let unresolvedIndexToItem = {};
+    let numUnresolved = 0;
+
+    if (cache) {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        let item = aItems[iItem];
+
+        if (item.id in cache._idMap) {
+          let realItem = cache._idMap[item.id];
+          // update the caller's array with the reference to the 'real' item
+          aItems[iItem] = realItem;
+          cache.hit(realItem);
+        }
+        else {
+          unresolvedIndexToItem[iItem] = item;
+          numUnresolved++;
+        }
+      }
+
+      // we're done if everyone was a hit.
+      if (numUnresolved == 0)
+        return;
+    }
+    else {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        unresolvedIndexToItem[iItem] = aItems[iItem];
+      }
+      numUnresolved = aItems.length;
+    }
+
+    let needToCache = [];
+    // next, let's fall back to our collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let [iItem, item] in Iterator(unresolvedIndexToItem)) {
+        if (item.id in collection._idMap) {
+          let realItem = collection._idMap[item.id];
+          // update the caller's array to now have the 'real' object
+          aItems[iItem] = realItem;
+          // flag that we need to cache this guy (we use an inclusive cache)
+          needToCache.push(realItem);
+          // we no longer need to resolve this item...
+          delete unresolvedIndexToItem[iItem];
+          // stop checking collections if we got everybody
+          if (--numUnresolved == 0)
+            break;
+        }
+      }
+    }
+
+    // anything left in unresolvedIndexToItem should be added to the cache
+    //  unless !aCacheIfMissing.  plus, we already have 'needToCache'
+    if (cache && aCacheIfMissing) {
+      cache.add(needToCache.concat(Object.keys(unresolvedIndexToItem).
+                                   map(key => unresolvedIndexToItem[key])));
+    }
+
+    return aItems;
+  },
+
+  cacheCommitDirty: function glod_colm_cacheCommitDirty() {
+    for (let id in this._cachesByNoun) {
+      let cache = this._cachesByNoun[id];
+      cache.commitDirty();
+    }
+  },
+
+  /**
+   * Notifies the collection manager that an item has been loaded and should
+   *  be cached, assuming caching is active.
+   */
+  itemLoaded: function gloda_colm_itemsLoaded(aItem) {
+    let cache = this._cachesByNoun[aItem.NOUN_ID];
+    if (cache) {
+      cache.add([aItem]);
+    }
+  },
+
+  /**
+   * Notifies the collection manager that multiple items has been loaded and
+   *  should be cached, assuming caching is active.
+   */
+  itemsLoaded: function gloda_colm_itemsLoaded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+  },
+
+  /**
+   * This should be called when items are added to the global database.  This
+   *  should generally mean during indexing by indexers or an attribute
+   *  provider.
+   * We walk all existing collections for the given noun type and add the items
+   *  to the collection if the item meets the query that defines the collection.
+   */
+  itemsAdded: function gloda_colm_itemsAdded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let addItems = aItems.filter(item => collection.query.test(item));
+      if (addItems.length)
+        collection._onItemsAdded(addItems);
+    }
+  },
+  /**
+   * This should be called when items in the global database are modified.  For
+   *  example, as a result of indexing.  This should generally only be called
+   *  by indexers or by attribute providers.
+   * We walk all existing collections for the given noun type.  For items
+   *  currently included in each collection but should no longer be (per the
+   *  collection's defining query) we generate onItemsRemoved events.  For items
+   *  not currently included in the collection but should now be, we generate
+   *  onItemsAdded events.  For items included that still match the query, we
+   *  generate onItemsModified events.
+   */
+  itemsModified: function gloda_colm_itemsModified(aNounID, aItems) {
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let added = [], modified = [], removed = [];
+      for (let item of aItems) {
+        if (item.id in collection._idMap) {
+          // currently in... but should it still be there?
+          if (collection.query.test(item))
+            modified.push(item); // yes, keep it
+          // oy, so null queries really don't want any notifications, and they
+          //  sorta fit into our existing model, except for the removal bit.
+          //  so we need a specialized check for them, and we're using the
+          //  frozen attribute to this end.
+          else if (!collection.query.frozen)
+            removed.push(item); // no, bin it
+        }
+        else if (collection.query.test(item)) // not in, should it be?
+          added.push(item); // yep, add it
+      }
+      if (added.length)
+        collection._onItemsAdded(added);
+      if (modified.length)
+        collection._onItemsModified(modified);
+      if (removed.length)
+        collection._onItemsRemoved(removed);
+    }
+  },
+  /**
+   * This should be called when items in the global database are permanently-ish
+   *  deleted.  (This is distinct from concepts like message deletion which may
+   *  involved trash folders or other modified forms of existence.  Deleted
+   *  means the data is gone and if it were to come back, it would come back
+   *  via an itemsAdded event.)
+   * We walk all existing collections for the given noun type.  For items
+   *  currently in the collection, we generate onItemsRemoved events.
+   * 
+   * @param aItemIds A list of item ids that are being deleted.
+   */
+  itemsDeleted: function gloda_colm_itemsDeleted(aNounID, aItemIds) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let itemId of aItemIds) {
+        if (itemId in cache._idMap)
+          cache.deleted(cache._idMap[itemId]);
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = aItemIds.filter(itemId => itemId in collection._idMap).
+        map(itemId => collection._idMap[itemId]);
+      if (removeItems.length)
+        collection._onItemsRemoved(removeItems);
+    }
+  },
+  /**
+   * Like |itemsDeleted| but for the case where the deletion is based on an
+   *  attribute that SQLite can more efficiently check than we can and where the
+   *  cost of scanning the in-memory items is presumably much cheaper than
+   *  trying to figure out what actually got deleted.
+   * 
+   * Since we are doing an in-memory walk, this is obviously O(n) where n is the
+   *  number of noun instances of a given type in-memory.  We are assuming this
+   *  is a reasonable number of things and that this type of deletion call is
+   *  not going to happen all that frequently.  If these assumptions are wrong,
+   *  callers are advised to re-think the whole situation.
+   * 
+   * @param aNounID Type of noun we are talking about here.
+   * @param aFilter A filter function that returns true when the item should be
+   *     thought of as deleted, or false if the item is still good.  Screw this
+   *     up and you will get some seriously wacky bugs, yo.
+   */
+  itemsDeletedByAttribute: function gloda_colm_itemsDeletedByAttribute(
+      aNounID, aFilter) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let id in cache._idMap) {
+        let item = cache._idMap[id];
+        if (aFilter(item))
+          cache.deleted(item);
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = collection.items.filter(aFilter);
+      if (removeItems.length)
+        collection._onItemsRemoved(removeItems);
+    }
+  },
+};
+
+/**
+ * @class A current view of the set of first-class nouns meeting a given query.
+ *  Assuming a listener is present, events are
+ *  generated when new objects meet the query, existing objects no longer meet
+ *  the query, or existing objects have experienced a change in attributes that
+ *  does not affect their ability to be present (but the listener may care about
+ *  because it is exposing those attributes).
+ * @constructor
+ */
+function GlodaCollection(aNounDef, aItems, aQuery, aListener,
+      aMasterCollection) {
+  // if aNounDef is null, we are just being invoked for subclassing
+  if (aNounDef === undefined)
+    return;
+
+  this._nounDef = aNounDef;
+  // should we also maintain a unique value mapping...
+  if (this._nounDef.usesUniqueValue)
+    this._uniqueValueMap = {};
+
+  this.pendingItems = [];
+  this._pendingIdMap = {};
+  this.items = [];
+  this._idMap = {};
+
+  // force the listener to null for our call to _onItemsAdded; no events for
+  //  the initial load-out.
+  this._listener = null;
+  if (aItems && aItems.length)
+    this._onItemsAdded(aItems);
+
+  this.query = aQuery || null;
+  if (this.query) {
+    this.query.collection = this;
+    if (this.query.options.stashColumns)
+      this.stashedColumns = {};
+  }
+  this._listener = aListener || null;
+
+  this.deferredCount = 0;
+  this.resolvedCount = 0;
+
+  if (aMasterCollection) {
+    this.masterCollection = aMasterCollection.masterCollection;
+  }
+  else {
+    this.masterCollection = this;
+    /** a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of these sub-dictionaries maps the IDs of desired noun instances to
+     * the actual instance, or null if it has not yet been loaded.
+     */
+    this.referencesByNounID = {};
+    /**
+     * a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of the sub-dictionaries maps the IDs of the _recognized parent
+     * noun_ to the list of children, or null if the list has not yet been
+     * populated.
+     *
+     * So if we have a noun definition A with ID 1 who is the recognized parent
+     *  noun of noun definition B with ID 2, AND we have an instance A(1) with
+     *  two children B(10), B(11), then an example might be: {2: {1: [10, 11]}}.
+     */
+    this.inverseReferencesByNounID = {};
+    this.subCollections = {};
+  }
+}
+
+GlodaCollection.prototype = {
+  get listener() { return this._listener; },
+  set listener(aListener) { this._listener = aListener; },
+
+  /**
+   * If this collection still has a query associated with it, drop the query
+   *  and replace it with an 'explicit query'.  This means that the Collection
+   *  Manager will not attempt to match new items indexed to the system against
+   *  our query criteria.
+   * Once you call this method, your collection's listener will no longer
+   *  receive onItemsAdded notifications that are not the result of your
+   *  initial database query.  It will, however, receive onItemsModified
+   *  notifications if items in the collection are re-indexed.
+   */
+  becomeExplicit: function gloda_coll_becomeExplicit() {
+    if (!(this.query instanceof this._nounDef.explicitQueryClass)) {
+      this.query = new this._nounDef.explicitQueryClass(this);
+    }
+  },
+
+  /**
+   * Clear the contents of this collection.  This only makes sense for explicit
+   *  collections or wildcard collections.  (Actual query-based collections
+   *  should represent the state of the query, so unless we're going to delete
+   *  all the items, clearing the collection would violate that constraint.)
+   */
+  clear: function gloda_coll_clear() {
+    this._idMap = {};
+    if (this._uniqueValueMap)
+      this._uniqueValueMap = {};
+    this.items = [];
+  },
+
+  _onItemsAdded: function gloda_coll_onItemsAdded(aItems) {
+    this.items.push.apply(this.items, aItems);
+    if (this._uniqueValueMap) {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+        this._uniqueValueMap[item.uniqueValue] = item;
+      }
+    }
+    else {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+      }
+    }
+    if (this._listener) {
+      try {
+        this._listener.onItemsAdded(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsAdded: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  _onItemsModified: function gloda_coll_onItemsModified(aItems) {
+    if (this._listener) {
+      try {
+        this._listener.onItemsModified(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsModified: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  /**
+   * Given a list of items that definitely no longer belong in this collection,
+   *  remove them from the collection and notify the listener.  The 'tricky'
+   *  part is that we need to remove the deleted items from our list of items.
+   */
+  _onItemsRemoved: function gloda_coll_onItemsRemoved(aItems) {
+    // we want to avoid the O(n^2) deletion performance case, and deletion
+    //  should be rare enough that the extra cost of building the deletion map
+    //  should never be a real problem.
+    let deleteMap = {};
+    // build the delete map while also nuking from our id map/unique value map
+    for (let item of aItems) {
+      deleteMap[item.id] = true;
+      delete this._idMap[item.id];
+      if (this._uniqueValueMap)
+        delete this._uniqueValueMap[item.uniqueValue];
+    }
+    let items = this.items;
+    // in-place filter.  probably needless optimization.
+    let iWrite=0;
+    for (let iRead = 0; iRead < items.length; iRead++) {
+      let item = items[iRead];
+      if (!(item.id in deleteMap))
+        items[iWrite++] = item;
+    }
+    items.splice(iWrite);
+
+    if (this._listener) {
+      try {
+        this._listener.onItemsRemoved(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsRemoved: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  _onQueryCompleted: function gloda_coll_onQueryCompleted() {
+    this.query.completed = true;
+    if (this._listener && this._listener.onQueryCompleted)
+      this._listener.onQueryCompleted(this);
+  }
+};
+
+/**
+ * Create an LRU cache collection for the given noun with the given size.
+ * @constructor
+ */
+function GlodaLRUCacheCollection(aNounDef, aCacheSize) {
+  GlodaCollection.call(this, aNounDef, null, null, null);
+
+  this._head = null; // aka oldest!
+  this._tail = null; // aka newest!
+  this._size = 0;
+  // let's keep things sane, and simplify our logic a little...
+  if (aCacheSize < 32)
+    aCacheSize = 32;
+  this._maxCacheSize = aCacheSize;
+}
+/**
+ * @class A LRU-discard cache.  We use a doubly linked-list for the eviction
+ *  tracking.  Since we require that there is at most one LRU-discard cache per
+ *  noun class, we simplify our lives by adding our own attributes to the
+ *  cached objects.
+ * @augments GlodaCollection
+ */
+GlodaLRUCacheCollection.prototype = new GlodaCollection;
+GlodaLRUCacheCollection.prototype.add = function cache_add(aItems) {
+  for (let item of aItems) {
+    if (item.id in this._idMap) {
+      // DEBUGME so, we're dealing with this, but it shouldn't happen.  need
+      //  trace-debuggage.
+      continue;
+    }
+    this._idMap[item.id] = item;
+    if (this._uniqueValueMap)
+      this._uniqueValueMap[item.uniqueValue] = item;
+
+    item._lruPrev = this._tail;
+    // we do have to make sure that we will set _head the first time we insert
+    //  something
+    if (this._tail !== null)
+      this._tail._lruNext = item;
+    else
+      this._head = item;
+    item._lruNext = null;
+    this._tail = item;
+
+    this._size++;
+  }
+
+  while (this._size > this._maxCacheSize) {
+    let item = this._head;
+
+    // we never have to deal with the possibility of needing to make _head/_tail
+    //  null.
+    this._head = item._lruNext;
+    this._head._lruPrev = null;
+    // (because we are nice, we will delete the properties...)
+    delete item._lruNext;
+    delete item._lruPrev;
+
+    // nuke from our id map
+    delete this._idMap[item.id];
+    if (this._uniqueValueMap)
+      delete this._uniqueValueMap[item.uniqueValue];
+
+    // flush dirty items to disk (they may not have this attribute, in which
+    //  case, this returns false, which is fine.)
+    if (item.dirty) {
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+
+    this._size--;
+  }
+};
+
+GlodaLRUCacheCollection.prototype.hit = function cache_hit(aItem) {
+  // don't do anything in the 0 or 1 items case, or if we're already
+  //  the last item
+  if ((this._head === this._tail) || (this._tail === aItem))
+    return aItem;
+
+  // - unlink the item
+  if (aItem._lruPrev !== null)
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  else
+    this._head = aItem._lruNext;
+  // (_lruNext cannot be null)
+  aItem._lruNext._lruPrev = aItem._lruPrev;
+  // - link it in to the end
+  this._tail._lruNext = aItem;
+  aItem._lruPrev = this._tail;
+  aItem._lruNext = null;
+  // update tail tracking
+  this._tail = aItem;
+
+  return aItem;
+};
+
+GlodaLRUCacheCollection.prototype.deleted = function cache_deleted(aItem) {
+  // unlink the item
+  if (aItem._lruPrev !== null)
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  else
+    this._head = aItem._lruNext;
+  if (aItem._lruNext !== null)
+    aItem._lruNext._lruPrev = aItem._lruPrev;
+  else
+    this._tail = aItem._lruPrev;
+
+  // (because we are nice, we will delete the properties...)
+  delete aItem._lruNext;
+  delete aItem._lruPrev;
+
+  // nuke from our id map
+  delete this._idMap[aItem.id];
+  if (this._uniqueValueMap)
+    delete this._uniqueValueMap[aItem.uniqueValue];
+
+  this._size--;
+};
+
+/**
+ * If any of the cached items are dirty, commit them, and make them no longer
+ *  dirty.
+ */
+GlodaLRUCacheCollection.prototype.commitDirty = function cache_commitDirty() {
+  // we can only do this if there is an update method available...
+  if (!this._nounDef.objUpdate)
+    return;
+
+  for (let iItem in this._idMap) {
+    let item = this._idMap[iItem];
+    if (item.dirty) {
+      LOG.debug("flushing dirty: " + item);
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+  }
+};

mailnews/db/gloda/modules/connotent.js

@@ -0,0 +1,273 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaContent', 'whittlerRegistry',
+                          'mimeMsgToContentAndMeta', 'mimeMsgToContentSnippetAndMeta'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.connotent");
+
+
+
+/**
+ * Given a MimeMsg and the corresponding folder, return the GlodaContent object.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @return an array containing the GlodaContent instance, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentAndMeta(aMimeMsg, folder) {
+  let content = new GlodaContent();
+  let meta = {subject: aMimeMsg.get("subject")};
+  let bodyLines = aMimeMsg.coerceBodyToPlaintext(folder).split(/\r?\n/);
+
+  for (let whittler of whittlerRegistry.getWhittlers())
+    whittler.contentWhittle(meta, bodyLines, content);
+
+  return [content, meta];
+}
+
+
+/**
+ * Given a MimeMsg, return the whittled content string, suitable for summarizing
+ * a message.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @param length: optional number of characters to trim the whittled content.
+ * If the actual length of the message is greater than |length|, then the return
+ * value is the first (length-1) characters with an ellipsis appended.
+ * @return an array containing the text of the snippet, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentSnippetAndMeta(aMimeMsg, folder, length) {
+  let [content, meta] = mimeMsgToContentAndMeta(aMimeMsg, folder);
+
+  let text = content.getContentSnippet(length + 1);
+  if (length && text.length > length)
+    text = text.substring(0, length-1) + "\u2026"; // ellipsis
+
+  return [text, meta];
+}
+
+
+/**
+ * A registry of gloda providers that have contentWhittle() functions.
+ * used by mimeMsgToContentSnippet, but populated by the Gloda object as it's
+ * processing providers.
+ */
+function WhittlerRegistry() {
+  this._whittlers = [];
+}
+
+WhittlerRegistry.prototype = {
+  /**
+   * Add a provider as a content whittler.
+   */
+  registerWhittler: function whittler_registry_registerWhittler(provider) {
+    this._whittlers.push(provider);
+  },
+  /**
+   * get the list of content whittlers, sorted from the most specific to
+   * the most generic
+   */
+  getWhittlers: function whittler_registry_getWhittlers() {
+    // Use the concat() trick to avoid mutating the internal object and
+    // leaking an internal representation.
+    return this._whittlers.concat().reverse();
+  }
+}
+
+this.whittlerRegistry = new WhittlerRegistry();
+
+function GlodaContent() {
+  this._contentPriority = null;
+  this._producing = false;
+  this._hunks = [];
+}
+
+GlodaContent.prototype = {
+  kPriorityBase: 0,
+  kPriorityPerfect: 100,
+
+  kHunkMeta: 1,
+  kHunkQuoted: 2,
+  kHunkContent: 3,
+
+  _resetContent: function gloda_content__resetContent() {
+    this._keysAndValues = [];
+    this._keysAndDeltaValues = [];
+    this._hunks = [];
+    this._curHunk = null;
+  },
+
+  /* ===== Consumer API ===== */
+  hasContent: function gloda_content_hasContent() {
+    return (this._contentPriority != null);
+  },
+
+  /**
+   * Return content suitable for snippet display.  This means that no quoting
+   *  or meta-data should be returned.
+   *
+   * @param aMaxLength The maximum snippet length desired.
+   */
+  getContentSnippet: function gloda_content_getContentSnippet(aMaxLength) {
+    let content = this.getContentString();
+    if (aMaxLength)
+      content = content.substring(0, aMaxLength);
+    return content;
+  },
+
+  getContentString: function gloda_content_getContent(aIndexingPurposes) {
+    let data = "";
+    for (let hunk of this._hunks) {
+      if (hunk.hunkType == this.kHunkContent) {
+        if (data)
+          data += "\n" + hunk.data;
+        else
+          data = hunk.data;
+      }
+    }
+
+    if (aIndexingPurposes) {
+      // append the values for indexing.  we assume the keywords are cruft.
+      // this may be crazy, but things that aren't a science aren't an exact
+      // science.
+      for (let kv of this._keysAndValues) {
+        data += "\n" + kv[1];
+      }
+      for (let kon of this._keysAndValues) {
+        data += "\n" + kon[1] + "\n" + kon[2];
+      }
+    }
+
+    return data;
+  },
+
+  /* ===== Producer API ===== */
+  /**
+   * Called by a producer with the priority they believe their interpretation
+   *  of the content comes in at.
+   *
+   * @returns true if we believe the producer's interpretation will be
+   *     interesting and they should go ahead and generate events.  We return
+   *     false if we don't think they are interesting, in which case they should
+   *     probably not issue calls to us, although we don't care.  (We will
+   *     ignore their calls if we return false, this allows the simplification
+   *     of code that needs to run anyways.)
+   */
+  volunteerContent: function gloda_content_volunteerContent(aPriority) {
+    if (this._contentPriority === null || this._contentPriority < aPriority) {
+      this._contentPriority = aPriority;
+      this._resetContent();
+      this._producing = true;
+      return true;
+    }
+    this._producing = false;
+    return false;
+  },
+
+  keyValue: function gloda_content_keyValue(aKey, aValue) {
+    if (!this._producing)
+      return;
+
+    this._keysAndValues.push([aKey, aValue]);
+  },
+  keyValueDelta: function gloda_content_keyValueDelta (aKey, aOldValue,
+      aNewValue) {
+    if (!this._producing)
+      return;
+
+    this._keysAndDeltaValues.push([aKey, aOldValue, aNewValue]);
+  },
+
+  /**
+   * Meta lines are lines that have to do with the content but are not the
+   *  content and can generally be related to an attribute that has been derived
+   *  and stored on the item.
+   * For example, a bugzilla bug may note that an attachment was created; this
+   *  is not content and wouldn't be desired in a snippet, but is still
+   *  potentially interesting meta-data.
+   *
+   * @param aLineOrLines The line or list of lines that are meta-data.
+   * @param aAttr The attribute this meta-data is associated with.
+   * @param aIndex If the attribute is non-singular, indicate the specific
+   *     index of the item in the attribute's bound list that the meta-data
+   *     is associated with.
+   */
+  meta: function gloda_content_meta(aLineOrLines, aAttr, aIndex) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    this._curHunk = {hunkType: this.kHunkMeta, attr: aAttr, index: aIndex,
+                     data: data};
+    this._hunks.push(this._curHunk);
+  },
+  /**
+   * Quoted lines reference previous messages or what not.
+   *
+   * @param aLineOrLiens The line or list of lines that are quoted.
+   * @param aDepth The depth of the quoting.
+   * @param aOrigin The item that originated the original content, if known.
+   *     For example, perhaps a GlodaMessage?
+   * @param aTarget A reference to the location in the original content, if
+   *     known.  For example, the index of a line in a message or something?
+   */
+  quoted: function gloda_content_quoted(aLineOrLines, aDepth, aOrigin,
+      aTarget) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    if (!this._curHunk ||
+        this._curHunk.hunkType != this.kHunkQuoted ||
+        this._curHunk.depth != aDepth ||
+        this._curHunk.origin != aOrigin || this._curHunk.target != aTarget) {
+      this._curHunk = {hunkType: this.kHunkQuoted, data: data,
+                       depth: aDepth, origin: aOrigin, target: aTarget};
+      this._hunks.push(this._curHunk);
+    }
+    else
+      this._curHunk.data += "\n" + data;
+  },
+
+  content: function gloda_content_content(aLineOrLines) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    if (!this._curHunk || this._curHunk.hunkType != this.kHunkContent) {
+      this._curHunk = {hunkType: this.kHunkContent, data: data};
+      this._hunks.push(this._curHunk);
+    }
+    else
+      this._curHunk.data += "\n" + data;
+  },
+};

mailnews/db/gloda/modules/databind.js

@@ -0,0 +1,194 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaDatabind"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var DBC_LOG = Log4Moz.repository.getLogger("gloda.ds.dbc");
+
+function GlodaDatabind(aNounDef, aDatastore) {
+  this._nounDef = aNounDef;
+  this._tableName = aNounDef.tableName;
+  this._tableDef = aNounDef.schema;
+  this._datastore = aDatastore;
+  this._log = Log4Moz.repository.getLogger("gloda.databind." + this._tableName);
+
+  // process the column definitions and make sure they have an attribute mapping
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    // default to the other dude's thing.
+    if (coldef.length < 3)
+      coldef[2] = coldef[0];
+    if (coldef[0] == "id")
+      this._idAttr = coldef[2];
+    // colDef[3] is the index of us in our SQL bindings, storage-numbering
+    coldef[3] = iColDef;
+  }
+
+  // XXX This is obviously synchronous and not perfectly async.  Since we are
+  //  doing this, we don't actually need to move to ordinal binding below
+  //  since we could just as well compel creation of the name map and thereby
+  //  avoid ever acquiring the mutex after bootstrap.
+  // However, this specific check can be cleverly avoided with future work.
+  // Namely, at startup we can scan for extension-defined tables and get their
+  //  maximum id so that we don't need to do it here.  The table will either
+  //  be brand new and thus have a maximum id of 1 or we will already know it
+  //  because of that scan.
+  this._nextId = 1;
+  let stmt = this._datastore._createSyncStatement(
+    "SELECT MAX(id) FROM " + this._tableName, true);
+  if (stmt.executeStep()) {  // no chance of this SQLITE_BUSY on this call
+    this._nextId = stmt.getInt64(0) + 1;
+  }
+  stmt.finalize();
+
+  let insertColumns = [];
+  let insertValues = [];
+  let updateItems = [];
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    let column = coldef[0];
+    let placeholder = "?" + (iColDef + 1);
+    insertColumns.push(column);
+    insertValues.push(placeholder);
+    if (column != "id") {
+      updateItems.push(column + " = " + placeholder);
+    }
+  }
+
+  let insertSql = "INSERT INTO " + this._tableName + " (" +
+    insertColumns.join(", ") + ") VALUES (" + insertValues.join(", ") + ")";
+
+  // For the update, we want the 'id' to be a constraint and not a value
+  //  that gets set...
+  let updateSql = "UPDATE " + this._tableName + " SET " +
+    updateItems.join(", ") + " WHERE id = ?1";
+  this._insertStmt = aDatastore._createAsyncStatement(insertSql);
+  this._updateStmt = aDatastore._createAsyncStatement(updateSql);
+
+  if (this._tableDef.fulltextColumns) {
+    for (let [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      if (coldef.length < 3)
+        coldef[2] = coldef[0];
+      // colDef[3] is the index of us in our SQL bindings, storage-numbering
+      coldef[3] = iColDef + 1;
+    }
+
+    let insertColumns = [];
+    let insertValues = [];
+    let updateItems = [];
+    for (var [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      let column = coldef[0];
+      // +2 instead of +1 because docid is implied
+      let placeholder = "?" + (iColDef + 2);
+      insertColumns.push(column);
+      insertValues.push(placeholder);
+      if (column != "id") {
+        updateItems.push(column + " = " + placeholder);
+      }
+    }
+
+    let insertFulltextSql = "INSERT INTO " + this._tableName + "Text (docid," +
+      insertColumns.join(", ") + ") VALUES (?1," + insertValues.join(", ") +
+      ")";
+
+    // For the update, we want the 'id' to be a constraint and not a value
+    //  that gets set...
+    let updateFulltextSql = "UPDATE " + this._tableName + "Text SET " +
+      updateItems.join(", ") + " WHERE docid = ?1";
+
+    this._insertFulltextStmt =
+      aDatastore._createAsyncStatement(insertFulltextSql);
+    this._updateFulltextStmt =
+      aDatastore._createAsyncStatement(updateFulltextSql);
+  }
+}
+
+GlodaDatabind.prototype = {
+  /**
+   * Perform appropriate binding coercion based on the schema provided to us.
+   * Although we end up effectively coercing JS Date objects to numeric values,
+   *  we should not be provided with JS Date objects!  There is no way for us
+   *  to know to turn them back into JS Date objects on the way out.
+   *  Additionally, there is the small matter of storage's bias towards
+   *  PRTime representations which may not always be desirable.
+   */
+  bindByType: function(aStmt, aColDef, aValue) {
+    if (aValue == null)
+      aStmt.bindNullParameter(aColDef[3]);
+    else if (aColDef[1] == "STRING" || aColDef[1] == "TEXT")
+      aStmt.bindStringParameter(aColDef[3], aValue);
+    else
+      aStmt.bindInt64Parameter(aColDef[3], aValue);
+  },
+
+  objFromRow: function(aRow) {
+    let getVariant = this._datastore._getVariant;
+    let obj = new this._nounDef.class();
+    for (let [iCol, colDef] of this._tableDef.columns.entries()) {
+      obj[colDef[2]] = getVariant(aRow, iCol);
+    }
+    return obj;
+  },
+
+  objInsert: function(aThing) {
+    let bindByType = this.bindByType;
+    if (!aThing[this._idAttr])
+      aThing[this._idAttr] = this._nextId++;
+
+    let stmt = this._insertStmt;
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._insertFulltextStmt) {
+      stmt = this._insertFulltextStmt;
+      stmt.bindInt64Parameter(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  objUpdate: function(aThing) {
+    let bindByType = this.bindByType;
+    let stmt = this._updateStmt;
+    // note, we specially bound the location of 'id' for the insert, but since
+    //  we're using named bindings, there is nothing special about setting it
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._updateFulltextStmt) {
+      stmt = this._updateFulltextStmt;
+      // fulltextColumns doesn't include id/docid, need to explicitly set it
+      stmt.bindInt64Parameter(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  adjustAttributes: function() {
+    // just proxy the call over to the datastore... we have to do this for
+    //  'this' reasons.  we don't refactor things to avoid this because it does
+    //  make some sense to have all the methods exposed from a single object,
+    //  even if the implementation does live elsewhere.
+    return this._datastore.adjustAttributes.apply(this._datastore, arguments);
+  },
+
+  // also proxied...
+  queryFromQuery: function() {
+    return this._datastore.queryFromQuery.apply(this._datastore, arguments);
+  }
+};

mailnews/db/gloda/modules/datamodel.js

@@ -0,0 +1,907 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaAttributeDBDef", "GlodaAccount",
+                    "GlodaConversation", "GlodaFolder", "GlodaMessage",
+                    "GlodaContact", "GlodaIdentity", "GlodaAttachment"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/mailServices.js");
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+var LOG = Log4Moz.repository.getLogger("gloda.datamodel");
+
+Cu.import("resource:///modules/gloda/utils.js");
+
+// Make it lazy.
+var gMessenger;
+function getMessenger () {
+  if (!gMessenger)
+    gMessenger = Cc["@mozilla.org/messenger;1"].createInstance(Ci.nsIMessenger);
+  return gMessenger;
+}
+
+/**
+ * @class Represents a gloda attribute definition's DB form.  This class
+ *  stores the information in the database relating to this attribute
+ *  definition.  Access its attrDef attribute to get at the realy juicy data.
+ *  This main interesting thing this class does is serve as the keeper of the
+ *  mapping from parameters to attribute ids in the database if this is a
+ *  parameterized attribute.
+ */
+function GlodaAttributeDBDef(aDatastore, aID, aCompoundName, aAttrType,
+                           aPluginName, aAttrName) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._compoundName = aCompoundName;
+  this._attrType = aAttrType;
+  this._pluginName = aPluginName;
+  this._attrName = aAttrName;
+
+  this.attrDef = null;
+
+  /** Map parameter values to the underlying database id. */
+  this._parameterBindings = {};
+}
+
+GlodaAttributeDBDef.prototype = {
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get attributeName() { return this._attrName; },
+
+  get parameterBindings() { return this._parameterBindings; },
+
+  /**
+   * Bind a parameter value to the attribute definition, allowing use of the
+   *  attribute-parameter as an attribute.
+   *
+   * @return
+   */
+  bindParameter: function gloda_attr_bindParameter(aValue) {
+    // people probably shouldn't call us with null, but handle it
+    if (aValue == null) {
+      return this._id;
+    }
+    if (aValue in this._parameterBindings) {
+      return this._parameterBindings[aValue];
+    }
+    // no database entry exists if we are here, so we must create it...
+    let id = this._datastore._createAttributeDef(this._attrType,
+                 this._pluginName, this._attrName, aValue);
+    this._parameterBindings[aValue] = id;
+    this._datastore.reportBinding(id, this, aValue);
+    return id;
+  },
+
+  /**
+   * Given a list of values, return a list (regardless of plurality) of
+   *  database-ready [attribute id,  value] tuples.  This is intended to be used
+   *  to directly convert the value of a property on an object that corresponds
+   *  to a bound attribute.
+   *
+   * @param {Array} aInstanceValues An array of instance values regardless of
+   *     whether or not the attribute is singular.
+   */
+  convertValuesToDBAttributes:
+      function gloda_attr_convertValuesToDBAttributes(aInstanceValues) {
+    let nounDef = this.attrDef.objectNounDef;
+    let dbAttributes = [];
+    if (nounDef.usesParameter) {
+      for (let instanceValue of aInstanceValues) {
+        let [param, dbValue] = nounDef.toParamAndValue(instanceValue);
+        dbAttributes.push([this.bindParameter(param), dbValue]);
+      }
+    }
+    else {
+      // Not generating any attributes is ok. This basically means the noun is
+      // just an informative property on the Gloda Message and has no real
+      // indexing purposes.
+      if ("toParamAndValue" in nounDef) {
+        for (let instanceValue of aInstanceValues) {
+          dbAttributes.push([this._id,
+                             nounDef.toParamAndValue(instanceValue)[1]]);
+        }
+      }
+    }
+    return dbAttributes;
+  },
+
+  toString: function() {
+    return this._compoundName;
+  }
+};
+
+var GlodaHasAttributesMixIn = {
+  enumerateAttributes: function* gloda_attrix_enumerateAttributes() {
+    let nounDef = this.NOUN_DEF;
+    for (let key in this) {
+      let value = this[key];
+      let attrDef = nounDef.attribsByBoundName[key];
+      // we expect to not have attributes for underscore prefixed values (those
+      //  are managed by the instance's logic.  we also want to not explode
+      //  should someone crap other values in there, we get both birds with this
+      //  one stone.
+      if (attrDef === undefined)
+        continue;
+      if (attrDef.singular) {
+        // ignore attributes with null values
+        if (value != null)
+          yield [attrDef, [value]];
+      }
+      else {
+        // ignore attributes with no values
+        if (value.length)
+          yield [attrDef, value];
+      }
+    }
+  },
+
+  domContribute: function gloda_attrix_domContribute(aDomNode) {
+    let nounDef = this.NOUN_DEF;
+    for (let attrName in nounDef.domExposeAttribsByBoundName) {
+      let attr = nounDef.domExposeAttribsByBoundName[attrName];
+      if (this[attrName])
+        aDomNode.setAttribute(attr.domExpose, this[attrName]);
+    }
+  },
+};
+
+function MixIn(aConstructor, aMixIn) {
+  let proto = aConstructor.prototype;
+  for (let [name, func] in Iterator(aMixIn)) {
+    if (name.startsWith("get_"))
+      proto.__defineGetter__(name.substring(4), func);
+    else
+      proto[name] = func;
+  }
+}
+
+/**
+ * @class A gloda wrapper around nsIMsgIncomingServer.
+ */
+function GlodaAccount(aIncomingServer) {
+  this._incomingServer = aIncomingServer;
+}
+
+GlodaAccount.prototype = {
+  NOUN_ID: 106,
+  get id() { return this._incomingServer.key; },
+  get name() { return this._incomingServer.prettyName; },
+  get incomingServer() { return this._incomingServer; },
+  toString: function gloda_account_toString() {
+    return "Account: " + this.id;
+  },
+
+  toLocaleString: function gloda_account_toLocaleString() {
+    return this.name;
+  }
+};
+
+/**
+ * @class A gloda conversation (thread) exists so that messages can belong.
+ */
+function GlodaConversation(aDatastore, aID, aSubject, aOldestMessageDate,
+                           aNewestMessageDate) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._subject = aSubject;
+  this._oldestMessageDate = aOldestMessageDate;
+  this._newestMessageDate = aNewestMessageDate;
+}
+
+GlodaConversation.prototype = {
+  NOUN_ID: 101,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get subject() { return this._subject; },
+  get oldestMessageDate() { return this._oldestMessageDate; },
+  get newestMessageDate() { return this._newestMessageDate; },
+
+  getMessagesCollection: function gloda_conversation_getMessagesCollection(
+    aListener, aData) {
+    let query = new GlodaMessage.prototype.NOUN_DEF.queryClass();
+    query.conversation(this._id).orderBy("date");
+    return query.getCollection(aListener, aData);
+  },
+
+  toString: function gloda_conversation_toString() {
+    return "Conversation:" + this._id;
+  },
+
+  toLocaleString: function gloda_conversation_toLocaleString() {
+    return this._subject;
+  }
+};
+
+function GlodaFolder(aDatastore, aID, aURI, aDirtyStatus, aPrettyName,
+                     aIndexingPriority) {
+  // _datastore is now set by GlodaDatastore
+  this._id = aID;
+  this._uri = aURI;
+  this._dirtyStatus = aDirtyStatus;
+  this._prettyName = aPrettyName;
+  this._xpcomFolder = null;
+  this._account = null;
+  this._activeIndexing = false;
+  this._activeHeaderRetrievalLastStamp = 0;
+  this._indexingPriority = aIndexingPriority;
+  this._deleted = false;
+  this._compacting = false;
+}
+
+GlodaFolder.prototype = {
+  NOUN_ID: 100,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  /** The folder is believed to be up-to-date */
+  kFolderClean: 0,
+  /** The folder has some un-indexed or dirty messages */
+  kFolderDirty: 1,
+  /** The folder needs to be entirely re-indexed, regardless of the flags on
+   * the messages in the folder. This state will be downgraded to dirty */
+  kFolderFilthy: 2,
+
+  _kFolderDirtyStatusMask: 0x7,
+  /**
+   * The (local) folder has been compacted and all of its message keys are
+   *  potentially incorrect.  This is not a possible state for IMAP folders
+   *  because their message keys are based on UIDs rather than offsets into
+   *  the mbox file.
+   */
+  _kFolderCompactedFlag: 0x8,
+
+  /** The folder should never be indexed. */
+  kIndexingNeverPriority: -1,
+  /** The lowest priority assigned to a folder. */
+  kIndexingLowestPriority: 0,
+  /** The highest priority assigned to a folder. */
+  kIndexingHighestPriority: 100,
+
+  /** The indexing priority for a folder if no other priority is assigned. */
+  kIndexingDefaultPriority: 20,
+  /** Folders marked check new are slightly more important I guess. */
+  kIndexingCheckNewPriority: 30,
+  /** Favorite folders are more interesting to the user, presumably. */
+  kIndexingFavoritePriority: 40,
+  /** The indexing priority for inboxes. */
+  kIndexingInboxPriority: 50,
+  /** The indexing priority for sent mail folders. */
+  kIndexingSentMailPriority: 60,
+
+  get id() { return this._id; },
+  get uri() { return this._uri; },
+  get dirtyStatus() {
+    return this._dirtyStatus & this._kFolderDirtyStatusMask;
+  },
+  /**
+   * Mark a folder as dirty if it was clean.  Do nothing if it was already dirty
+   *  or filthy.  For use by GlodaMsgIndexer only.  And maybe rkent and his
+   *  marvelous extensions.
+   */
+  _ensureFolderDirty: function gloda_folder__markFolderDirty() {
+    if (this.dirtyStatus == this.kFolderClean) {
+      this._dirtyStatus = (this.kFolderDirty & this._kFolderDirtyStatusMask) |
+                          (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Definitely for use only by GlodaMsgIndexer to downgrade the dirty status of
+   *  a folder.
+   */
+  _downgradeDirtyStatus: function gloda_folder__downgradeDirtyStatus(
+                           aNewStatus) {
+    if (this.dirtyStatus != aNewStatus) {
+      this._dirtyStatus = (aNewStatus & this._kFolderDirtyStatusMask) |
+                          (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Indicate whether this folder is currently being compacted.  The
+   *  |GlodaMsgIndexer| keeps this in-memory-only value up-to-date.
+   */
+  get compacting() {
+    return this._compacting;
+  },
+  /**
+   * Set whether this folder is currently being compacted.  This is really only
+   *  for the |GlodaMsgIndexer| to set.
+   */
+  set compacting(aCompacting) {
+    this._compacting = aCompacting;
+  },
+  /**
+   * Indicate whether this folder was compacted and has not yet been
+   *  compaction processed.
+   */
+  get compacted() {
+    return Boolean(this._dirtyStatus & this._kFolderCompactedFlag);
+  },
+  /**
+   * For use only by GlodaMsgIndexer to set/clear the compaction state of this
+   *  folder.
+   */
+  _setCompactedState: function gloda_folder__clearCompactedState(aCompacted) {
+    if (this.compacted != aCompacted) {
+      if (aCompacted)
+        this._dirtyStatus |= this._kFolderCompactedFlag;
+      else
+        this._dirtyStatus &= ~this._kFolderCompactedFlag;
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+
+  get name() { return this._prettyName; },
+  toString: function gloda_folder_toString() {
+    return "Folder:" + this._id;
+  },
+
+  toLocaleString: function gloda_folder_toLocaleString() {
+    let xpcomFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+    if (!xpcomFolder)
+      return this._prettyName;
+    return xpcomFolder.prettiestName +
+      " (" + xpcomFolder.rootFolder.prettiestName + ")";
+  },
+
+  get indexingPriority() {
+    return this._indexingPriority;
+  },
+
+  /** We are going to index this folder. */
+  kActivityIndexing: 0,
+  /** Asking for the folder to perform header retrievals. */
+  kActivityHeaderRetrieval: 1,
+  /** We only want the folder for its metadata but are not going to open it. */
+  kActivityFolderOnlyNoData: 2,
+
+
+  /** Is this folder known to be actively used for indexing? */
+  _activeIndexing: false,
+  /** Get our indexing status. */
+  get indexing() {
+    return this._activeIndexing;
+  },
+  /**
+   * Set our indexing status.  Normally, this will be enabled through passing
+   *  an activity type of kActivityIndexing (which will set us), but we will
+   *  still need to be explicitly disabled by the indexing code.
+   * When disabling indexing, we will call forgetFolderIfUnused to take care of
+   *  shutting things down.
+   * We are not responsible for committing changes to the message database!
+   *  That is on you!
+   */
+  set indexing(aIndexing) {
+    this._activeIndexing = aIndexing;
+    if (!aIndexing)
+      this.forgetFolderIfUnused();
+  },
+  /** When was this folder last used for header retrieval purposes? */
+  _activeHeaderRetrievalLastStamp: 0,
+
+  /**
+   * Retrieve the nsIMsgFolder instance corresponding to this folder, providing
+   *  an explanation of why you are requesting it for tracking/cleanup purposes.
+   *
+   * @param aActivity One of the kActivity* constants.  If you pass
+   *     kActivityIndexing, we will set indexing for you, but you will need to
+   *     clear it when you are done.
+   * @return The nsIMsgFolder if available, null on failure.
+   */
+  getXPCOMFolder: function gloda_folder_getXPCOMFolder(aActivity) {
+    if (!this._xpcomFolder) {
+      let rdfService = Cc['@mozilla.org/rdf/rdf-service;1']
+                         .getService(Ci.nsIRDFService);
+      this._xpcomFolder = rdfService.GetResource(this.uri)
+                                    .QueryInterface(Ci.nsIMsgFolder);
+    }
+    switch (aActivity) {
+      case this.kActivityIndexing:
+        // mark us as indexing, but don't bother with live tracking.  we do
+        //  that independently and only for header retrieval.
+        this.indexing = true;
+        break;
+      case this.kActivityHeaderRetrieval:
+        if (this._activeHeaderRetrievalLastStamp === 0)
+          this._datastore.markFolderLive(this);
+        this._activeHeaderRetrievalLastStamp = Date.now();
+        break;
+      case this.kActivityFolderOnlyNoData:
+        // we don't have to do anything here.
+        break;
+    }
+
+    return this._xpcomFolder;
+  },
+
+  /**
+   * Retrieve a GlodaAccount instance corresponding to this folder.
+   *
+   * @return The GlodaAccount instance.
+   */
+  getAccount: function gloda_folder_getAccount() {
+    if (!this._account) {
+      let msgFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+      this._account = new GlodaAccount(msgFolder.server);
+    }
+    return this._account;
+  },
+
+  /**
+   * How many milliseconds must a folder have not had any header retrieval
+   *  activity before it's okay to lose the database reference?
+   */
+  ACCEPTABLY_OLD_THRESHOLD: 10000,
+
+  /**
+   * Cleans up our nsIMsgFolder reference if we have one and it's not "in use".
+   * In use, from our perspective, means that it is not being used for indexing
+   *  and some arbitrary interval of time has elapsed since it was last
+   *  retrieved for header retrieval reasons.  The time interval is because if
+   *  we have one GlodaMessage requesting a header, there's a high probability
+   *  that another message will request a header in the near future.
+   * Because setting indexing to false disables us, we are written in an
+   *  idempotent fashion.  (It is possible for disabling indexing's call to us
+   *  to cause us to return true but for the datastore's timer call to have not
+   *  yet triggered.)
+   *
+   * @returns true if we are cleaned up and can be considered 'dead', false if
+   *     we should still be considered alive and this method should be called
+   *     again in the future.
+   */
+  forgetFolderIfUnused: function gloda_folder_forgetFolderIfUnused() {
+    // we are not cleaning/cleaned up if we are indexing
+    if (this._activeIndexing)
+      return false;
+
+    // set a point in the past as the threshold.  the timestamp must be older
+    //  than this to be eligible for cleanup.
+    let acceptablyOld = Date.now() - this.ACCEPTABLY_OLD_THRESHOLD;
+    // we are not cleaning/cleaned up if we have retrieved a header more
+    //  recently than the acceptably old threshold.
+    if (this._activeHeaderRetrievalLastStamp > acceptablyOld)
+      return false;
+
+    if (this._xpcomFolder) {
+      // This is the key action we take; the nsIMsgFolder will continue to
+      //  exist, but we want it to forget about its database so that it can
+      //  be closed and its memory can be reclaimed.
+      this._xpcomFolder.msgDatabase = null;
+      this._xpcomFolder = null;
+      // since the last retrieval time tracks whether we have marked live or
+      //  not, this needs to be reset to 0 too.
+      this._activeHeaderRetrievalLastStamp = 0;
+    }
+
+    return true;
+  }
+};
+
+/**
+ * @class A message representation.
+ */
+function GlodaMessage(aDatastore, aID, aFolderID, aMessageKey,
+                      aConversationID, aConversation, aDate,
+                      aHeaderMessageID, aDeleted, aJsonText,
+                      aNotability,
+                      aSubject, aIndexedBodyText, aAttachmentNames) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._folderID = aFolderID;
+  this._messageKey = aMessageKey;
+  this._conversationID = aConversationID;
+  this._conversation = aConversation;
+  this._date = aDate;
+  this._headerMessageID = aHeaderMessageID;
+  this._jsonText = aJsonText;
+  this._notability = aNotability;
+  this._subject = aSubject;
+  this._indexedBodyText = aIndexedBodyText;
+  this._attachmentNames = aAttachmentNames;
+
+  // only set _deleted if we're deleted, otherwise the undefined does our
+  //  speaking for us.
+  if (aDeleted)
+    this._deleted = aDeleted;
+}
+
+GlodaMessage.prototype = {
+  NOUN_ID: 102,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get folderID() { return this._folderID; },
+  get messageKey() { return this._messageKey; },
+  get conversationID() { return this._conversationID; },
+  // conversation is special
+  get headerMessageID() { return this._headerMessageID; },
+  get notability() { return this._notability; },
+  set notability(aNotability) { this._notability = aNotability; },
+
+  get subject() { return this._subject; },
+  get indexedBodyText() { return this._indexedBodyText; },
+  get attachmentNames() { return this._attachmentNames; },
+
+  get date() { return this._date; },
+  set date(aNewDate) { this._date = aNewDate; },
+
+  get folder() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID != null)
+        return this._datastore._mapFolderID(this._folderID);
+    }
+    catch (ex) {
+    }
+    return null;
+  },
+  get folderURI() {
+    // XXX just like for folder, handle mapping failures and return null
+    try {
+      if (this._folderID != null)
+        return this._datastore._mapFolderID(this._folderID).uri;
+    }
+    catch (ex) {
+    }
+    return null;
+  },
+  get account() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID == null)
+        return null;
+      let folder = this._datastore._mapFolderID(this._folderID);
+      return folder.getAccount();
+    }
+    catch (ex) { }
+    return null;
+  },
+  get conversation() {
+    return this._conversation;
+  },
+
+  toString: function gloda_message_toString() {
+    // uh, this is a tough one...
+    return "Message:" + this._id;
+  },
+
+  _clone: function gloda_message_clone() {
+    return new GlodaMessage(/* datastore */ null, this._id, this._folderID,
+      this._messageKey, this._conversationID, this._conversation, this._date,
+      this._headerMessageID, "_deleted" in this ? this._deleted : undefined,
+      "_jsonText" in this ? this._jsonText : undefined, this._notability,
+      this._subject, this._indexedBodyText, this._attachmentNames);
+  },
+
+  /**
+   * Provide a means of propagating changed values on our clone back to
+   *  ourselves.  This is required because of an object identity trick gloda
+   *  does; when indexing an already existing object, all mutations happen on
+   *  a clone of the existing object so that
+   */
+  _declone: function gloda_message_declone(aOther) {
+    if ("_content" in aOther)
+      this._content = aOther._content;
+
+    // The _indexedAuthor/_indexedRecipients fields don't get updated on
+    //  fulltext update so we don't need to propagate.
+    this._indexedBodyText = aOther._indexedBodyText;
+    this._attachmentNames = aOther._attachmentNames;
+  },
+
+  /**
+   * Mark this message as a ghost.  Ghosts are characterized by having no folder
+   *  id and no message key.  They also are not deleted or they would be of
+   *  absolutely no use to us.
+   *
+   * These changes are suitable for persistence.
+   */
+  _ghost: function gloda_message_ghost() {
+    this._folderID = null;
+    this._messageKey = null;
+    if ("_deleted" in this)
+      delete this._deleted;
+  },
+
+  /**
+   * Are we a ghost (which implies not deleted)?  We are not a ghost if we have
+   *  a definite folder location (we may not know our message key in the case
+   *  of IMAP moves not fully completed) and are not deleted.
+   */
+  get _isGhost() {
+    return this._folderID == null && !this._isDeleted;
+  },
+
+  /**
+   * If we were dead, un-dead us.
+   */
+  _ensureNotDeleted: function gloda_message__ensureNotDeleted() {
+    if ("_deleted" in this)
+      delete this._deleted;
+  },
+
+  /**
+   * Are we deleted?  This is private because deleted gloda messages are not
+   *  visible to non-core-gloda code.
+   */
+  get _isDeleted() {
+    return ("_deleted" in this) && this._deleted;
+  },
+
+  /**
+   * Trash this message's in-memory representation because it should no longer
+   *  be reachable by any code.  The database record is gone, it's not coming
+   *  back.
+   */
+  _objectPurgedMakeYourselfUnpleasant: function gloda_message_nuke() {
+    this._id = null;
+    this._folderID = null;
+    this._messageKey = null;
+    this._conversationID = null;
+    this._conversation = null;
+    this.date = null;
+    this._headerMessageID = null;
+  },
+
+  /**
+   * Return the underlying nsIMsgDBHdr from the folder storage for this, or
+   *  null if the message does not exist for one reason or another.  We may log
+   *  to our logger in the failure cases.
+   *
+   * This method no longer caches the result, so if you need to hold onto it,
+   *  hold onto it.
+   *
+   * In the process of retrieving the underlying message header, we may have to
+   *  open the message header database associated with the folder.  This may
+   *  result in blocking while the load happens, so you may want to try and find
+   *  an alternate way to initiate the load before calling us.
+   * We provide hinting to the GlodaDatastore via the GlodaFolder so that it
+   *  knows when it's a good time for it to go and detach from the database.
+   *
+   * @returns The nsIMsgDBHdr associated with this message if available, null on
+   *     failure.
+   */
+  get folderMessage() {
+    if (this._folderID === null || this._messageKey === null)
+      return null;
+
+    // XXX like for folder and folderURI, return null if we can't map the folder
+    let glodaFolder;
+    try {
+      glodaFolder = this._datastore._mapFolderID(this._folderID);
+    }
+    catch (ex) {
+      return null;
+    }
+    let folder = glodaFolder.getXPCOMFolder(
+                   glodaFolder.kActivityHeaderRetrieval);
+    if (folder) {
+      let folderMessage;
+      try {
+        folderMessage = folder.GetMessageHeader(this._messageKey);
+      }
+      catch (ex) {
+        folderMessage = null;
+      }
+      if (folderMessage !== null) {
+        // verify the message-id header matches what we expect...
+        if (folderMessage.messageId != this._headerMessageID) {
+          LOG.info("Message with message key " + this._messageKey +
+                   " in folder '" + folder.URI + "' does not match expected " +
+                   "header! (" + this._headerMessageID + " expected, got " +
+                   folderMessage.messageId + ")");
+          folderMessage = null;
+        }
+      }
+      return folderMessage;
+    }
+
+    // this only gets logged if things have gone very wrong.  we used to throw
+    //  here, but it's unlikely our caller can do anything more meaningful than
+    //  treating this as a disappeared message.
+    LOG.info("Unable to locate folder message for: " + this._folderID + ":" +
+             this._messageKey);
+    return null;
+  },
+  get folderMessageURI() {
+    let folderMessage = this.folderMessage;
+    if (folderMessage)
+      return folderMessage.folder.getUriForMsg(folderMessage);
+    else
+      return null;
+  }
+};
+MixIn(GlodaMessage, GlodaHasAttributesMixIn);
+
+/**
+ * @class Contacts correspond to people (one per person), and may own multiple
+ *  identities (e-mail address, IM account, etc.)
+ */
+function GlodaContact(aDatastore, aID, aDirectoryUUID, aContactUUID, aName,
+                      aPopularity, aFrecency, aJsonText) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._directoryUUID = aDirectoryUUID;
+  this._contactUUID = aContactUUID;
+  this._name = aName;
+  this._popularity = aPopularity;
+  this._frecency = aFrecency;
+  if (aJsonText)
+    this._jsonText = aJsonText;
+
+  this._identities = null;
+}
+
+GlodaContact.prototype = {
+  NOUN_ID: 103,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  get id() { return this._id; },
+  get directoryUUID() { return this._directoryUUID; },
+  get contactUUID() { return this._contactUUID; },
+  get name() { return this._name; },
+  set name(aName) { this._name = aName; },
+
+  get popularity() { return this._popularity; },
+  set popularity(aPopularity) {
+    this._popularity = aPopularity;
+    this.dirty = true;
+  },
+
+  get frecency() { return this._frecency; },
+  set frecency(aFrecency) {
+    this._frecency = aFrecency;
+    this.dirty = true;
+  },
+
+  get identities() {
+    return this._identities;
+  },
+
+  toString: function gloda_contact_toString() {
+    return "Contact:" + this._id;
+  },
+
+  get accessibleLabel() {
+    return "Contact: " + this._name;
+  },
+
+  _clone: function gloda_contact_clone() {
+    return new GlodaContact(/* datastore */ null, this._id, this._directoryUUID,
+      this._contactUUID, this._name, this._popularity, this._frecency);
+  },
+};
+MixIn(GlodaContact, GlodaHasAttributesMixIn);
+
+
+/**
+ * @class A specific means of communication for a contact.
+ */
+function GlodaIdentity(aDatastore, aID, aContactID, aContact, aKind, aValue,
+                       aDescription, aIsRelay) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._contactID = aContactID;
+  this._contact = aContact;
+  this._kind = aKind;
+  this._value = aValue;
+  this._description = aDescription;
+  this._isRelay = aIsRelay;
+  /// Cached indication of whether there is an address book card for this
+  ///  identity.  We keep this up-to-date via address book listener
+  ///  notifications in |GlodaABIndexer|.
+  this._hasAddressBookCard = undefined;
+}
+
+GlodaIdentity.prototype = {
+  NOUN_ID: 104,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get contactID() { return this._contactID; },
+  get contact() { return this._contact; },
+  get kind() { return this._kind; },
+  get value() { return this._value; },
+  get description() { return this._description; },
+  get isRelay() { return this._isRelay; },
+
+  get uniqueValue() {
+    return this._kind + "@" + this._value;
+  },
+
+  toString: function gloda_identity_toString() {
+    return "Identity:" + this._kind + ":" + this._value;
+  },
+
+  toLocaleString: function gloda_identity_toLocaleString() {
+    if (this.contact.name == this.value)
+      return this.value;
+    return this.contact.name + " : " + this.value;
+  },
+
+  get abCard() {
+    // for our purposes, the address book only speaks email
+    if (this._kind != "email")
+      return false;
+    let card = GlodaUtils.getCardForEmail(this._value);
+    this._hasAddressBookCard = (card != null);
+    return card;
+  },
+
+  /**
+   * Indicates whether we have an address book card for this identity.  This
+   *  value is cached once looked-up and kept up-to-date by |GlodaABIndexer|
+   *  and its notifications.
+   */
+  get inAddressBook() {
+    if (this._hasAddressBookCard !== undefined)
+      return this._hasAddressBookCard;
+    return (this.abCard && true) || false;
+  },
+
+  pictureURL: function(aSize) {
+    if (this.inAddressBook) {
+      // XXX should get the photo if we have it.
+    }
+    return "";
+  }
+};
+
+
+/**
+ * An attachment, with as much information as we can gather on it
+ */
+function GlodaAttachment(aGlodaMessage, aName, aContentType, aSize, aPart, aExternalUrl, aIsExternal) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._glodaMessage = aGlodaMessage;
+  this._name = aName;
+  this._contentType = aContentType;
+  this._size = aSize;
+  this._part = aPart;
+  this._externalUrl = aExternalUrl;
+  this._isExternal = aIsExternal;
+}
+
+GlodaAttachment.prototype = {
+  NOUN_ID: 105,
+  // set by GlodaDatastore
+  get name() { return this._name; },
+  get contentType() { return this._contentType; },
+  get size() { return this._size; },
+  get url() {
+    if (this.isExternal)
+      return this._externalUrl;
+    else {
+      let uri = this._glodaMessage.folderMessageURI;
+      if (!uri)
+        throw new Error("The message doesn't exist anymore, unable to rebuild attachment URL");
+      let neckoURL = {};
+      let msgService = getMessenger().messageServiceFromURI(uri);
+      msgService.GetUrlForUri(uri, neckoURL, null);
+      let url = neckoURL.value.spec;
+      let hasParamAlready = url.match(/\?[a-z]+=[^\/]+$/);
+      let sep = hasParamAlready ? "&" : "?";
+      return url+sep+"part="+this._part+"&filename="+encodeURIComponent(this._name);
+    }
+  },
+  get isExternal() { return this._isExternal; },
+
+  toString: function gloda_attachment_toString() {
+    return "attachment: " + this._name + ":" + this._contentType;
+  },
+
+};

mailnews/db/gloda/modules/dbview.js

@@ -0,0 +1,178 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file is charged with providing you a way to have a pretty gloda-backed
+ *  nsIMsgDBView.
+ */
+
+this.EXPORTED_SYMBOLS = ["GlodaSyntheticView"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+Cu.import("resource:///modules/gloda/public.js");
+Cu.import("resource:///modules/gloda/msg_search.js");
+
+/**
+ * Create a synthetic view suitable for passing to |FolderDisplayWidget.show|.
+ * You must pass a query, collection, or conversation in.
+ *
+ * @param {GlodaQuery} [aArgs.query] A gloda query to run.
+ * @param {GlodaCollection} [aArgs.collection] An already-populated collection
+ *     to display.  Do not call getCollection on a query and hand us that.  We
+ *     will not register ourselves as a listener and things will not work.
+ * @param {GlodaConversation} [aArgs.conversation] A conversation whose messages
+ *     you want to display.
+ */
+function GlodaSyntheticView(aArgs) {
+  if ("query" in aArgs) {
+    this.query = aArgs.query;
+    this.collection = this.query.getCollection(this);
+    this.completed = false;
+    this.viewType = "global";
+  }
+  else if ("collection" in aArgs) {
+    this.query = null;
+    this.collection = aArgs.collection;
+    this.completed = true;
+    this.viewType = "global";
+  }
+  else if ("conversation" in aArgs) {
+    this.collection = aArgs.conversation.getMessagesCollection(this);
+    this.query = this.collection.query;
+    this.completed = false;
+    this.viewType = "conversation";
+  }
+  else {
+    throw new Error("You need to pass a query or collection");
+  }
+
+  this.customColumns = [];
+}
+GlodaSyntheticView.prototype = {
+  defaultSort: [[Ci.nsMsgViewSortType.byDate, Ci.nsMsgViewSortOrder.descending]],
+
+  /**
+   * Request the search be performed and notification provided to
+   *  aSearchListener.  If results are already available, they should
+   *  be provided to aSearchListener without re-performing the search.
+   */
+  search: function(aSearchListener, aCompletionCallback) {
+    this.searchListener = aSearchListener;
+    this.completionCallback = aCompletionCallback;
+
+    this.searchListener.onNewSearch();
+    if (this.completed) {
+      this.reportResults(this.collection.items);
+      // we're not really aborting, but it closes things out nicely
+      this.abortSearch();
+      return;
+    }
+  },
+
+  abortSearch: function() {
+    if (this.searchListener)
+      this.searchListener.onSearchDone(Cr.NS_OK);
+    if (this.completionCallback)
+      this.completionCallback();
+    this.searchListener = null;
+    this.completionCallback = null;
+  },
+
+  reportResults: function(aItems) {
+    for (let item of aItems) {
+      let hdr = item.folderMessage;
+      if (hdr)
+        this.searchListener.onSearchHit(hdr, hdr.folder);
+    }
+  },
+
+  /**
+   * Helper function used by |DBViewWrapper.getMsgHdrForMessageID| since there
+   *  are no actual backing folders for it to check.
+   */
+  getMsgHdrForMessageID: function(aMessageId) {
+    for (let item of this.collection.items) {
+      if (item.headerMessageID == aMessageId) {
+        let hdr = item.folderMessage;
+        if (hdr)
+          return hdr;
+      }
+    }
+    return null;
+  },
+
+  /**
+   * The default set of columns to show.
+   */
+  DEFAULT_COLUMN_STATES: {
+    threadCol: {
+      visible: true,
+    },
+    flaggedCol: {
+      visible: true,
+    },
+    subjectCol: {
+      visible: true,
+    },
+    correspondentCol: {
+      visible: Services.prefs.getBoolPref("mail.threadpane.use_correspondents"),
+    },
+    senderCol: {
+      visible: !Services.prefs.getBoolPref("mail.threadpane.use_correspondents"),
+    },
+    dateCol: {
+      visible: true,
+    },
+    locationCol: {
+      visible: true,
+    },
+  },
+
+  // --- settings persistence
+  getPersistedSetting: function(aSetting) {
+    try {
+      return JSON.parse(Services.prefs.getCharPref(
+        "mailnews.database.global.views." + this.viewType + "." + aSetting
+      ));
+    }
+    catch (e) {
+      return this.getDefaultSetting(aSetting);
+    }
+  },
+  setPersistedSetting: function(aSetting, aValue) {
+    Services.prefs.setCharPref(
+      "mailnews.database.global.views." + this.viewType + "." + aSetting,
+      JSON.stringify(aValue)
+    );
+  },
+  getDefaultSetting: function(aSetting) {
+    if (aSetting == "columns")
+      return this.DEFAULT_COLUMN_STATES;
+    else
+      return undefined;
+  },
+
+  // --- collection listener
+  onItemsAdded: function(aItems, aCollection) {
+    if (this.searchListener)
+      this.reportResults(aItems);
+  },
+  onItemsModified: function(aItems, aCollection) {
+  },
+  onItemsRemoved: function(aItems, aCollection) {
+  },
+  onQueryCompleted: function(aCollection) {
+    this.completed = true;
+    this.searchListener.onSearchDone(Cr.NS_OK);
+    if (this.completionCallback)
+      this.completionCallback();
+  },
+};

mailnews/db/gloda/modules/everybody.js

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+var LOG = Log4Moz.repository.getLogger("gloda.everybody");
+
+var importNS = {};
+
+function loadModule(aModuleURI, aNSContrib) {
+  try {
+    LOG.info("... loading " + aModuleURI);
+    Cu.import(aModuleURI, importNS);
+  }
+  catch (ex) {
+    LOG.error("!!! error loading " + aModuleURI);
+    LOG.error("(" + ex.fileName + ":" + ex.lineNumber + ") " + ex);
+    return false;
+  }
+  LOG.info("+++ loaded " + aModuleURI);
+
+  if (aNSContrib) {
+    try {
+      importNS[aNSContrib].init();
+    }
+    catch (ex) {
+      LOG.error("!!! error initializing " + aModuleURI);
+      LOG.error("(" + ex.fileName + ":" + ex.lineNumber + ") " + ex);
+      return false;
+    }
+    LOG.info("+++ inited " + aModuleURI);
+  }
+  return true;
+}
+
+loadModule("resource:///modules/gloda/fundattr.js", "GlodaFundAttr");
+loadModule("resource:///modules/gloda/explattr.js", "GlodaExplicitAttr");
+
+loadModule("resource:///modules/gloda/noun_tag.js");
+loadModule("resource:///modules/gloda/noun_freetag.js");
+loadModule("resource:///modules/gloda/noun_mimetype.js");
+loadModule("resource:///modules/gloda/index_msg.js");
+loadModule("resource:///modules/gloda/index_ab.js", "GlodaABAttrs");

mailnews/db/gloda/modules/explattr.js

@@ -0,0 +1,191 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides the "explicit attribute" provider for messages.  It is
+ *  concerned with attributes that are the result of user actions.  For example,
+ *  whether a message is starred (flagged), message tags, whether it is
+ *  read/unread, etc.
+ */
+
+this.EXPORTED_SYMBOLS = ['GlodaExplicitAttr'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/noun_tag.js");
+Cu.import("resource:///modules/mailServices.js");
+
+
+var nsMsgMessageFlags_Replied = Ci.nsMsgMessageFlags.Replied;
+var nsMsgMessageFlags_Forwarded = Ci.nsMsgMessageFlags.Forwarded;
+
+var EXT_BUILTIN = "built-in";
+
+/**
+ * @namespace Explicit attribute provider.  Indexes/defines attributes that are
+ *  explicitly a result of user action.  This dubiously includes marking a
+ *  message as read.
+ */
+var GlodaExplicitAttr = {
+  providerName: "gloda.explattr",
+  strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+  _log: null,
+  _msgTagService: null,
+
+  init: function gloda_explattr_init() {
+    this._log =  Log4Moz.repository.getLogger("gloda.explattr");
+
+    this._msgTagService = MailServices.tags;
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  /** Boost for starred messages. */
+  NOTABILITY_STARRED: 16,
+  /** Boost for tagged messages, first tag. */
+  NOTABILITY_TAGGED_FIRST: 8,
+  /** Boost for tagged messages, each additional tag. */
+  NOTABILITY_TAGGED_ADDL: 1,
+
+  defineAttributes: function() {
+    // Tag
+    this._attrTag = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "tag",
+                        bindName: "tags",
+                        singular: false,
+                        emptySetIsSignificant: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_TAG,
+                        parameterNoun: null,
+                        // Property change notifications that we care about:
+                        propertyChanges: ["keywords"],
+                        }); // not-tested
+
+    // Star
+    this._attrStar = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "star",
+                        bindName: "starred",
+                        singular: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_BOOLEAN,
+                        parameterNoun: null,
+                        }); // tested-by: test_attributes_explicit
+    // Read/Unread
+    this._attrRead = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "read",
+                        singular: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_BOOLEAN,
+                        parameterNoun: null,
+                        }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this message been replied to by the user.
+     */
+    this._attrRepliedTo = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "repliedTo",
+      singular: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this user forwarded this message to someone.
+     */
+    this._attrForwarded = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "forwarded",
+      singular: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+  },
+
+  process: function* Gloda_explattr_process(aGlodaMessage, aRawReps, aIsNew,
+                                            aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+
+    aGlodaMessage.starred = aMsgHdr.isFlagged;
+    if (aGlodaMessage.starred)
+      aGlodaMessage.notability += this.NOTABILITY_STARRED;
+
+    aGlodaMessage.read = aMsgHdr.isRead;
+
+    let flags = aMsgHdr.flags;
+    aGlodaMessage.repliedTo = Boolean(flags & nsMsgMessageFlags_Replied);
+    aGlodaMessage.forwarded = Boolean(flags & nsMsgMessageFlags_Forwarded);
+
+    let tags = aGlodaMessage.tags = [];
+
+    // -- Tag
+    // build a map of the keywords
+    let keywords = aMsgHdr.getStringProperty("keywords");
+    let keywordList = keywords.split(' ');
+    let keywordMap = {};
+    for (let iKeyword = 0; iKeyword < keywordList.length; iKeyword++) {
+      let keyword = keywordList[iKeyword];
+      keywordMap[keyword] = true;
+    }
+
+    let tagArray = TagNoun.getAllTags();
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      if (tag.key in keywordMap)
+        tags.push(tag);
+    }
+
+    if (tags.length)
+      aGlodaMessage.notability += this.NOTABILITY_TAGGED_FIRST +
+        (tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+
+    yield Gloda.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from process().  Arguably process should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score: function Gloda_explattr_score(aMessage, aContext) {
+    let score = 0;
+    if (aMessage.starred)
+      score += this.NOTABILITY_STARRED;
+    if (aMessage.tags.length)
+      score += this.NOTABILITY_TAGGED_FIRST +
+        (aMessage.tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+    return score;
+  },
+};

mailnews/db/gloda/modules/facet.js

@@ -0,0 +1,582 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides faceting logic.
+ */
+
+var EXPORTED_SYMBOLS = ["FacetDriver", "FacetUtils"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/public.js");
+
+/**
+ * Decides the appropriate faceters for the noun type and drives the faceting
+ *  process.  This class and the faceters are intended to be reusable so that
+ *  you only need one instance per faceting session.  (Although each faceting
+ *  pass is accordingly destructive to previous results.)
+ *
+ * Our strategy for faceting is to process one attribute at a time across all
+ *  the items in the provided set.  The alternative would be to iterate over
+ *  the items and then iterate over the attributes on each item.  While both
+ *  approaches have caching downsides
+ */
+function FacetDriver(aNounDef, aWindow) {
+  this.nounDef = aNounDef;
+  this._window = aWindow;
+
+  this._makeFaceters();
+}
+FacetDriver.prototype = {
+  /**
+   * Populate |this.faceters| with a set of faceters appropriate to the noun
+   *  definition associated with this instance.
+   */
+  _makeFaceters: function() {
+    let faceters = this.faceters = [];
+
+    function makeFaceter(aAttrDef, aFacetDef) {
+      let facetType = aFacetDef.type;
+
+      if (aAttrDef.singular) {
+        if (facetType == "date")
+          faceters.push(new DateFaceter(aAttrDef, aFacetDef));
+        else
+          faceters.push(new DiscreteFaceter(aAttrDef, aFacetDef));
+      }
+      else {
+        if (facetType == "nonempty?")
+          faceters.push(new NonEmptySetFaceter(aAttrDef, aFacetDef));
+        else
+          faceters.push(new DiscreteSetFaceter(aAttrDef, aFacetDef));
+      }
+    }
+
+    for (let key in this.nounDef.attribsByBoundName) {
+      let attrDef = this.nounDef.attribsByBoundName[key];
+      // ignore attributes that do not want to be faceted
+      if (!attrDef.facet)
+        continue;
+
+      makeFaceter(attrDef, attrDef.facet);
+
+      if ("extraFacets" in attrDef) {
+        for (let facetDef of attrDef.extraFacets) {
+          makeFaceter(attrDef, facetDef);
+        }
+      }
+    }
+  },
+  /**
+   * Asynchronously facet the provided items, calling the provided callback when
+   *  completed.
+   */
+  go: function FacetDriver_go(aItems, aCallback, aCallbackThis) {
+    this.items = aItems;
+    this.callback = aCallback;
+    this.callbackThis = aCallbackThis;
+
+    this._nextFaceter = 0;
+    this._drive();
+  },
+
+  _MAX_FACETING_TIMESLICE_MS: 100,
+  _FACETING_YIELD_DURATION_MS: 0,
+  _driveWrapper: function(aThis) {
+    aThis._drive();
+  },
+  _drive: function() {
+    let start = Date.now();
+
+    while (this._nextFaceter < this.faceters.length) {
+      let faceter = this.faceters[this._nextFaceter++];
+      // for now we facet in one go, but the long-term plan allows for them to
+      //  be generators.
+      faceter.facetItems(this.items);
+
+      let delta = Date.now() - start;
+      if (delta > this._MAX_FACETING_TIMESLICE_MS) {
+        this._window.setTimeout(this._driveWrapper,
+                                this._FACETING_YIELD_DURATION_MS,
+                                this);
+        return;
+      }
+    }
+
+    // we only get here once we are done with the faceters
+    this.callback.call(this.callbackThis);
+  }
+};
+
+var FacetUtils = {
+  _groupSizeComparator: function(a, b) {
+    return b[1].length - a[1].length;
+  },
+
+  /**
+   * Given a list where each entry is a tuple of [group object, list of items
+   *  belonging to that group], produce a new list of the top grouped items.  We
+   *  used to also produce an "other" aggregation, but that turned out to be
+   *  conceptually difficult to deal with, so that's gone, leaving this method
+   *  with much less to do.
+   *
+   * @param aAttrDef The attribute for the facet we are working with.
+   * @param aGroups The list of groups built for the facet.
+   * @param aMaxCount The number of result rows you want back.
+   */
+  makeTopGroups: function FacetUtils_makeTopGroups(aAttrDef, aGroups,
+                                                   aMaxCount) {
+    let nounDef = aAttrDef.objectNounDef;
+    let realGroupsToUse = aMaxCount;
+
+    let orderedBySize = aGroups.concat();
+    orderedBySize.sort(this._groupSizeComparator);
+
+    // - get the real groups to use and order them by the attribute comparator
+    let outGroups = orderedBySize.slice(0, realGroupsToUse);
+    let comparator = nounDef.comparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    outGroups.sort(comparatorHelper);
+
+    return outGroups;
+  }
+};
+
+/**
+ * Facet discrete things like message authors, boolean values, etc.  Only
+ *  appropriate for use on singular values.  Use |DiscreteSetFaceter| for
+ *  non-singular values.
+ */
+function DiscreteFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive)
+      return this.facetPrimitiveItems(aItems);
+    else
+      return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+
+    let valStrToVal = {};
+    let groups = this.groups = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      if (val === Gloda.IGNORE_FACET)
+        continue;
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val))
+        continue;
+
+      // We need to use hasOwnProperty because we cannot guarantee that the
+      //  contents of val won't collide with the attributes in Object.prototype.
+      if (groups.hasOwnProperty(val))
+        groups[val].push(item);
+      else {
+        groups[val] = [item];
+        valStrToVal[val] = val;
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [valStrToVal[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = this.groups = {};
+    let groupMap = this.groupMap = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      if (val === Gloda.IGNORE_FACET)
+        continue;
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val))
+        continue;
+
+      let valId = (val == null) ? null : val[idAttr];
+      // We need to use hasOwnProperty because tag nouns are complex objects
+      //  with id's that are non-numeric and so can collide with the contents
+      //  of Object.prototype.  (Note: the "tags" attribute is actually handled
+      //  by the DiscreteSetFaceter.)
+      if (groupMap.hasOwnProperty(valId)) {
+        groups[valId].push(item);
+      }
+      else {
+        groupMap[valId] = val;
+        groups[valId] = [item];
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [groupMap[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Facet sets of discrete items.  For example, tags applied to messages.
+ *
+ * The main differences between us and |DiscreteFaceter| are:
+ * - The empty set is notable.
+ * - Specific set configurations could be interesting, but are not low-hanging
+ *    fruit.
+ */
+function DiscreteSetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteSetFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive)
+      return this.facetPrimitiveItems(aItems);
+    else
+      return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+
+    let groups = this.groups = {};
+    let valStrToVal = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals === Gloda.IGNORE_FACET)
+        continue;
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val))
+          continue;
+
+        // We need to use hasOwnProperty because we cannot guarantee that the
+        //  contents of val won't collide with the attributes in
+        //  Object.prototype.
+        if (groups.hasOwnProperty(val))
+          groups[val].push(item);
+        else {
+          groups[val] = [item];
+          valStrToVal[val] = val;
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [valStrToVal[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = this.groups = {};
+    let groupMap = this.groupMap = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals === Gloda.IGNORE_FACET)
+        continue;
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val))
+          continue;
+
+        let valId = (val == null) ? null : val[idAttr];
+        // We need to use hasOwnProperty because tag nouns are complex objects
+        //  with id's that are non-numeric and so can collide with the contents
+        //  of Object.prototype.
+        if (groupMap.hasOwnProperty(valId)) {
+          groups[valId].push(item);
+        }
+        else {
+          groupMap[valId] = val;
+          groups[valId] = [item];
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [groupMap[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Given a non-singular attribute, facet it as if it were a boolean based on
+ *  whether there is anything in the list (set).
+ */
+function NonEmptySetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+NonEmptySetFaceter.prototype = {
+  type: "boolean",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+
+    let trueValues = [];
+    let falseValues = [];
+
+    let groups = this.groups = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals == null || vals.length == 0)
+        falseValues.push(item);
+      else
+        trueValues.push(item);
+    }
+
+    this.orderedGroups = [];
+    if (trueValues.length)
+      this.orderedGroups.push([true, trueValues]);
+    if (falseValues.length)
+      this.orderedGroups.push([false, falseValues]);
+    this.groupCount = this.orderedGroups.length;
+  },
+  makeQuery: function(aGroupValues, aInclusive) {
+    let query = this.query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+
+    let constraintFunc = query[this.attrDef.boundName];
+    constraintFunc.call(query);
+
+    // Our query is always for non-empty lists (at this time), so we want to
+    //  invert if they're excluding 'true' or including 'false', which means !=.
+    let invert = aGroupValues[0] != aInclusive;
+
+    return [query, invert];
+  }
+};
+
+
+/**
+ * Facet dates.  We build a hierarchical nested structure of year, month, and
+ *  day nesting levels.  This decision was made speculatively in the hopes that
+ *  it would allow us to do clustered analysis and that there might be a benefit
+ *  for that.  For example, if you search for "Christmas", we might notice
+ *  clusters of messages around December of each year.  We could then present
+ *  these in a list as likely candidates, rather than a graphical timeline.
+ *  Alternately, it could be used to inform a non-linear visualization.  As it
+ *  stands (as of this writing), it's just a complicating factor.
+ */
+function DateFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DateFaceter.prototype = {
+  type: "date",
+  /**
+   *
+   */
+  facetItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+
+    let years = this.years = {_subCount: 0};
+    // generally track the time range
+    let oldest = null, newest = null;
+
+    let validItems = this.validItems = [];
+
+    // just cheat and put us at the front...
+    this.groupCount = aItems.length ? 1000 : 0;
+    this.orderedGroups = null;
+
+    /** The number of items with a null/missing attribute. */
+    this.missing = 0;
+
+    /**
+     * The number of items with a date that is unreasonably far in the past or
+     *  in the future.  Old-wise, we are concerned about incorrectly formatted
+     *  messages (spam) that end up placed around the UNIX epoch.  New-wise,
+     *  we are concerned about messages that can't be explained by users who
+     *  don't know how to set their clocks (both the current user and people
+     *  sending them mail), mainly meaning spam.
+     * We want to avoid having our clever time-scale logic being made useless by
+     *  these unreasonable messages.
+     */
+    this.unreasonable = 0;
+    // feb 1, 1970
+    let tooOld = new Date(1970, 1, 1);
+    // 3 days from now
+    let tooNew = new Date(Date.now() + 3 * 24 * 60 * 60 * 1000);
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      // -- missing
+      if (val == null) {
+        this.missing++;
+        continue;
+      }
+
+      // -- unreasonable
+      if (val < tooOld || val > tooNew) {
+        this.unreasonable++;
+        continue;
+      }
+
+      this.validItems.push(item);
+
+      // -- time range
+      if (oldest == null)
+        oldest = newest = val;
+      else if (val < oldest)
+        oldest = val;
+      else if (val > newest)
+        newest = val;
+
+      // -- bucket
+      // - year
+      let year, valYear = val.getYear();
+      if (valYear in years) {
+        year = years[valYear];
+        year._dateCount++;
+      }
+      else {
+        year = years[valYear] = {
+          _dateCount: 1,
+          _subCount: 0
+        };
+        years._subCount++;
+      }
+
+      // - month
+      let month, valMonth = val.getMonth();
+      if (valMonth in year) {
+        month = year[valMonth];
+        month._dateCount++;
+      }
+      else {
+        month = year[valMonth] = {
+          _dateCount: 1,
+          _subCount: 0
+        };
+        year._subCount++;
+      }
+
+      // - day
+      let valDate = val.getDate();
+      if (valDate in month) {
+        month[valDate].push(item);
+      }
+      else {
+        month[valDate] = [item];
+      }
+    }
+
+    this.oldest = oldest;
+    this.newest = newest;
+  },
+
+  _unionMonth: function(aMonthObj) {
+    let dayItemLists = [];
+    for (let key in aMonthObj) {
+      let dayItemList = aMonthObj[key];
+      if (typeof(key) == "string" && key.startsWith('_'))
+        continue;
+      dayItemLists.push(dayItemList);
+    }
+    return Array.concat.apply([], dayItemLists);
+  },
+
+  _unionYear: function(aYearObj) {
+    let monthItemLists = [];
+    for (let key in aYearObj) {
+      let monthObj = aYearObj[key];
+      if (typeof(key) == "string" && key.startsWith('_'))
+        continue;
+      monthItemLists.push(this._unionMonth(monthObj));
+    }
+    return Array.concat.apply([], monthItemLists);
+  }
+};

mailnews/db/gloda/modules/fundattr.js

@@ -0,0 +1,907 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaFundAttr'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/datastore.js");
+Cu.import("resource:///modules/gloda/datamodel.js"); // for GlodaAttachment
+
+Cu.import("resource:///modules/gloda/noun_mimetype.js");
+Cu.import("resource:///modules/gloda/connotent.js");
+
+/**
+ * @namespace The Gloda Fundamental Attribute provider is a special attribute
+ *  provider; it provides attributes that the rest of the providers should be
+ *  able to assume exist.  Also, it may end up accessing things at a lower level
+ *  than most extension providers should do.  In summary, don't mimic this code
+ *  unless you won't complain when your code breaks.
+ */
+var GlodaFundAttr = {
+  providerName: "gloda.fundattr",
+  strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+  _log: null,
+
+  init: function gloda_explattr_init() {
+    this._log =  Log4Moz.repository.getLogger("gloda.fundattr");
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  POPULARITY_FROM_ME_TO: 10,
+  POPULARITY_FROM_ME_CC: 4,
+  POPULARITY_FROM_ME_BCC: 3,
+  POPULARITY_TO_ME: 5,
+  POPULARITY_CC_ME: 1,
+  POPULARITY_BCC_ME: 1,
+
+  /** Boost for messages 'I' sent */
+  NOTABILITY_FROM_ME: 10,
+  /** Boost for messages involving 'me'. */
+  NOTABILITY_INVOLVING_ME: 1,
+  /** Boost for message from someone in 'my' address book. */
+  NOTABILITY_FROM_IN_ADDR_BOOK: 10,
+  /** Boost for the first person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_FIRST: 8,
+  /** Boost for each additional person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_ADDL: 2,
+
+  defineAttributes: function gloda_fundattr_defineAttributes() {
+    /* ***** Conversations ***** */
+    // conversation: subjectMatches
+    this._attrConvSubject = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [Gloda.NOUN_CONVERSATION],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    /* ***** Messages ***** */
+    // folder
+    this._attrFolder = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "folder",
+      singular: true,
+      facet: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "folderID",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FOLDER,
+      }); // tested-by: test_attributes_fundamental
+    this._attrAccount = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "account",
+      canQuery: "memory",
+      singular: true,
+      facet: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_ACCOUNT
+      });
+    this._attrMessageKey = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "messageKey",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "messageKey",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+
+    // We need to surface the deleted attribute for querying, but there is no
+    //  reason for user code, so let's call it "_deleted" rather than deleted.
+    // (In fact, our validity constraints require a special query formulation
+    //  that user code should have no clue exists.  That's right user code,
+    //  that's a dare.)
+    Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "_deleted",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "deleted",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      });
+
+
+    // -- fulltext search helpers
+    // fulltextMatches.  Match over message subject, body, and attachments
+    // @testpoint gloda.noun.message.attr.fulltextMatches
+    this._attrFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "fulltextMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "messagesText",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // subjectMatches.  Fulltext match on subject
+    // @testpoint gloda.noun.message.attr.subjectMatches
+    this._attrSubjectText = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // bodyMatches. super-synthetic full-text matching...
+    // @testpoint gloda.noun.message.attr.bodyMatches
+    this._attrBody = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "bodyMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "body",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // attachmentNamesMatch
+    // @testpoint gloda.noun.message.attr.attachmentNamesMatch
+    this._attrAttachmentNames = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "attachmentNamesMatch",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "attachmentNames",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // @testpoint gloda.noun.message.attr.authorMatches
+    this._attrAuthorFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "authorMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "author",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // @testpoint gloda.noun.message.attr.recipientsMatch
+    this._attrRecipientsFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "recipientsMatch",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "recipients",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // --- synthetic stuff for some reason
+    // conversation
+    // @testpoint gloda.noun.message.attr.conversation
+    this._attrConversation = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "conversation",
+      singular: true,
+      special: Gloda.kSpecialColumnParent,
+      specialColumnName: "conversationID",
+      idStorageAttributeName: "_conversationID",
+      valueStorageAttributeName: "_conversation",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_CONVERSATION,
+      canQuery: true,
+      });
+
+    // --- Fundamental
+    // From
+    this._attrFrom = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "from",
+                        singular: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // tested-by: test_attributes_fundamental
+    // To
+    this._attrTo = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "to",
+                        singular: false,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // tested-by: test_attributes_fundamental
+    // Cc
+    this._attrCc = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "cc",
+                        singular: false,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // not-tested
+    /**
+     * Bcc'ed recipients; only makes sense for sent messages.
+     */
+    this._attrBcc = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "bcc",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+    }); // not-tested
+
+    // Date.  now lives on the row.
+    this._attrDate = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "date",
+                        singular: true,
+                        facet: {
+                          type: "date",
+                        },
+                        special: Gloda.kSpecialColumn,
+                        specialColumnName: "date",
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_DATE,
+                        }); // tested-by: test_attributes_fundamental
+
+    // Header message ID.
+    this._attrHeaderMessageID = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "headerMessageID",
+                        singular: true,
+                        special: Gloda.kSpecialString,
+                        specialColumnName: "headerMessageID",
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_STRING,
+                        canQuery: true,
+                        }); // tested-by: test_attributes_fundamental
+
+    // Attachment MIME Types
+    this._attrAttachmentTypes = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "attachmentTypes",
+      singular: false,
+      emptySetIsSignificant: true,
+      facet: {
+        type: "default",
+        // This will group the MIME types by their category.
+        groupIdAttr: "category",
+        queryHelper: "Category",
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_MIME_TYPE,
+      });
+
+    // Attachment infos
+    this._attrIsEncrypted = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "isEncrypted",
+      singular: true,
+      emptySetIsSignificant: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      });
+
+    // Attachment infos
+    this._attrAttachmentInfos = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "attachmentInfos",
+      singular: false,
+      emptySetIsSignificant: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_ATTACHMENT,
+      });
+
+    // --- Optimization
+    /**
+     * Involves means any of from/to/cc/bcc.  The queries get ugly enough
+     *  without this that it seems to justify the cost, especially given the
+     *  frequent use case.  (In fact, post-filtering for the specific from/to/cc
+     *  is probably justifiable rather than losing this attribute...)
+     */
+    this._attrInvolves = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "involves",
+      singular: false,
+      facet: {
+        type: "default",
+        /**
+         * Filter out 'me', as we have other facets that deal with that, and the
+         *  'me' identities are so likely that they distort things.
+         *
+         * @return true if the identity is not one of my identities, false if it
+         *   is.
+         */
+        filter: function gloda_explattr_involves_filter(aItem) {
+          return (!(aItem.id in Gloda.myIdentities));
+        }
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // not-tested
+
+    /**
+     * Any of to/cc/bcc.
+     */
+    this._attrRecipients = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "recipients",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // not-tested
+
+    // From Me (To/Cc/Bcc)
+    this._attrFromMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "fromMe",
+      singular: false,
+      // The interesting thing to a facet is whether the message is from me.
+      facet: {
+        type: "nonempty?"
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_PARAM_IDENTITY,
+      }); // not-tested
+    // To/Cc/Bcc Me
+    this._attrToMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "toMe",
+      // The interesting thing to a facet is whether the message is to me.
+      facet: {
+        type: "nonempty?"
+      },
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_PARAM_IDENTITY,
+      }); // not-tested
+
+
+    // -- Mailing List
+    // Non-singular, but a hard call.  Namely, it is obvious that a message can
+    //  be addressed to multiple mailing lists.  However, I don't see how you
+    //  could receive a message with more than one set of List-* headers,
+    //  since each list-serve would each send you a copy.  Based on our current
+    //  decision to treat each physical message as separate, it almost seems
+    //  right to limit the list attribute to the copy that originated at the
+    //  list.  That may sound entirely wrong, but keep in mind that until we
+    //  have seen a message from the list with the List headers, we can't
+    //  definitely know it's a mailing list (although heuristics could take us
+    //  pretty far).  As such, the quasi-singular thing is appealing.
+    // Of course, the reality is that we really want to know if a message was
+    //  sent to multiple mailing lists and be able to query on that.
+    //  Additionally, our implicit-to logic needs to work on messages that
+    //  weren't relayed by the list-serve, especially messages sent to the list
+    //  by the user.
+    this._attrList = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "mailing-list",
+                        bindName: "mailingLists",
+                        singular: false,
+                        emptySetIsSignificant: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // not-tested, not-implemented
+  },
+
+  RE_LIST_POST: /<mailto:([^>]+)>/,
+
+  /**
+   *
+   * Specializations:
+   * - Mailing Lists.  Replies to a message on a mailing list frequently only
+   *   have the list-serve as the 'to', so we try to generate a synthetic 'to'
+   *   based on the author of the parent message when possible.  (The 'possible'
+   *   part is that we may not have a copy of the parent message at the time of
+   *   processing.)
+   * - Newsgroups.  Same deal as mailing lists.
+   */
+  process: function* gloda_fundattr_process(aGlodaMessage, aRawReps,
+                                            aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+    let aMimeMsg = aRawReps.mime;
+
+    // -- From
+    // Let's use replyTo if available.
+    // er, since we are just dealing with mailing lists for now, forget the
+    //  reply-to...
+    // TODO: deal with default charset issues
+    let author = null;
+    /*
+    try {
+      author = aMsgHdr.getStringProperty("replyTo");
+    }
+    catch (ex) {
+    }
+    */
+    if (author == null || author == "")
+      author = aMsgHdr.author;
+
+    let normalizedListPost = "";
+    if (aMimeMsg && aMimeMsg.has("list-post")) {
+      let match = this.RE_LIST_POST.exec(aMimeMsg.get("list-post"));
+      if (match)
+        normalizedListPost = "<" + match[1] + ">";
+    }
+
+    // Do not use the MIME decoded variants of any of the email addresses
+    //  because if name is encoded and has a comma in it, it will break the
+    //  address parser (which already knows how to do the decoding anyways).
+    let [authorIdentities, toIdentities, ccIdentities, bccIdentities,
+         listIdentities] =
+      yield aCallbackHandle.pushAndGo(
+        Gloda.getOrCreateMailIdentities(aCallbackHandle,
+                                        author, aMsgHdr.recipients,
+                                        aMsgHdr.ccList, aMsgHdr.bccList,
+                                        normalizedListPost));
+
+    if (authorIdentities.length != 1) {
+      throw new Gloda.BadItemContentsError(
+        "Message with subject '" + aMsgHdr.mime2DecodedSubject +
+          "' somehow lacks a valid author.  Bailing.");
+    }
+    let authorIdentity = authorIdentities[0];
+    aGlodaMessage.from = authorIdentity;
+
+    // -- To, Cc, Bcc
+    aGlodaMessage.to = toIdentities;
+    aGlodaMessage.cc = ccIdentities;
+    aGlodaMessage.bcc = bccIdentities;
+
+    // -- Mailing List
+    if (listIdentities.length)
+      aGlodaMessage.mailingLists = listIdentities;
+
+    let findIsEncrypted = x =>
+      x.isEncrypted || (x.parts ? x.parts.some(findIsEncrypted) : false);
+
+    // -- Encryption
+    aGlodaMessage.isEncrypted = false;
+    if (aMimeMsg) {
+      aGlodaMessage.isEncrypted = findIsEncrypted(aMimeMsg);
+    }
+
+    // -- Attachments
+    if (aMimeMsg) {
+      // nsParseMailbox.cpp puts the attachment flag on msgHdrs as soon as it
+      // finds a multipart/mixed part. This is a good heuristic, but if it turns
+      // out the part has no filename, then we don't treat it as an attachment.
+      // We just streamed the message, and we have all the information to figure
+      // that out, so now is a good place to clear the flag if needed.
+      let foundRealAttachment = false;
+      let attachmentTypes = [];
+      for (let attachment of aMimeMsg.allAttachments) {
+        // We don't care about would-be attachments that are not user-intended
+        //  attachments but rather artifacts of the message content.
+        // We also want to avoid dealing with obviously bogus mime types.
+        //  (If you don't have a "/", you are probably bogus.)
+        if (attachment.isRealAttachment &&
+            attachment.contentType.includes("/")) {
+          attachmentTypes.push(MimeTypeNoun.getMimeType(attachment.contentType));
+        }
+        if (attachment.isRealAttachment)
+          foundRealAttachment = true;
+      }
+      if (attachmentTypes.length) {
+        aGlodaMessage.attachmentTypes = attachmentTypes;
+      }
+
+      let aMsgHdr = aRawReps.header;
+      let wasStreamed = aMsgHdr &&
+        !aGlodaMessage.isEncrypted &&
+        ((aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline) ||
+        (aMsgHdr.folder instanceof Ci.nsIMsgLocalMailFolder));
+
+      // Clear the flag if it turns out there's no attachment after all and we
+      // streamed completely the message (if we didn't, then we have no
+      // knowledge of attachments, unless bug 673370 is fixed).
+      if (!foundRealAttachment && wasStreamed)
+        aMsgHdr.markHasAttachments(false);
+
+      // This is not the same kind of attachments as above. Now, we want to
+      // provide convenience attributes to Gloda consumers, so that they can run
+      // through the list of attachments of a given message, to possibly build a
+      // visualization on top of it. We still reject bogus mime types, which
+      // means yencode won't be supported. Oh, I feel really bad.
+      let attachmentInfos = [];
+      for (let att of aMimeMsg.allUserAttachments) {
+        attachmentInfos.push(this.glodaAttFromMimeAtt(aRawReps.trueGlodaRep,
+                                                      att));
+      }
+      aGlodaMessage.attachmentInfos = attachmentInfos;
+    }
+
+    // TODO: deal with mailing lists, including implicit-to.  this will require
+    //  convincing the indexer to pass us in the previous message if it is
+    //  available.  (which we'll simply pass to everyone... it can help body
+    //  logic for quoting purposes, etc. too.)
+
+    yield Gloda.kWorkDone;
+  },
+
+  glodaAttFromMimeAtt:
+      function gloda_fundattr_glodaAttFromMimeAtt(aGlodaMessage, aAtt) {
+    // So we don't want to store the URL because it can change over time if
+    // the message is moved. What we do is store the full URL if it's a
+    // detached attachment, otherwise just keep the part information, and
+    // rebuild the URL according to where the message is sitting.
+    let part, externalUrl;
+    if (aAtt.isExternal) {
+      externalUrl = aAtt.url;
+    } else {
+      let matches = aAtt.url.match(GlodaUtils.PART_RE);
+      if (matches && matches.length)
+        part = matches[1];
+      else
+        this._log.error("Error processing attachment: " + aAtt.url);
+    }
+    return new GlodaAttachment(aGlodaMessage,
+                               aAtt.name,
+                               aAtt.contentType,
+                               aAtt.size,
+                               part,
+                               externalUrl,
+                               aAtt.isExternal);
+  },
+
+  optimize: function* gloda_fundattr_optimize(aGlodaMessage, aRawReps,
+                                              aIsNew, aCallbackHandle) {
+
+    let aMsgHdr = aRawReps.header;
+
+    // for simplicity this is used for both involves and recipients
+    let involvesIdentities = {};
+    let involves = aGlodaMessage.involves || [];
+    let recipients = aGlodaMessage.recipients || [];
+
+    // 'me' specialization optimizations
+    let toMe = aGlodaMessage.toMe || [];
+    let fromMe = aGlodaMessage.fromMe || [];
+
+    let myIdentities = Gloda.myIdentities; // needless optimization?
+    let authorIdentity = aGlodaMessage.from;
+    let isFromMe = authorIdentity.id in myIdentities;
+
+    // The fulltext search column for the author.  We want to have in here:
+    // - The e-mail address and display name as enclosed on the message.
+    // - The name per the address book card for this e-mail address, if we have
+    //   one.
+    aGlodaMessage._indexAuthor = aMsgHdr.mime2DecodedAuthor;
+    // The fulltext search column for the recipients. (same deal)
+    aGlodaMessage._indexRecipients = aMsgHdr.mime2DecodedRecipients;
+
+    if (isFromMe)
+      aGlodaMessage.notability += this.NOTABILITY_FROM_ME;
+    else {
+      let authorCard = authorIdentity.abCard;
+      if (authorCard) {
+        aGlodaMessage.notability += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+        // @testpoint gloda.noun.message.attr.authorMatches
+        aGlodaMessage._indexAuthor += ' ' + authorCard.displayName;
+      }
+    }
+
+    involves.push(authorIdentity);
+    involvesIdentities[authorIdentity.id] = true;
+
+    let involvedAddrBookCount = 0;
+
+    for (let toIdentity of aGlodaMessage.to) {
+      if (!(toIdentity.id in involvesIdentities)) {
+        involves.push(toIdentity);
+        recipients.push(toIdentity);
+        involvesIdentities[toIdentity.id] = true;
+        let toCard = toIdentity.abCard;
+        if (toCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + toCard.displayName;
+        }
+      }
+
+      // optimization attribute to-me ('I' am the parameter)
+      if (toIdentity.id in myIdentities) {
+        toMe.push([toIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_TO_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, toIdentity]);
+        // also, popularity
+        if (aIsNew)
+          toIdentity.contact.popularity += this.POPULARITY_FROM_ME_TO;
+      }
+    }
+    for (let ccIdentity of aGlodaMessage.cc) {
+      if (!(ccIdentity.id in involvesIdentities)) {
+        involves.push(ccIdentity);
+        recipients.push(ccIdentity);
+        involvesIdentities[ccIdentity.id] = true;
+        let ccCard = ccIdentity.abCard;
+        if (ccCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + ccCard.displayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (ccIdentity.id in myIdentities) {
+        toMe.push([ccIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_CC_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, ccIdentity]);
+        // also, popularity
+        if (aIsNew)
+          ccIdentity.contact.popularity += this.POPULARITY_FROM_ME_CC;
+      }
+    }
+    // just treat bcc like cc; the intent is the same although the exact
+    //  semantics differ.
+    for (let bccIdentity of aGlodaMessage.bcc) {
+      if (!(bccIdentity.id in involvesIdentities)) {
+        involves.push(bccIdentity);
+        recipients.push(bccIdentity);
+        involvesIdentities[bccIdentity.id] = true;
+        let bccCard = bccIdentity.abCard;
+        if (bccCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + bccCard.displayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (bccIdentity.id in myIdentities) {
+        toMe.push([bccIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_BCC_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, bccIdentity]);
+        // also, popularity
+        if (aIsNew)
+          bccIdentity.contact.popularity += this.POPULARITY_FROM_ME_BCC;
+      }
+    }
+
+    if (involvedAddrBookCount)
+      aGlodaMessage.notability += this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+
+    aGlodaMessage.involves = involves;
+    aGlodaMessage.recipients = recipients;
+    if (toMe.length) {
+      aGlodaMessage.toMe = toMe;
+      aGlodaMessage.notability += this.NOTABILITY_INVOLVING_ME;
+    }
+    if (fromMe.length)
+      aGlodaMessage.fromMe = fromMe;
+
+    // Content
+    if (aRawReps.bodyLines) {
+      aGlodaMessage._content = aRawReps.content = new GlodaContent();
+      if (this.contentWhittle({}, aRawReps.bodyLines, aGlodaMessage._content)) {
+        // we were going to do something here?
+      }
+    }
+    else {
+      aRawReps.content = null;
+    }
+
+    yield Gloda.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from optimize().  Arguably optimize should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score: function gloda_fundattr_score(aMessage, aContext) {
+    let score = 0;
+
+    let authorIdentity = aMessage.from;
+    if (authorIdentity.id in Gloda.myIdentities)
+      score += this.NOTABILITY_FROM_ME;
+    else if (authorIdentity.inAddressBook)
+      score += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+    if (aMessage.toMe)
+      score += this.NOTABILITY_INVOLVING_ME;
+
+    let involvedAddrBookCount = 0;
+    for (let [, identity] in Iterator(aMessage.to))
+      if (identity.inAddressBook)
+        involvedAddrBookCount++;
+    for (let [, identity] in Iterator(aMessage.cc))
+      if (identity.inAddressBook)
+        involvedAddrBookCount++;
+    if (involvedAddrBookCount)
+      score += this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+    return score;
+  },
+
+  _countQuoteDepthAndNormalize:
+    function gloda_fundattr__countQuoteDepthAndNormalize(aLine) {
+    let count = 0;
+    let lastStartOffset = 0;
+
+    for (let i = 0; i < aLine.length; i++) {
+      let c = aLine[i];
+      if (c == ">") {
+        count++;
+        lastStartOffset = i+1;
+      }
+      else if (c == " ") {
+      }
+      else {
+        return [count,
+                lastStartOffset ? aLine.substring(lastStartOffset) : aLine];
+      }
+    }
+
+    return [count, lastStartOffset ? aLine.substring(lastStartOffset) : aLine];
+  },
+
+  /**
+   * Attempt to understand simple quoting constructs that use ">" with
+   * obvious phrases to enter the quoting block.  No support for other types
+   * of quoting at this time.  Also no support for piercing the wrapper of
+   * forwarded messages to actually be the content of the forwarded message.
+   */
+  contentWhittle: function gloda_fundattr_contentWhittle(aMeta,
+      aBodyLines, aContent) {
+    if (!aContent.volunteerContent(aContent.kPriorityBase))
+      return false;
+
+    // duplicate the list; we mutate somewhat...
+    let bodyLines = aBodyLines.concat();
+
+    // lastNonBlankLine originally was just for detecting quoting idioms where
+    //  the "wrote" line was separated from the quoted block by a blank line.
+    // Now we also use it for whitespace suppression at the boundaries of
+    //  quoted and un-quoted text.  (We keep blank lines within the same
+    //  'block' of quoted or non-quoted text.)
+    // Because we now have two goals for it, and we still want to suppress blank
+    //  lines when there is a 'wrote' line involved, we introduce...
+    //  prevLastNonBlankLine!  This arguably suggests refactoring should be the
+    //  next step, but things work for now.
+    let rangeStart = 0, lastNonBlankLine = null, prevLastNonBlankLine = null;
+    let inQuoteDepth = 0;
+    for (let [iLine, line] of bodyLines.entries()) {
+      if (!line || (line == "\xa0")) /* unicode non breaking space */
+        continue;
+
+      if (line.startsWith(">")) {
+        if (!inQuoteDepth) {
+          let rangeEnd = iLine - 1;
+          let quoteRangeStart = iLine;
+          // see if the last non-blank-line was a lead-in...
+          if (lastNonBlankLine != null) {
+            // TODO: localize quote range start detection
+            if (aBodyLines[lastNonBlankLine].includes("wrote")) {
+              quoteRangeStart = lastNonBlankLine;
+              rangeEnd = lastNonBlankLine - 1;
+              // we 'used up' lastNonBlankLine, let's promote the prev guy to
+              //  be the new lastNonBlankLine for the next logic block
+              lastNonBlankLine = prevLastNonBlankLine;
+            }
+            // eat the trailing whitespace...
+            if (lastNonBlankLine != null)
+              rangeEnd = Math.min(rangeEnd, lastNonBlankLine);
+          }
+          if (rangeEnd >= rangeStart)
+            aContent.content(aBodyLines.slice(rangeStart, rangeEnd+1));
+
+          [inQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+          rangeStart = quoteRangeStart;
+        }
+        else {
+          let curQuoteDepth;
+          [curQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+
+          if (curQuoteDepth != inQuoteDepth) {
+            // we could do some "wrote" compensation here, but it's not really
+            //  as important.  let's wait for a more clever algorithm.
+            aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+            inQuoteDepth = curQuoteDepth;
+            rangeStart = iLine;
+          }
+        }
+      }
+      else {
+        if (inQuoteDepth) {
+          aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+          inQuoteDepth = 0;
+          rangeStart = iLine;
+        }
+      }
+
+      prevLastNonBlankLine = lastNonBlankLine;
+      lastNonBlankLine = iLine;
+    }
+
+    if (inQuoteDepth) {
+      aContent.quoted(aBodyLines.slice(rangeStart), inQuoteDepth);
+    }
+    else {
+      aContent.content(aBodyLines.slice(rangeStart, lastNonBlankLine+1));
+    }
+
+    return true;
+  },
+};

mailnews/db/gloda/modules/index_ab.js

@@ -0,0 +1,287 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaABIndexer', 'GlodaABAttrs'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/collection.js");
+Cu.import("resource:///modules/gloda/datastore.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/indexer.js");
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/gloda/noun_freetag.js");
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/mailServices.js");
+
+
+var GlodaABIndexer = {
+  _log: null,
+
+  name: "index_ab",
+  enable: function() {
+    if (this._log == null)
+      this._log =  Log4Moz.repository.getLogger("gloda.index_ab");
+
+    MailServices.ab.addAddressBookListener(this,
+                                           Ci.nsIAbListener.itemAdded |
+                                           Ci.nsIAbListener.itemChanged |
+                                           Ci.nsIAbListener.directoryItemRemoved);
+  },
+
+  disable: function() {
+    MailServices.ab.removeAddressBookListener(this);
+  },
+
+  // it's a getter so we can reference 'this'
+  get workers() {
+    return [
+      ["ab-card", {
+         worker: this._worker_index_card,
+       }],
+    ];
+  },
+
+  _worker_index_card: function*(aJob, aCallbackHandle) {
+    let card = aJob.id;
+
+    if (card.primaryEmail) {
+      // load the identity
+      let query = Gloda.newQuery(Gloda.NOUN_IDENTITY);
+      query.kind("email");
+      // we currently normalize all e-mail addresses to be lowercase
+      query.value(card.primaryEmail.toLowerCase());
+      let identityCollection = query.getCollection(aCallbackHandle);
+      yield Gloda.kWorkAsync;
+
+      if (identityCollection.items.length) {
+        let identity = identityCollection.items[0];
+        // force the identity to know it has an associated ab card.
+        identity._hasAddressBookCard = true;
+
+        this._log.debug("Found identity, processing card.");
+        yield aCallbackHandle.pushAndGo(
+            Gloda.grokNounItem(identity.contact, {card: card}, false, false,
+                               aCallbackHandle));
+        this._log.debug("Done processing card.");
+      }
+    }
+
+    yield GlodaIndexer.kWorkDone;
+  },
+
+  initialSweep: function() {
+  },
+
+  /* ------ nsIAbListener ------ */
+  /**
+   * When an address book card is added, update the cached GlodaIdentity
+   *  object's cached idea of whether the identity has an ab card.
+   */
+  onItemAdded: function ab_indexer_onItemAdded(aParentDir, aItem) {
+    if (!(aItem instanceof Ci.nsIAbCard))
+      return;
+
+    this._log.debug("Received Card Add Notification");
+    let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+      Gloda.NOUN_IDENTITY, "email@" + aItem.primaryEmail.toLowerCase());
+    if (identity)
+      identity._hasAddressBookCard = true;
+  },
+  /**
+   * When an address book card is added, update the cached GlodaIdentity
+   *  object's cached idea of whether the identity has an ab card.
+   */
+  onItemRemoved: function ab_indexer_onItemRemoved(aParentDir, aItem) {
+    if (!(aItem instanceof Ci.nsIAbCard))
+      return;
+
+    this._log.debug("Received Card Removal Notification");
+    let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+      Gloda.NOUN_IDENTITY, "email@" + aItem.primaryEmail.toLowerCase());
+    if (identity)
+      identity._hasAddressBookCard = false;
+
+  },
+  onItemPropertyChanged: function ab_indexer_onItemPropertyChanged(aItem,
+      aProperty, aOldValue, aNewValue) {
+    if (aProperty == null && aItem instanceof Ci.nsIAbCard) {
+      this._log.debug("Received Card Change Notification");
+
+      let card = aItem; // instanceof already QueryInterface'd for us.
+      let job = new IndexingJob("ab-card", card);
+      GlodaIndexer.indexJob(job);
+    }
+  }
+};
+GlodaIndexer.registerIndexer(GlodaABIndexer);
+
+var GlodaABAttrs = {
+  providerName: "gloda.ab_attr",
+  _log: null,
+
+  init: function() {
+    this._log =  Log4Moz.repository.getLogger("gloda.abattrs");
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  defineAttributes: function() {
+    /* ***** Contacts ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "identities",
+      singular: false,
+      special: Gloda.kSpecialColumnChildren,
+      //specialColumnName: "contactID",
+      storageAttributeName: "_identities",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // tested-by: test_attributes_fundamental
+    this._attrContactName = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "name",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "name",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrContactPopularity = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "popularity",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "popularity",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // not-tested
+    this._attrContactFrecency = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "frecency",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "frecency",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // not-tested
+
+    /* ***** Identities ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "contact",
+      singular: true,
+      special: Gloda.kSpecialColumnParent,
+      specialColumnName: "contactID", // the column in the db
+      idStorageAttributeName: "_contactID",
+      valueStorageAttributeName: "_contact",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_CONTACT,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrIdentityKind = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "kind",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "kind",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrIdentityValue = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "value",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "value",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+
+    /* ***** Contact Meta ***** */
+    // Freeform tags; not explicit like thunderbird's fundamental tags.
+    //  we differentiate for now because of fundamental implementation
+    //  differences.
+    this._attrFreeTag = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "freetag",
+      bind: true,
+      bindName: "freeTags",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.lookupNoun("freetag"),
+      parameterNoun: null,
+      canQuery: true,
+      }); // not-tested
+    // we need to find any existing bound freetag attributes, and use them to
+    //  populate to FreeTagNoun's understanding
+    if ("parameterBindings" in this._attrFreeTag) {
+      for (let freeTagName in this._attrFreeTag.parameterBindings) {
+        this._log.debug("Telling FreeTagNoun about: " + freeTagName);
+        FreeTagNoun.getFreeTag(freeTagName);
+      }
+    }
+  },
+
+  process: function*(aContact, aRawReps, aIsNew, aCallbackHandle) {
+    let card = aRawReps.card;
+    if (aContact.NOUN_ID != Gloda.NOUN_CONTACT) {
+      this._log.warn("Somehow got a non-contact: " + aContact);
+      return; // this will produce an exception; we like.
+    }
+
+    // update the name
+    if (card.displayName && card.displayName != aContact.name)
+      aContact.name = card.displayName;
+
+    aContact.freeTags = [];
+
+    let tags = null;
+    try {
+      tags = card.getProperty("Categories", null);
+    } catch (ex) {
+      this._log.error("Problem accessing property: " + ex);
+    }
+    if (tags) {
+      for (let tagName of tags.split(",")) {
+        tagName = tagName.trim();
+        if (tagName) {
+          aContact.freeTags.push(FreeTagNoun.getFreeTag(tagName));
+        }
+      }
+    }
+
+    yield Gloda.kWorkDone;
+  }
+};

mailnews/db/gloda/modules/log4moz.js

@@ -0,0 +1,932 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['Log4Moz'];
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+var MODE_RDONLY   = 0x01;
+var MODE_WRONLY   = 0x02;
+var MODE_CREATE   = 0x08;
+var MODE_APPEND   = 0x10;
+var MODE_TRUNCATE = 0x20;
+
+var PERMS_FILE      = parseInt("0644", 8);
+var PERMS_DIRECTORY = parseInt("0755", 8);
+
+var ONE_BYTE = 1;
+var ONE_KILOBYTE = 1024 * ONE_BYTE;
+var ONE_MEGABYTE = 1024 * ONE_KILOBYTE;
+
+var DEFAULT_NETWORK_TIMEOUT_DELAY = 5;
+
+var CDATA_START = "<![CDATA[";
+var CDATA_END = "]]>";
+var CDATA_ESCAPED_END = CDATA_END + "]]&gt;" + CDATA_START;
+
+var Log4Moz = {
+  Level: {
+    Fatal:  70,
+    Error:  60,
+    Warn:   50,
+    Info:   40,
+    Config: 30,
+    Debug:  20,
+    Trace:  10,
+    All:    0,
+    Desc: {
+      70: "FATAL",
+      60: "ERROR",
+      50: "WARN",
+      40: "INFO",
+      30: "CONFIG",
+      20: "DEBUG",
+      10: "TRACE",
+      0:  "ALL"
+    }
+  },
+
+  /**
+   * Create a logger and configure it with dump and console appenders as
+   * specified by prefs based on the logger name.
+   *
+   * E.g., if the loggername is foo, then look for prefs
+   *   foo.logging.console
+   *   foo.logging.dump
+   *
+   * whose values can be empty: no logging of that type; or any of
+   * 'Fatal', 'Error', 'Warn', 'Info', 'Config', 'Debug', 'Trace', 'All',
+   * in which case the logging level for each appender will be set accordingly
+   *
+   * Parameters:
+   *
+   * @param loggername The name of the logger
+   * @param level (optional) the level of the logger itself
+   * @param consoleLevel (optional) the level of the console appender
+   * @param dumpLevel (optional) the level of the dump appender
+   *
+   * As described above, well-named prefs override the last two parameters
+   **/
+
+  getConfiguredLogger: function(loggername, level, consoleLevel, dumpLevel) {
+    let log = Log4Moz.repository.getLogger(loggername);
+    if (log._configured)
+      return log
+
+    let formatter = new Log4Moz.BasicFormatter();
+
+    level = level || Log4Moz.Level.Error;
+
+    consoleLevel = consoleLevel || -1;
+    dumpLevel = dumpLevel || -1;
+    let branch = Services.prefs.getBranch(loggername + ".logging.");
+    if (branch)
+    {
+      try {
+        // figure out if event-driven indexing should be enabled...
+        let consoleLevelString = branch.getCharPref("console");
+        if (consoleLevelString) {
+          // capitalize to fit with Log4Moz.Level expectations
+          consoleLevelString = consoleLevelString.charAt(0).toUpperCase() +
+             consoleLevelString.substr(1).toLowerCase();
+          consoleLevel = (consoleLevelString == 'None') ?
+                          100 : Log4Moz.Level[consoleLevelString];
+        }
+      } catch (ex) {
+        // Ignore if preference is not found
+      }
+      try {
+        let dumpLevelString = branch.getCharPref("dump");
+        if (dumpLevelString) {
+          // capitalize to fit with Log4Moz.Level expectations
+          dumpLevelString = dumpLevelString.charAt(0).toUpperCase() +
+             dumpLevelString.substr(1).toLowerCase();
+          dumpLevel = (dumpLevelString == 'None') ?
+                       100 : Log4Moz.Level[dumpLevelString];
+        }
+      } catch (ex) {
+        // Ignore if preference is not found
+      }
+    }
+
+    if (consoleLevel != 100) {
+      if (consoleLevel == -1)
+        consoleLevel = Log4Moz.Level.Error;
+      let capp = new Log4Moz.ConsoleAppender(formatter);
+      capp.level = consoleLevel;
+      log.addAppender(capp);
+    }
+
+    if (dumpLevel != 100) {
+      if (dumpLevel == -1)
+        dumpLevel = Log4Moz.Level.Error;
+      let dapp = new Log4Moz.DumpAppender(formatter);
+      dapp.level = dumpLevel;
+      log.addAppender(dapp);
+    }
+
+    log.level = Math.min(level, Math.min(consoleLevel, dumpLevel));
+
+    log._configured = true;
+
+    return log;
+  },
+
+  get repository() {
+    delete Log4Moz.repository;
+    Log4Moz.repository = new LoggerRepository();
+    return Log4Moz.repository;
+  },
+  set repository(value) {
+    delete Log4Moz.repository;
+    Log4Moz.repository = value;
+  },
+
+  get LogMessage() { return LogMessage; },
+  get Logger() { return Logger; },
+  get LoggerRepository() { return LoggerRepository; },
+
+  get Formatter() { return Formatter; },
+  get BasicFormatter() { return BasicFormatter; },
+  get XMLFormatter() { return XMLFormatter; },
+  get JSONFormatter() { return JSONFormatter; },
+  get Appender() { return Appender; },
+  get DumpAppender() { return DumpAppender; },
+  get ConsoleAppender() { return ConsoleAppender; },
+  get TimeAwareMemoryBucketAppender() { return TimeAwareMemoryBucketAppender; },
+  get FileAppender() { return FileAppender; },
+  get SocketAppender() { return SocketAppender; },
+  get RotatingFileAppender() { return RotatingFileAppender; },
+  get ThrowingAppender() { return ThrowingAppender; },
+
+  // Logging helper:
+  // let logger = Log4Moz.repository.getLogger("foo");
+  // logger.info(Log4Moz.enumerateInterfaces(someObject).join(","));
+  enumerateInterfaces: function Log4Moz_enumerateInterfaces(aObject) {
+    let interfaces = [];
+
+    for (i in Ci) {
+      try {
+        aObject.QueryInterface(Ci[i]);
+        interfaces.push(i);
+      }
+      catch(ex) {}
+    }
+
+    return interfaces;
+  },
+
+  // Logging helper:
+  // let logger = Log4Moz.repository.getLogger("foo");
+  // logger.info(Log4Moz.enumerateProperties(someObject).join(","));
+  enumerateProperties: function Log4Moz_enumerateProps(aObject,
+                                                       aExcludeComplexTypes) {
+    let properties = [];
+
+    for (var p in aObject) {
+      try {
+        if (aExcludeComplexTypes &&
+            (typeof aObject[p] == "object" || typeof aObject[p] == "function"))
+          continue;
+        properties.push(p + " = " + aObject[p]);
+      }
+      catch(ex) {
+        properties.push(p + " = " + ex);
+      }
+    }
+
+    return properties;
+  }
+};
+
+function LoggerContext() {
+  this._started = this._lastStateChange = Date.now();
+  this._state = "started";
+}
+LoggerContext.prototype = {
+  _jsonMe: true,
+  _id: "unknown",
+  setState: function LoggerContext_state(aState) {
+    this._state = aState;
+    this._lastStateChange = Date.now();
+    return this;
+  },
+  finish: function LoggerContext_finish() {
+    this._finished = Date.now();
+    this._state = "finished";
+    return this;
+  },
+  toString: function LoggerContext_toString() {
+    return "[Context: " + this._id + " state: " + this._state + "]";
+  }
+};
+
+
+/*
+ * LogMessage
+ * Encapsulates a single log event's data
+ */
+function LogMessage(loggerName, level, messageObjects){
+  this.loggerName = loggerName;
+  this.messageObjects = messageObjects;
+  this.level = level;
+  this.time = Date.now();
+}
+LogMessage.prototype = {
+  get levelDesc() {
+    if (this.level in Log4Moz.Level.Desc)
+      return Log4Moz.Level.Desc[this.level];
+    return "UNKNOWN";
+  },
+
+  toString: function LogMsg_toString(){
+    return "LogMessage [" + this.time + " " + this.level + " " +
+      this.messageObjects + "]";
+  }
+};
+
+/*
+ * Logger
+ * Hierarchical version.  Logs to all appenders, assigned or inherited
+ */
+
+function Logger(name, repository) {
+  this._init(name, repository);
+}
+Logger.prototype = {
+  _init: function Logger__init(name, repository) {
+    if (!repository)
+      repository = Log4Moz.repository;
+    this._name = name;
+    this.children = [];
+    this.ownAppenders = [];
+    this.appenders = [];
+    this._repository = repository;
+  },
+
+  get name() {
+    return this._name;
+  },
+
+  _level: null,
+  get level() {
+    if (this._level != null)
+      return this._level;
+    if (this.parent)
+      return this.parent.level;
+    dump("log4moz warning: root logger configuration error: no level defined\n");
+    return Log4Moz.Level.All;
+  },
+  set level(level) {
+    this._level = level;
+  },
+
+  _parent: null,
+  get parent() { return this._parent; },
+  set parent(parent) {
+    if (this._parent == parent) {
+      return;
+    }
+    // Remove ourselves from parent's children
+    if (this._parent) {
+      let index = this._parent.children.indexOf(this);
+      if (index != -1) {
+        this._parent.children.splice(index, 1);
+      }
+    }
+    this._parent = parent;
+    parent.children.push(this);
+    this.updateAppenders();
+  },
+
+  updateAppenders: function updateAppenders() {
+    if (this._parent) {
+      let notOwnAppenders = this._parent.appenders.filter(function(appender) {
+        return this.ownAppenders.indexOf(appender) == -1;
+      }, this);
+      this.appenders = notOwnAppenders.concat(this.ownAppenders);
+    } else {
+      this.appenders = this.ownAppenders.slice();
+    }
+
+    // Update children's appenders.
+    for (let i = 0; i < this.children.length; i++) {
+      this.children[i].updateAppenders();
+    }
+  },
+
+  addAppender: function Logger_addAppender(appender) {
+    if (this.ownAppenders.indexOf(appender) != -1) {
+      return;
+    }
+    this.ownAppenders.push(appender);
+    this.updateAppenders();
+  },
+
+  _nextContextId: 0,
+  newContext: function Logger_newContext(objWithProps) {
+    if (!("_id" in objWithProps))
+      objWithProps._id = this._name + ":" + (++this._nextContextId);
+
+    let c = new LoggerContext();
+    c._isContext = true;
+    for (let key in objWithProps) {
+      c[key] = objWithProps[key];
+    }
+    return c;
+  },
+
+  log: function Logger_log(message) {
+    if (this.level > message.level)
+      return;
+    let appenders = this.appenders;
+    for (let i = 0; i < appenders.length; i++){
+      appenders[i].append(message);
+    }
+  },
+
+  removeAppender: function Logger_removeAppender(appender) {
+    let index = this.ownAppenders.indexOf(appender);
+    if (index == -1) {
+      return;
+    }
+    this.ownAppenders.splice(index, 1);
+    this.updateAppenders();
+  },
+
+  log: function Logger_log(level, args) {
+    if (this.level > level)
+      return;
+
+    // Hold off on creating the message object until we actually have
+    // an appender that's responsible.
+    let message;
+    let appenders = this.appenders;
+    for (let i = 0; i < appenders.length; i++){
+      let appender = appenders[i];
+      if (appender.level > level)
+        continue;
+
+      if (!message)
+        message = new LogMessage(this._name, level,
+                                 Array.prototype.slice.call(args));
+
+      appender.append(message);
+    }
+  },
+
+  fatal: function Logger_fatal() {
+    this.log(Log4Moz.Level.Fatal, arguments);
+  },
+  error: function Logger_error() {
+    this.log(Log4Moz.Level.Error, arguments);
+  },
+  warn: function Logger_warn() {
+    this.log(Log4Moz.Level.Warn, arguments);
+  },
+  info: function Logger_info(string) {
+    this.log(Log4Moz.Level.Info, arguments);
+  },
+  config: function Logger_config(string) {
+    this.log(Log4Moz.Level.Config, arguments);
+  },
+  debug: function Logger_debug(string) {
+    this.log(Log4Moz.Level.Debug, arguments);
+  },
+  trace: function Logger_trace(string) {
+    this.log(Log4Moz.Level.Trace, arguments);
+  }
+};
+
+/*
+ * LoggerRepository
+ * Implements a hierarchy of Loggers
+ */
+
+function LoggerRepository() {}
+LoggerRepository.prototype = {
+  _loggers: {},
+
+  _rootLogger: null,
+  get rootLogger() {
+    if (!this._rootLogger) {
+      this._rootLogger = new Logger("root", this);
+      this._rootLogger.level = Log4Moz.Level.All;
+    }
+    return this._rootLogger;
+  },
+  set rootLogger(logger) {
+    throw "Cannot change the root logger";
+  },
+
+  _updateParents: function LogRep__updateParents(name) {
+    let pieces = name.split('.');
+    let cur, parent;
+
+    // find the closest parent
+    // don't test for the logger name itself, as there's a chance it's already
+    // there in this._loggers
+    for (let i = 0; i < pieces.length - 1; i++) {
+      if (cur)
+        cur += '.' + pieces[i];
+      else
+        cur = pieces[i];
+      if (cur in this._loggers)
+        parent = cur;
+    }
+
+    // if we didn't assign a parent above, there is no parent
+    if (!parent)
+      this._loggers[name].parent = this.rootLogger;
+    else
+      this._loggers[name].parent = this._loggers[parent];
+
+    // trigger updates for any possible descendants of this logger
+    for (let logger in this._loggers) {
+      if (logger != name && logger.indexOf(name) == 0)
+        this._updateParents(logger);
+    }
+  },
+
+  getLogger: function LogRep_getLogger(name) {
+    if (name in this._loggers)
+      return this._loggers[name];
+    this._loggers[name] = new Logger(name, this);
+    this._updateParents(name);
+    return this._loggers[name];
+  }
+};
+
+/*
+ * Formatters
+ * These massage a LogMessage into whatever output is desired
+ * Only the BasicFormatter is currently implemented
+ */
+
+// Abstract formatter
+function Formatter() {}
+Formatter.prototype = {
+  format: function Formatter_format(message) {}
+};
+
+// services' log4moz lost the date formatting default...
+function BasicFormatter(dateFormat) {
+  if (dateFormat)
+    this.dateFormat = dateFormat;
+}
+BasicFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  _dateFormat: null,
+
+  get dateFormat() {
+    if (!this._dateFormat)
+      this._dateFormat = "%Y-%m-%d %H:%M:%S";
+    return this._dateFormat;
+  },
+
+  set dateFormat(format) {
+    this._dateFormat = format;
+  },
+
+  format: function BF_format(message) {
+    let date = new Date(message.time);
+    // The trick below prevents errors further down because mo is null or
+    //  undefined.
+    let messageString = message.messageObjects.map(mo => "" + mo).join(" ");
+    return date.toLocaleFormat(this.dateFormat) + "\t" +
+      message.loggerName + "\t" + message.levelDesc + "\t" +
+      messageString + "\n";
+  }
+};
+
+/*
+ * XMLFormatter
+ * Format like log4j's XMLLayout.  The intent is that you can hook this up to
+ * a SocketAppender and point them at a Chainsaw GUI running with an
+ * XMLSocketReceiver running.  Then your output comes out in Chainsaw.
+ * (Chainsaw is log4j's GUI that displays log output with niceties such as
+ * filtering and conditional coloring.)
+ */
+
+function XMLFormatter() {}
+XMLFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  format: function XF_format(message) {
+    let cdataEscapedMessage =
+      message.messageObjects
+        .map(mo => (typeof(mo) == "object") ? mo.toString() : mo)
+        .join(" ")
+        .split(CDATA_END).join(CDATA_ESCAPED_END);
+    return "<log4j:event logger='" + message.loggerName + "' " +
+                        "level='" + message.levelDesc + "' thread='unknown' " +
+                        "timestamp='" + message.time + "'>" +
+      "<log4j:message><![CDATA[" + cdataEscapedMessage + "]]></log4j:message>" +
+      "</log4j:event>";
+  }
+};
+
+function JSONFormatter() {
+}
+JSONFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  format: function JF_format(message) {
+    // XXX I did all kinds of questionable things in here; they should be
+    //  resolved...
+    // 1) JSON does not walk the __proto__ chain; there is no need to clobber
+    //   it.
+    // 2) Our net mutation is sorta redundant messageObjects alongside
+    //   msgObjects, although we only serialize one.
+    let origMessageObjects = message.messageObjects;
+    message.messageObjects = [];
+    let reProto = [];
+    for (let messageObject of origMessageObjects) {
+      if (messageObject)
+        if (messageObject._jsonMe) {
+          message.messageObjects.push(messageObject);
+// FIXME: the commented out code should be fixed in a better way.
+// See bug 984539: find a good way to avoid JSONing the impl in log4moz
+//          // temporarily strip the prototype to avoid JSONing the impl.
+//          reProto.push([messageObject, messageObject.__proto__]);
+//          messageObject.__proto__ = undefined;
+        }
+        else
+          message.messageObjects.push(messageObject.toString());
+      else
+        message.messageObjects.push(messageObject);
+    }
+    let encoded = JSON.stringify(message) + "\r\n";
+    message.msgObjects = origMessageObjects;
+//    for (let objectAndProtoPair of reProto) {
+//      objectAndProtoPair[0].__proto__ = objectAndProtoPair[1];
+//    }
+    return encoded;
+  }
+};
+
+
+/*
+ * Appenders
+ * These can be attached to Loggers to log to different places
+ * Simply subclass and override doAppend to implement a new one
+ */
+
+function Appender(formatter) {
+  this._name = "Appender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+Appender.prototype = {
+  _level: Log4Moz.Level.All,
+
+  append: function App_append(message) {
+    this.doAppend(this._formatter.format(message));
+  },
+  toString: function App_toString() {
+    return this._name + " [level=" + this._level +
+      ", formatter=" + this._formatter + "]";
+  },
+  doAppend: function App_doAppend(message) {}
+};
+
+/*
+ * DumpAppender
+ * Logs to standard out
+ */
+
+function DumpAppender(formatter) {
+  this._name = "DumpAppender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+DumpAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  doAppend: function DApp_doAppend(message) {
+    dump(message);
+  }
+};
+
+/**
+ * An in-memory appender that always logs to its in-memory bucket and associates
+ * each message with a timestamp.  Whoever creates us is responsible for causing
+ * us to switch to a new bucket using whatever criteria is appropriate.
+ *
+ * This is intended to be used roughly like an in-memory circular buffer.  The
+ * expectation is that we are being used for unit tests and that each unit test
+ * function will get its own bucket.  In the event that a test fails we would
+ * be asked for the contents of the current bucket and some portion of the
+ * previous bucket using up to some duration.
+ */
+function TimeAwareMemoryBucketAppender() {
+  this._name = "TimeAwareMemoryBucketAppender";
+  this._level = Log4Moz.Level.All;
+
+  this._lastBucket = null;
+  // to minimize object construction, even indices are timestamps, odd indices
+  //  are the message objects.
+  this._curBucket = [];
+  this._curBucketStartedAt = Date.now();
+}
+TimeAwareMemoryBucketAppender.prototype = {
+  get level() { return this._level; },
+  set level(level) { this._level = level; },
+
+  append: function TAMBA_append(message) {
+    if (this._level <= message.level)
+      this._curBucket.push(message);
+  },
+
+  newBucket: function() {
+    this._lastBucket = this._curBucket;
+    this._curBucketStartedAt = Date.now();
+    this._curBucket = [];
+  },
+
+  getPreviousBucketEvents: function(aNumMS) {
+    let lastBucket = this._lastBucket;
+    if (lastBucket == null || !lastBucket.length)
+      return [];
+    let timeBound = this._curBucketStartedAt - aNumMS;
+    // seek backwards through the list...
+    let i;
+    for (i = lastBucket.length - 1; i >= 0; i --) {
+      if (lastBucket[i].time < timeBound)
+        break;
+    }
+    return lastBucket.slice(i+1);
+  },
+
+  getBucketEvents: function() {
+    return this._curBucket.concat();
+  },
+
+  toString: function() {
+    return "[TimeAwareMemoryBucketAppender]";
+  },
+};
+
+/*
+ * ConsoleAppender
+ * Logs to the javascript console
+ */
+
+function ConsoleAppender(formatter) {
+  this._name = "ConsoleAppender";
+  this._formatter = formatter;
+}
+ConsoleAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  // override to send Error and higher level messages to Components.utils.reportError()
+  append: function CApp_append(message) {
+    let stringMessage = this._formatter.format(message);
+    if (message.level > Log4Moz.Level.Warn) {
+      Cu.reportError(stringMessage);
+    }
+    this.doAppend(stringMessage);
+  },
+
+  doAppend: function CApp_doAppend(message) {
+    Services.console.logStringMessage(message);
+  }
+};
+
+/*
+ * FileAppender
+ * Logs to a file
+ */
+
+function FileAppender(file, formatter) {
+  this._name = "FileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+FileAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  __fos: null,
+  get _fos() {
+    if (!this.__fos)
+      this.openStream();
+    return this.__fos;
+  },
+
+  openStream: function FApp_openStream() {
+    this.__fos = Cc["@mozilla.org/network/file-output-stream;1"].
+      createInstance(Ci.nsIFileOutputStream);
+    let flags = MODE_WRONLY | MODE_CREATE | MODE_APPEND;
+    this.__fos.init(this._file, flags, PERMS_FILE, 0);
+  },
+
+  closeStream: function FApp_closeStream() {
+    if (!this.__fos)
+      return;
+    try {
+      this.__fos.close();
+      this.__fos = null;
+    } catch(e) {
+      dump("Failed to close file output stream\n" + e);
+    }
+  },
+
+  doAppend: function FApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this._fos.write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+
+  clear: function FApp_clear() {
+    this.closeStream();
+    this._file.remove(false);
+  }
+};
+
+/*
+ * RotatingFileAppender
+ * Similar to FileAppender, but rotates logs when they become too large
+ */
+
+function RotatingFileAppender(file, formatter, maxSize, maxBackups) {
+  if (maxSize === undefined)
+    maxSize = ONE_MEGABYTE * 2;
+
+  if (maxBackups === undefined)
+    maxBackups = 0;
+
+  this._name = "RotatingFileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._maxSize = maxSize;
+  this._maxBackups = maxBackups;
+}
+RotatingFileAppender.prototype = {
+  __proto__: FileAppender.prototype,
+
+  doAppend: function RFApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this.rotateLogs();
+      this._fos.write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+  rotateLogs: function RFApp_rotateLogs() {
+    if(this._file.exists() &&
+       this._file.fileSize < this._maxSize)
+      return;
+
+    this.closeStream();
+
+    for (let i = this.maxBackups - 1; i > 0; i--){
+      let backup = this._file.parent.clone();
+      backup.append(this._file.leafName + "." + i);
+      if (backup.exists())
+        backup.moveTo(this._file.parent, this._file.leafName + "." + (i + 1));
+    }
+
+    let cur = this._file.clone();
+    if (cur.exists())
+      cur.moveTo(cur.parent, cur.leafName + ".1");
+
+    // Note: this._file still points to the same file
+  }
+};
+
+/*
+ * SocketAppender
+ * Logs via TCP to a given host and port.  Attempts to automatically reconnect
+ * when the connection drops or cannot be initially re-established.  Connection
+ * attempts will happen at most every timeoutDelay seconds (has a sane default
+ * if left blank).  Messages are dropped when there is no connection.
+ */
+
+function SocketAppender(host, port, formatter, timeoutDelay) {
+  this._name = "SocketAppender";
+  this._host = host;
+  this._port = port;
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._timeout_delay = timeoutDelay || DEFAULT_NETWORK_TIMEOUT_DELAY;
+
+  this._socketService = Cc["@mozilla.org/network/socket-transport-service;1"]
+                          .getService(Ci.nsISocketTransportService);
+  this._mainThread = Services.tm.mainThread;
+}
+SocketAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  __nos: null,
+  get _nos() {
+    if (!this.__nos)
+      this.openStream();
+    return this.__nos;
+  },
+  _nextCheck: 0,
+  openStream: function SApp_openStream() {
+    let now = Date.now();
+    if (now <= this._nextCheck) {
+      return;
+    }
+    this._nextCheck = now + this._timeout_delay * 1000;
+    try {
+      this._transport = this._socketService.createTransport(
+        null, 0, // default socket type
+        this._host, this._port,
+        null); // no proxy
+      this._transport.setTimeout(Ci.nsISocketTransport.TIMEOUT_CONNECT,
+                                 this._timeout_delay);
+      // do not set a timeout for TIMEOUT_READ_WRITE. The timeout is not
+      //  entirely intuitive; your socket will time out if no one reads or
+      //  writes to the socket within the timeout.  That, as you can imagine,
+      //  is not what we want.
+      this._transport.setEventSink(this, this._mainThread);
+
+      let outputStream = this._transport.openOutputStream(
+        0, // neither blocking nor unbuffered operation is desired
+        0, // default buffer size is fine
+        0 // default buffer count is fine
+        );
+
+      let uniOutputStream = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                              .createInstance(Ci.nsIConverterOutputStream);
+      uniOutputStream.init(outputStream, "utf-8", 0, 0x0000);
+
+      this.__nos = uniOutputStream;
+    } catch (ex) {
+      dump("Unexpected SocketAppender connection problem: " +
+           ex.fileName + ":" + ex.lineNumber + ": " + ex + "\n");
+    }
+  },
+
+  closeStream: function SApp_closeStream() {
+    if (!this._transport)
+      return;
+    try {
+      this._connected = false;
+      this._transport = null;
+      let nos = this.__nos;
+      this.__nos = null;
+      nos.close();
+    } catch(e) {
+      // this shouldn't happen, but no one cares
+    }
+  },
+
+  doAppend: function SApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      let nos = this._nos;
+      if (nos)
+        nos.writeString(message);
+    } catch(e) {
+      if (this._transport && !this._transport.isAlive()) {
+        this.closeStream();
+      }
+    }
+  },
+
+  clear: function SApp_clear() {
+    this.closeStream();
+  },
+
+  /* nsITransportEventSink */
+  onTransportStatus: function SApp_onTransportStatus(aTransport, aStatus,
+      aProgress, aProgressMax) {
+    if (aStatus == 0x804b0004) // STATUS_CONNECTED_TO is not a constant.
+      this._connected = true;
+  },
+};
+
+/**
+ * Throws an exception whenever it gets a message.  Intended to be used in
+ * automated testing situations where the code would normally log an error but
+ * not die in a fatal manner.
+ */
+function ThrowingAppender(thrower, formatter) {
+  this._name = "ThrowingAppender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._thrower = thrower;
+}
+ThrowingAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  doAppend: function TApp_doAppend(message) {
+    if (this._thrower)
+      this._thrower(message);
+    else
+      throw message;
+  }
+};

mailnews/db/gloda/modules/mimeTypeCategories.js

@@ -0,0 +1,204 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file wants to be a data file of some sort.  It might do better as a real
+ *  raw JSON file.  It is trying to be one right now, but it obviously is not.
+ */
+
+var EXPORTED_SYMBOLS = ['MimeCategoryMapping'];
+
+/**
+ * Input data structure to allow us to build a fast mapping from mime type to
+ *  category name.  The keys in MimeCategoryMapping are the top-level
+ *  categories.  Each value can either be a list of MIME types or a nested
+ *  object which recursively defines sub-categories.  We currently do not use
+ *  the sub-categories.  They are just there to try and organize the MIME types
+ *  a little and open the door to future enhancements.
+ *
+ * Do _not_ add additional top-level categories unless you have added
+ *  corresponding entries to gloda.properties under the
+ *  "gloda.mimetype.category" branch and are making sure localizers are aware
+ *  of the change and have time to localize it.
+ *
+ * Entries with wildcards in them are part of a fallback strategy by the
+ *  |mimeTypeNoun| and do not actually use regular expressions or anything like
+ *  that.  Everything is a straight string lookup.  Given "foo/bar" we look for
+ *  "foo/bar", then "foo/*", and finally "*".
+ */
+var MimeCategoryMapping = {
+  archives: [
+    "application/java-archive",
+    "application/x-java-archive",
+    "application/x-jar",
+    "application/x-java-jnlp-file",
+
+    "application/mac-binhex40",
+    "application/vnd.ms-cab-compressed",
+
+    "application/x-arc",
+    "application/x-arj",
+    "application/x-compress",
+    "application/x-compressed-tar",
+    "application/x-cpio",
+    "application/x-cpio-compressed",
+    "application/x-deb",
+
+    "application/x-bittorrent",
+
+    "application/x-rar",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/zip",
+    "application/x-zip-compressed",
+    "application/x-zip",
+
+    "application/x-bzip",
+    "application/x-bzip-compressed-tar",
+    "application/x-bzip2",
+    "application/x-gzip",
+    "application/x-tar",
+    "application/x-tar-gz",
+    "application/x-tarz",
+  ],
+  documents: {
+    database: [
+      "application/vnd.ms-access",
+      "application/x-msaccess",
+      "application/msaccess",
+      "application/vnd.msaccess",
+      "application/x-msaccess",
+      "application/mdb",
+      "application/x-mdb",
+
+      "application/vnd.oasis.opendocument.database",
+
+    ],
+    graphics: [
+      "application/postscript",
+      "application/x-bzpostscript",
+      "application/x-dvi",
+      "application/x-gzdvi",
+
+      "application/illustrator",
+
+      "application/vnd.corel-draw",
+      "application/cdr",
+      "application/coreldraw",
+      "application/x-cdr",
+      "application/x-coreldraw",
+      "image/cdr",
+      "image/x-cdr",
+      "zz-application/zz-winassoc-cdr",
+
+      "application/vnd.oasis.opendocument.graphics",
+      "application/vnd.oasis.opendocument.graphics-template",
+      "application/vnd.oasis.opendocument.image",
+
+      "application/x-dia-diagram",
+    ],
+    presentation: [
+      "application/vnd.ms-powerpoint.presentation.macroenabled.12",
+      "application/vnd.ms-powerpoint.template.macroenabled.12",
+      "application/vnd.ms-powerpoint",
+      "application/powerpoint",
+      "application/mspowerpoint",
+      "application/x-mspowerpoint",
+      "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      "application/vnd.openxmlformats-officedocument.presentationml.template",
+
+      "application/vnd.oasis.opendocument.presentation",
+      "application/vnd.oasis.opendocument.presentation-template"
+    ],
+    spreadsheet: [
+      "application/vnd.lotus-1-2-3",
+      "application/x-lotus123",
+      "application/x-123",
+      "application/lotus123",
+      "application/wk1",
+
+      "application/x-quattropro",
+
+      "application/vnd.ms-excel.sheet.binary.macroenabled.12",
+      "application/vnd.ms-excel.sheet.macroenabled.12",
+      "application/vnd.ms-excel.template.macroenabled.12",
+      "application/vnd.ms-excel",
+      "application/msexcel",
+      "application/x-msexcel",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+
+      "application/vnd.oasis.opendocument.formula",
+      "application/vnd.oasis.opendocument.formula-template",
+      "application/vnd.oasis.opendocument.chart",
+      "application/vnd.oasis.opendocument.chart-template",
+      "application/vnd.oasis.opendocument.spreadsheet",
+      "application/vnd.oasis.opendocument.spreadsheet-template",
+
+      "application/x-gnumeric",
+    ],
+    wordProcessor: [
+      "application/msword",
+      "application/vnd.ms-word",
+      "application/x-msword",
+      "application/msword-template",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+      "application/vnd.ms-word.document.macroenabled.12",
+      "application/vnd.ms-word.template.macroenabled.12",
+      "application/x-mswrite",
+      "application/x-pocket-word",
+
+      "application/rtf",
+      "text/rtf",
+
+
+      "application/vnd.oasis.opendocument.text",
+      "application/vnd.oasis.opendocument.text-master",
+      "application/vnd.oasis.opendocument.text-template",
+      "application/vnd.oasis.opendocument.text-web",
+
+      "application/vnd.wordperfect",
+
+      "application/x-abiword",
+      "application/x-amipro",
+    ],
+    suite: [
+      "application/vnd.ms-works"
+    ],
+  },
+  images: [
+    "image/*"
+  ],
+  media: {
+    audio: [
+      "audio/*",
+    ],
+    video: [
+      "video/*",
+    ],
+    container: [
+      "application/ogg",
+
+      "application/smil",
+      "application/vnd.ms-asf",
+      "application/vnd.rn-realmedia",
+      "application/x-matroska",
+      "application/x-quicktime-media-link",
+      "application/x-quicktimeplayer",
+    ]
+  },
+  other: [
+    "*"
+  ],
+  pdf: [
+    "application/pdf",
+    "application/x-pdf",
+    "image/pdf",
+    "file/pdf",
+
+    "application/x-bzpdf",
+    "application/x-gzpdf",
+  ],
+}

mailnews/db/gloda/modules/mimemsg.js

@@ -0,0 +1,719 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['MsgHdrToMimeMessage',
+                          'MimeMessage', 'MimeContainer',
+                          'MimeBody', 'MimeUnknown',
+                          'MimeMessageAttachment'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var EMITTER_MIME_CODE = "application/x-js-mime-message";
+
+/**
+ * The URL listener is surplus because the CallbackStreamListener ends up
+ *  getting the same set of events, effectively.
+ */
+var dumbUrlListener = {
+  OnStartRunningUrl: function (aUrl) {
+  },
+  OnStopRunningUrl: function (aUrl, aExitCode) {
+  },
+};
+
+/**
+ * Maintain a list of all active stream listeners so that we can cancel them all
+ *  during shutdown.  If we don't cancel them, we risk calls into javascript
+ *  from C++ after the various XPConnect contexts have already begun their
+ *  teardown process.
+ */
+var activeStreamListeners = {};
+
+var shutdownCleanupObserver = {
+  _initialized: false,
+  ensureInitialized: function mimemsg_shutdownCleanupObserver_init() {
+    if (this._initialized)
+      return;
+
+    Services.obs.addObserver(this, "quit-application", false);
+
+    this._initialized = true;
+  },
+
+  observe: function mimemsg_shutdownCleanupObserver_observe(
+      aSubject, aTopic, aData) {
+    if (aTopic == "quit-application") {
+      Services.obs.removeObserver(this, "quit-application");
+
+      for (let uri in activeStreamListeners) {
+        let streamListener = activeStreamListeners[uri];
+        if (streamListener._request)
+          streamListener._request.cancel(Cr.NS_BINDING_ABORTED);
+      }
+    }
+  }
+};
+
+function CallbackStreamListener(aMsgHdr, aCallbackThis, aCallback) {
+  this._msgHdr = aMsgHdr;
+  let hdrURI = aMsgHdr.folder.getUriForMsg(aMsgHdr);
+  this._request = null;
+  this._stream = null;
+  if (aCallback === undefined) {
+    this._callbacksThis = [null];
+    this._callbacks = [aCallbackThis];
+  }
+  else {
+    this._callbacksThis = [aCallbackThis];
+    this._callbacks =[aCallback];
+  }
+  activeStreamListeners[hdrURI] = this;
+}
+
+CallbackStreamListener.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIStreamListener]),
+
+  // nsIRequestObserver part
+  onStartRequest: function (aRequest, aContext) {
+    this._request = aRequest;
+  },
+  onStopRequest: function (aRequest, aContext, aStatusCode) {
+    let msgURI = this._msgHdr.folder.getUriForMsg(this._msgHdr);
+    delete activeStreamListeners[msgURI];
+
+    aContext.QueryInterface(Ci.nsIURI);
+    let message = MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aContext.spec];
+    if (message === undefined)
+      message = null;
+
+    delete MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aContext.spec];
+
+    for (let i = 0; i < this._callbacksThis.length; i++) {
+      try {
+        this._callbacks[i].call(this._callbacksThis[i], this._msgHdr, message);
+      } catch (e) {
+        // Most of the time, exceptions will silently disappear into the endless
+        // deeps of XPConnect, and never reach the surface ever again. At least
+        // warn the user if he has dump enabled.
+        dump("The MsgHdrToMimeMessage callback threw an exception: "+e+"\n");
+        // That one will probably never make it to the original caller.
+        throw(e);
+      }
+    }
+
+    this._msgHdr = null;
+    this._request = null;
+    this._stream = null;
+    this._callbacksThis = null;
+    this._callbacks = null;
+  },
+
+  /* okay, our onDataAvailable should actually never be called.  the stream
+     converter is actually eating everything except the start and stop
+     notification. */
+  // nsIStreamListener part
+  onDataAvailable: function (aRequest,aContext,aInputStream,aOffset,aCount) {
+    dump("this should not be happening! arrgggggh!\n");
+    if (this._stream === null) {
+      this._stream = Cc["@mozilla.org/scriptableinputstream;1"].
+                    createInstance(Ci.nsIScriptableInputStream);
+      this._stream.init(aInputStream);
+    }
+    this._stream.read(aCount);
+
+  },
+};
+
+var gMessenger = Cc["@mozilla.org/messenger;1"].
+                   createInstance(Ci.nsIMessenger);
+
+function stripEncryptedParts(aPart) {
+  if (aPart.parts && aPart.isEncrypted) {
+    aPart.parts = []; // Show an empty container.
+  } else if (aPart.parts) {
+    aPart.parts = aPart.parts.map(stripEncryptedParts);
+  }
+  return aPart;
+}
+
+/**
+ * Starts retrieval of a MimeMessage instance for the given message header.
+ *  Your callback will be called with the message header you provide and the
+ *
+ * @param aMsgHdr The message header to retrieve the body for and build a MIME
+ *     representation of the message.
+ * @param aCallbackThis The (optional) 'this' to use for your callback function.
+ * @param aCallback The callback function to invoke on completion of message
+ *     parsing or failure.  The first argument passed will be the nsIMsgDBHdr
+ *     you passed to this function.  The second argument will be the MimeMessage
+ *     instance resulting from the processing on success, and null on failure.
+ * @param [aAllowDownload=false] Should we allow the message to be downloaded
+ *     for this streaming request?  The default is false, which means that we
+ *     require that the message be available offline.  If false is passed and
+ *     the message is not available offline, we will propagate an exception
+ *     thrown by the underlying code.
+ * @param [aOptions] Optional options.
+ * @param [aOptions.saneBodySize] Limit body sizes to a 'reasonable' size in
+ *     order to combat corrupt offline/message stores creating pathological
+ *     situtations where we have erroneously multi-megabyte messages.  This
+ *     also likely reduces the impact of legitimately ridiculously large
+ *     messages.
+ * @param [aOptions.partsOnDemand] If this is a message stored on an IMAP
+ *     server, and for whatever reason, it isn't available locally, then setting
+ *     this option to true will make sure that attachments aren't downloaded.
+ *     This makes sure the message is available quickly.
+ * @param [aOptions.examineEncryptedParts] By default, we won't reveal the
+ *     contents of multipart/encrypted parts to the consumers, unless explicitly
+ *     requested. In the case of MIME/PGP messages, for instance, the message
+ *     will appear as an empty multipart/encrypted container, unless this option
+ *     is used.
+ */
+function MsgHdrToMimeMessage(aMsgHdr, aCallbackThis, aCallback,
+                             aAllowDownload, aOptions) {
+  shutdownCleanupObserver.ensureInitialized();
+
+  let requireOffline = !aAllowDownload;
+
+  let msgURI = aMsgHdr.folder.getUriForMsg(aMsgHdr);
+  let msgService = gMessenger.messageServiceFromURI(msgURI);
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = aOptions;
+  let partsOnDemandStr = (aOptions && aOptions.partsOnDemand)
+    ? "&fetchCompleteMessage=false"
+    : "";
+  // By default, Enigmail only decrypts a message streamed via libmime if it's
+  // the one currently on display in the message reader. With this option, we're
+  // letting Enigmail know that it should decrypt the message since the client
+  // explicitly asked for it.
+  let encryptedStr = (aOptions && aOptions.examineEncryptedParts)
+    ? "&examineEncryptedParts=true"
+    : "";
+
+  // S/MIME, our other encryption backend, is not that smart, and always
+  // decrypts data. In order to protect sensitive data (e.g. not index it in
+  // Gloda), unless the client asked for encrypted data, we pass to the client
+  // callback a stripped-down version of the MIME structure where encrypted
+  // parts have been removed.
+  let wrapCallback = function (aCallback, aCallbackThis) {
+    if (aOptions && aOptions.examineEncryptedParts)
+      return aCallback;
+    else
+      return ((aMsgHdr, aMimeMsg) =>
+        aCallback.call(aCallbackThis, aMsgHdr, stripEncryptedParts(aMimeMsg))
+      );
+  };
+
+  // Apparently there used to be an old syntax where the callback was the second
+  // argument...
+  let callback = aCallback ? aCallback : aCallbackThis;
+  let callbackThis = aCallback ? aCallbackThis : null;
+
+  // if we're already streaming this msg, just add the callback
+  // to the listener.
+  let listenerForURI = activeStreamListeners[msgURI];
+  if (listenerForURI != undefined) {
+    listenerForURI._callbacks.push(wrapCallback(callback, callbackThis));
+    listenerForURI._callbacksThis.push(callbackThis);
+    return;
+  }
+  let streamListener = new CallbackStreamListener(
+    aMsgHdr,
+    callbackThis,
+    wrapCallback(callback, callbackThis)
+  );
+
+  try {
+    let streamURI = msgService.streamMessage(
+      msgURI,
+      streamListener, // consumer
+      null, // nsIMsgWindow
+      dumbUrlListener, // nsIUrlListener
+      true, // have them create the converter
+      // additional uri payload, note that "header=" is prepended automatically
+      "filter&emitter=js"+partsOnDemandStr+encryptedStr,
+      requireOffline);
+  } catch (ex) {
+    // If streamMessage throws an exception, we should make sure to clear the
+    // activeStreamListener, or any subsequent attempt at sreaming this URI
+    // will silently fail
+    if (activeStreamListeners[msgURI]) {
+      delete activeStreamListeners[msgURI];
+    }
+    MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+    throw(ex);
+  }
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+}
+
+/**
+ * Let the jsmimeemitter provide us with results.  The poor emitter (if I am
+ *  understanding things correctly) is evaluated outside of the C.u.import
+ *  world, so if we were to import him, we would not see him, but rather a new
+ *  copy of him.  This goes for his globals, etc.  (and is why we live in this
+ *  file right here).  Also, it appears that the XPCOM JS wrappers aren't
+ *  magically unified so that we can try and pass data as expando properties
+ *  on things like the nsIUri instances either.  So we have the jsmimeemitter
+ *  import us and poke things into RESULT_RENDEVOUZ.  We put it here on this
+ *  function to try and be stealthy and avoid polluting the namespaces (or
+ *  encouraging bad behaviour) of our importers.
+ *
+ * If you can come up with a prettier way to shuttle this data, please do.
+ */
+MsgHdrToMimeMessage.RESULT_RENDEVOUZ = {};
+/**
+ * Cram rich options here for the MimeMessageEmitter to grab from.  We
+ *  leverage the known control-flow to avoid needing a whole dictionary here.
+ *  We set this immediately before constructing the emitter and clear it
+ *  afterwards.  Control flow is never yielded during the process and reentrancy
+ *  cannot happen via any other means.
+ */
+MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+
+var HeaderHandlerBase = {
+  /**
+   * Look-up a header that should be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @param aDefaultValue The value to return if the header was not found, null
+   *     if left unspecified.
+   * @return the value of the header if present, and the default value if not
+   *  (defaults to null).  If the header was present multiple times, the first
+   *  instance of the header is returned.  Use getAll if you want all of the
+   *  values for the multiply-defined header.
+   */
+  get: function MimeMessage_get(aHeaderName, aDefaultValue) {
+    if (aDefaultValue === undefined) {
+      aDefaultValue = null;
+    }
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers)
+      // we require that the list cannot be empty if present
+      return this.headers[lowerHeader][0];
+    else
+      return aDefaultValue;
+  },
+  /**
+   * Look-up a header that can be present multiple times.  Use get for headers
+   *  that you only expect to be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @return An array containing the values observed, which may mean a zero
+   *     length array.
+   */
+  getAll: function MimeMessage_getAll(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers)
+      return this.headers[lowerHeader];
+    else
+      return [];
+  },
+  /**
+   * @param aHeaderName Header name to test for its presence.
+   * @return true if the message has (at least one value for) the given header
+   *     name.
+   */
+  has: function MimeMessage_has(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    return lowerHeader in this.headers;
+  },
+  _prettyHeaderString: function MimeMessage__prettyHeaderString(aIndent) {
+    if (aIndent === undefined)
+      aIndent = "";
+    let s = "";
+    for (let header in this.headers) {
+      let values = this.headers[header];
+      s += "\n        " + aIndent + header + ": " + values;
+    }
+    return s;
+  }
+};
+
+/**
+ * @ivar partName The MIME part, ex "1.2.2.1".  The partName of a (top-level)
+ *     message is "1", its first child is "1.1", its second child is "1.2",
+ *     its first child's first child is "1.1.1", etc.
+ * @ivar headers Maps lower-cased header field names to a list of the values
+ *     seen for the given header.  Use get or getAll as convenience helpers.
+ * @ivar parts The list of the MIME part children of this message.  Children
+ *     will be either MimeMessage instances, MimeMessageAttachment instances,
+ *     MimeContainer instances, or MimeUnknown instances.  The latter two are
+ *     the result of limitations in the Javascript representation generation
+ *     at this time, combined with the need to most accurately represent the
+ *     MIME structure.
+ */
+function MimeMessage() {
+  this.partName = null;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeMessage.prototype = {
+  __proto__: HeaderHandlerBase,
+  contentType: "message/rfc822",
+
+  /**
+   * @return a list of all attachments contained in this message and all its
+   *     sub-messages.  Only MimeMessageAttachment instances will be present in
+   *     the list (no sub-messages).
+   */
+  get allAttachments() {
+    let results = []; // messages are not attachments, don't include self
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+
+  /**
+   * @return a list of all attachments contained in this message, with
+   *    included/forwarded messages treated as real attachments. Attachments
+   *    contained in inner messages won't be shown.
+   */
+  get allUserAttachments() {
+    if (this.url)
+      // The jsmimeemitter camouflaged us as a MimeAttachment
+      return [this];
+    else
+      // Why is there no flatten method for arrays?
+      return this.parts.map(child => child.allUserAttachments)
+        .reduce((a, b) => a.concat(b), []);
+  },
+
+  /**
+   * @return the total size of this message, that is, the size of all subparts
+   */
+  get size () {
+    return this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+
+  /**
+   * In the case of attached messages, libmime considers them as attachments,
+   * and if the body is, say, quoted-printable encoded, then libmime will start
+   * counting bytes and notify the js mime emitter about it. The JS mime emitter
+   * being a nice guy, it will try to set a size on us. While this is the
+   * expected behavior for MimeMsgAttachments, we must make sure we can handle
+   * that (failing to write a setter results in exceptions being thrown).
+   */
+  set size (whatever) {
+    // nop
+  },
+
+  /**
+   * @param aMsgFolder A message folder, any message folder.  Because this is
+   *    a hack.
+   * @return The concatenation of all of the body parts where parts
+   *    available as text/plain are pulled as-is, and parts only available
+   *    as text/html are converted to plaintext form first.  In other words,
+   *    if we see a multipart/alternative with a text/plain, we take the
+   *    text/plain.  If we see a text/html without an alternative, we convert
+   *    that to text.
+   */
+  coerceBodyToPlaintext:
+      function MimeMessage_coerceBodyToPlaintext(aMsgFolder) {
+    let bodies = [];
+    for (let part of this.parts) {
+      // an undefined value for something not having the method is fine
+      let body = part.coerceBodyToPlaintext &&
+                 part.coerceBodyToPlaintext(aMsgFolder);
+      if (body)
+        bodies.push(body);
+    }
+    if (bodies)
+      return bodies.join("");
+    else
+      return "";
+  },
+
+  /**
+   * Convert the message and its hierarchy into a "pretty string".  The message
+   *  and each MIME part get their own line.  The string never ends with a
+   *  newline.  For a non-multi-part message, only a single line will be
+   *  returned.
+   * Messages have their subject displayed, attachments have their filename and
+   *  content-type (ex: image/jpeg) displayed.  "Filler" classes simply have
+   *  their class displayed.
+   */
+  prettyString: function MimeMessage_prettyString(aVerbose, aIndent,
+                                                  aDumpBody) {
+    if (aIndent === undefined)
+      aIndent = "";
+    let nextIndent = aIndent + "  ";
+
+    let s = "Message "+(this.isEncrypted ? "[encrypted] " : "") +
+      "(" + this.size + " bytes): " +
+      "subject" in this.headers ? this.headers.subject : "";
+    if (aVerbose)
+      s += this._prettyHeaderString(nextIndent);
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+};
+
+
+/**
+ * @ivar contentType The content-type of this container.
+ * @ivar parts The parts held by this container.  These can be instances of any
+ *      of the classes found in this file.
+ */
+function MimeContainer(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeContainer.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    let results = [];
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+  get allUserAttachments () {
+    return this.parts.map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size () {
+    return this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+  set size (whatever) {
+    // nop
+  },
+  coerceBodyToPlaintext:
+      function MimeContainer_coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "multipart/alternative") {
+      let htmlPart;
+      // pick the text/plain if we can find one, otherwise remember the HTML one
+      for (let part of this.parts) {
+        if (part.contentType == "text/plain")
+          return part.body;
+        if (part.contentType == "text/html")
+          htmlPart = part;
+        // text/enriched gets transformed into HTML, use it if we don't already
+        //  have an HTML part.
+        else if (!htmlPart && part.contentType == "text/enriched")
+	  htmlPart = part;
+      }
+      // convert the HTML part if we have one
+      if (htmlPart)
+        return aMsgFolder.convertMsgSnippetToPlainText(htmlPart.body);
+    }
+    // if it's not alternative, recurse/aggregate using MimeMessage logic
+    return MimeMessage.prototype.coerceBodyToPlaintext.call(this, aMsgFolder);
+  },
+  prettyString: function MimeContainer_prettyString(aVerbose, aIndent,
+                                                    aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s = "Container "+(this.isEncrypted ? "[encrypted] " : "")+
+      "(" + this.size + " bytes): " + this.contentType;
+    if (aVerbose)
+      s += this._prettyHeaderString(nextIndent);
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+  toString: function MimeContainer_toString() {
+    return "Container: " + this.contentType;
+  }
+};
+
+/**
+ * @class Represents a body portion that we understand and do not believe to be
+ *  a proper attachment.  This means text/plain or text/html and it has no
+ *  filename.  (A filename suggests an attachment.)
+ *
+ * @ivar contentType The content type of this body materal; text/plain or
+ *     text/html.
+ * @ivar body The actual body content.
+ */
+function MimeBody(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.body = "";
+  this.isEncrypted = false;
+}
+
+MimeBody.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return []; // we are a leaf
+  },
+  get allUserAttachments() {
+    return []; // we are a leaf
+  },
+  get size() {
+    return this.body.length;
+  },
+  set size (whatever) {
+    // nop
+  },
+  appendBody: function MimeBody_append(aBuf) {
+    this.body += aBuf;
+  },
+  coerceBodyToPlaintext:
+      function MimeBody_coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "text/plain")
+      return this.body;
+    // text/enriched gets transformed into HTML by libmime
+    if (this.contentType == "text/html" ||
+        this.contentType == "text/enriched")
+      return aMsgFolder.convertMsgSnippetToPlainText(this.body);
+    return "";
+  },
+  prettyString: function MimeBody_prettyString(aVerbose, aIndent, aDumpBody) {
+    let s = "Body: "+(this.isEncrypted ? "[encrypted] " : "")+
+      "" + this.contentType + " (" + this.body.length + " bytes" +
+      (aDumpBody ? (": '" + this.body + "'") : "") + ")";
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+    return s;
+  },
+  toString: function MimeBody_toString() {
+    return "Body: " + this.contentType + " (" + this.body.length + " bytes)";
+  }
+};
+
+/**
+ * @class A MIME Leaf node that doesn't have a filename so we assume it's not
+ *  intended to be an attachment proper.  This is probably meant for inline
+ *  display or is the result of someone amusing themselves by composing messages
+ *  by hand or a bad client.  This class should probably be renamed or we should
+ *  introduce a better named class that we try and use in preference to this
+ *  class.
+ *
+ * @ivar contentType The content type of this part.
+ */
+function MimeUnknown(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  // Looks like libmime does not always intepret us as an attachment, which
+  //  means we'll have to have a default size. Returning undefined would cause
+  //  the recursive size computations to fail.
+  this._size = 0;
+  this.isEncrypted = false;
+  // We want to make sure MimeUnknown has a part property: S/MIME encrypted
+  // messages have a topmost MimeUnknown part, with the encrypted bit set to 1,
+  // and we need to ensure all other encrypted parts are children of this
+  // topmost part.
+  this.parts = [];
+}
+
+MimeUnknown.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return this.parts.map(child => child.allAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get allUserAttachments() {
+    return this.parts.map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size() {
+    return this._size + this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+  set size(aSize) {
+    this._size = aSize;
+  },
+  prettyString: function MimeUnknown_prettyString(aVerbose, aIndent,
+                                                  aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s = "Unknown: "+(this.isEncrypted ? "[encrypted] " : "")+
+      "" + this.contentType + " (" + this.size + " bytes)";
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        (part ? part.prettyString(aVerbose, nextIndent, aDumpBody) : "NULL");
+    }
+    return s;
+  },
+  toString: function MimeUnknown_toString() {
+    return "Unknown: " + this.contentType;
+  }
+};
+
+/**
+ * @class An attachment proper.  We think it's an attachment because it has a
+ *  filename that libmime was able to figure out.
+ *
+ * @ivar partName @see{MimeMessage.partName}
+ * @ivar name The filename of this attachment.
+ * @ivar contentType The MIME content type of this part.
+ * @ivar url The URL to stream if you want the contents of this part.
+ * @ivar isExternal Is the attachment stored someplace else than in the message?
+ * @ivar size The size of the attachment if available, -1 otherwise (size is set
+ *  after initialization by jsmimeemitter.js)
+ */
+function MimeMessageAttachment(aPartName, aName, aContentType, aUrl,
+                               aIsExternal) {
+  this.partName = aPartName;
+  this.name = aName;
+  this.contentType = aContentType;
+  this.url = aUrl;
+  this.isExternal = aIsExternal;
+  this.headers = {};
+  this.isEncrypted = false;
+  // parts is copied over from the part instance that preceded us
+  // headers is copied over from the part instance that preceded us
+  // isEncrypted is copied over from the part instance that preceded us
+}
+
+MimeMessageAttachment.prototype = {
+  __proto__: HeaderHandlerBase,
+  // This is a legacy property.
+  get isRealAttachment() {
+    return true;
+  },
+  get allAttachments() {
+    return [this]; // we are a leaf, so just us.
+  },
+  get allUserAttachments() {
+    return [this];
+  },
+  prettyString: function MimeMessageAttachment_prettyString(aVerbose, aIndent,
+                                                            aDumpBody) {
+    let s = "Attachment "+(this.isEncrypted ? "[encrypted] " : "")+
+      "(" + this.size+" bytes): "
+      + this.name + ", " + this.contentType;
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+    return s;
+  },
+  toString: function MimeMessageAttachment_toString() {
+    return this.prettyString(false, "");
+  },
+};

mailnews/db/gloda/modules/moz.build

@@ -0,0 +1,32 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXTRA_JS_MODULES.gloda += [
+    'collection.js',
+    'connotent.js',
+    'databind.js',
+    'datamodel.js',
+    'datastore.js',
+    'dbview.js',
+    'everybody.js',
+    'explattr.js',
+    'facet.js',
+    'fundattr.js',
+    'gloda.js',
+    'index_ab.js',
+    'index_msg.js',
+    'indexer.js',
+    'log4moz.js',
+    'mimemsg.js',
+    'mimeTypeCategories.js',
+    'msg_search.js',
+    'noun_freetag.js',
+    'noun_mimetype.js',
+    'noun_tag.js',
+    'public.js',
+    'query.js',
+    'suffixtree.js',
+    'utils.js',
+]

mailnews/db/gloda/modules/msg_search.js

@@ -0,0 +1,346 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaMsgSearcher"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource:///modules/gloda/public.js");
+
+/**
+ * How much time boost should a 'score point' amount to?  The authoritative,
+ *  incontrivertible answer, across all time and space, is a week.
+ *  Note that gloda stores timestamps as PRTimes for no exceedingly good
+ *  reason.
+ */
+var FUZZSCORE_TIMESTAMP_FACTOR = 1000 * 1000 * 60 * 60 * 24 * 7;
+
+var RANK_USAGE =
+  "glodaRank(matchinfo(messagesText), 1.0, 2.0, 2.0, 1.5, 1.5)";
+
+var DASCORE =
+  "(((" + RANK_USAGE + " + messages.notability) * " +
+    FUZZSCORE_TIMESTAMP_FACTOR +
+   ") + messages.date)";
+
+/**
+ * A new optimization decision we are making is that we do not want to carry
+ *  around any data in our ephemeral tables that is not used for whittling the
+ *  result set.  The idea is that the btree page cache or OS cache is going to
+ *  save us from the disk seeks and carrying around the extra data is just going
+ *  to be CPU/memory churn that slows us down.
+ *
+ * Additionally, we try and avoid row lookups that would have their results
+ *  discarded by the LIMIT.  Because of limitations in FTS3 (which might
+ *  be addressed in FTS4 by a feature request), we can't avoid the 'messages'
+ *  lookup since that has the message's date and static notability but we can
+ *  defer the 'messagesText' lookup.
+ *
+ * This is the access pattern we are after here:
+ * 1) Order the matches with minimized lookup and result storage costs.
+ * - The innermost MATCH does the doclist magic and provides us with
+ *    matchinfo() support which does not require content row retrieval
+ *    from messagesText.  Unfortunately, this is not enough to whittle anything
+ *    because we still need static interestingness, so...
+ * - Based on the match we retrieve the date and notability for that row from
+ *    'messages' using this in conjunction with matchinfo() to provide a score
+ *    that we can then use to LIMIT our results.
+ * 2) We reissue the MATCH query so that we will be able to use offsets(), but
+ *    we intersect the results of this MATCH against our LIMITed results from
+ *    step 1.
+ * - We use 'docid IN (phase 1 query)' to accomplish this because it results in
+ *    efficient lookup.  If we just use a join, we get O(mn) performance because
+ *    a cartesian join ends up being performed where either we end up performing
+ *    the fulltext query M times and table scan intersect with the results from
+ *    phase 1 or we do the fulltext once but traverse the entire result set from
+ *    phase 1 N times.
+ * - We believe that the re-execution of the MATCH query should have no disk
+ *    costs because it should still be cached by SQLite or the OS.  In the case
+ *    where memory is so constrained this is not true our behavior is still
+ *    probably preferable than the old way because that would have caused lots
+ *    of swapping.
+ * - This part of the query otherwise resembles the basic gloda query but with
+ *    the inclusion of the offsets() invocation.  The messages table lookup
+ *    should not involve any disk traffic because the pages should still be
+ *    cached (SQLite or OS) from phase 1.  The messagesText lookup is new, and
+ *    this is the major disk-seek reduction optimization we are making.  (Since
+ *    we avoid this lookup for all of the documents that were excluded by the
+ *    LIMIT.)  Since offsets() also needs to retrieve the row from messagesText
+ *    there is a nice synergy there.
+ */
+var NUEVO_FULLTEXT_SQL =
+  "SELECT messages.*, messagesText.*, offsets(messagesText) AS osets " +
+  "FROM messagesText, messages " +
+  "WHERE" +
+    " messagesText MATCH ?1 " +
+    " AND messagesText.docid IN (" +
+       "SELECT docid " +
+       "FROM messagesText JOIN messages ON messagesText.docid = messages.id " +
+       "WHERE messagesText MATCH ?1 " +
+       "ORDER BY " + DASCORE + " DESC " +
+       "LIMIT ?2" +
+    " )" +
+    " AND messages.id = messagesText.docid " +
+    " AND +messages.deleted = 0" +
+    " AND +messages.folderID IS NOT NULL" +
+    " AND +messages.messageKey IS NOT NULL";
+
+function identityFunc(x) {
+  return x;
+}
+
+function oneLessMaxZero(x) {
+  if (x <= 1)
+    return 0;
+  else
+    return x - 1;
+}
+
+function reduceSum(accum, curValue) {
+  return accum + curValue;
+}
+
+/*
+ * Columns are: body, subject, attachment names, author, recipients
+ */
+
+/**
+ * Scores if all search terms match in a column.  We bias against author
+ *  slightly and recipient a bit more in this case because a search that
+ *  entirely matches just on a person should give a mention of that person
+ *  in the subject or attachment a fighting chance.
+ * Keep in mind that because of our indexing in the face of address book
+ *  contacts (namely, we index the name used in the e-mail as well as the
+ *  display name on the address book card associated with the e-mail adress)
+ *  a contact is going to bias towards matching multiple times.
+ */
+var COLUMN_ALL_MATCH_SCORES = [4, 20, 20, 16, 12];
+/**
+ * Score for each distinct term that matches in the column.  This is capped
+ *  by COLUMN_ALL_SCORES.
+ */
+var COLUMN_PARTIAL_PER_MATCH_SCORES = [1, 4, 4, 4, 3];
+/**
+ * If a term matches multiple times, what is the marginal score for each
+ *  additional match.  We count the total number of matches beyond the
+ *  first match for each term.  In other words, if we have 3 terms which
+ *  matched 5, 3, and 0 times, then the total from our perspective is
+ *  (5 - 1) + (3 - 1) + 0 = 4 + 2 + 0 = 6.  We take the minimum of that value
+ *  and the value in COLUMN_MULTIPLE_MATCH_LIMIT and multiply by the value in
+ *  COLUMN_MULTIPLE_MATCH_SCORES.
+ */
+var COLUMN_MULTIPLE_MATCH_SCORES = [1, 0, 0, 0, 0];
+var COLUMN_MULTIPLE_MATCH_LIMIT = [10, 0, 0, 0, 0];
+
+/**
+ * Score the message on its offsets (from stashedColumns).
+ */
+function scoreOffsets(aMessage, aContext) {
+  let score = 0;
+
+  let termTemplate = aContext.terms.map(_ => 0);
+  // for each column, a list of the incidence of each term
+  let columnTermIncidence = [termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat()];
+
+  // we need a friendlyParseInt because otherwise the radix stuff happens
+  //  because of the extra arguments map parses.  curse you, map!
+  let offsetNums =
+    aContext.stashedColumns[aMessage.id][0].split(" ").map(x => parseInt(x));
+  for (let i=0; i < offsetNums.length; i += 4) {
+    let columnIndex = offsetNums[i];
+    let termIndex = offsetNums[i+1];
+    columnTermIncidence[columnIndex][termIndex]++;
+  }
+
+  for (let iColumn = 0; iColumn < COLUMN_ALL_MATCH_SCORES.length; iColumn++) {
+    let termIncidence = columnTermIncidence[iColumn];
+    // bestow all match credit
+    if (termIncidence.every(identityFunc))
+      score += COLUMN_ALL_MATCH_SCORES[iColumn];
+    // bestow partial match credit
+    else if (termIncidence.some(identityFunc))
+      score += Math.min(COLUMN_ALL_MATCH_SCORES[iColumn],
+                        COLUMN_PARTIAL_PER_MATCH_SCORES[iColumn] *
+                          termIncidence.filter(identityFunc).length);
+    // bestow multiple match credit
+    score += Math.min(termIncidence.map(oneLessMaxZero).reduce(reduceSum, 0),
+                      COLUMN_MULTIPLE_MATCH_LIMIT[iColumn]) *
+             COLUMN_MULTIPLE_MATCH_SCORES[iColumn];
+  }
+
+  return score;
+}
+
+/**
+ * The searcher basically looks like a query, but is specialized for fulltext
+ *  search against messages.  Most of the explicit specialization involves
+ *  crafting a SQL query that attempts to order the matches by likelihood that
+ *  the user was looking for it.  This is based on full-text matches combined
+ *  with an explicit (generic) interest score value placed on the message at
+ *  indexing time.  This is followed by using the more generic gloda scoring
+ *  mechanism to explicitly score the messages given the search context in
+ *  addition to the more generic score adjusting rules.
+ */
+function GlodaMsgSearcher(aListener, aSearchString, aAndTerms) {
+  this.listener = aListener;
+
+  this.searchString = aSearchString;
+  this.fulltextTerms = this.parseSearchString(aSearchString);
+  this.andTerms = (aAndTerms != null) ? aAndTerms : true;
+
+  this.query = null;
+  this.collection = null;
+
+  this.scores = null;
+}
+GlodaMsgSearcher.prototype = {
+  /**
+   * Number of messages to retrieve initially.
+   */
+  get retrievalLimit() {
+    return Services.prefs.getIntPref(
+      "mailnews.database.global.search.msg.limit"
+    );
+  },
+
+  /**
+   * Parse the string into terms/phrases by finding matching double-quotes.
+   */
+  parseSearchString: function GlodaMsgSearcher_parseSearchString(aSearchString) {
+    aSearchString = aSearchString.trim();
+    let terms = [];
+
+    /*
+     * Add the term as long as the trim on the way in didn't obliterate it.
+     *
+     * In the future this might have other helper logic; it did once before.
+     */
+    function addTerm(aTerm) {
+      if (aTerm)
+        terms.push(aTerm);
+    }
+
+    while (aSearchString) {
+      if (aSearchString.startsWith('"')) {
+        let endIndex = aSearchString.indexOf(aSearchString[0], 1);
+        // eat the quote if it has no friend
+        if (endIndex == -1) {
+          aSearchString = aSearchString.substring(1);
+          continue;
+        }
+
+        addTerm(aSearchString.substring(1, endIndex).trim());
+        aSearchString = aSearchString.substring(endIndex + 1);
+        continue;
+      }
+
+      let spaceIndex = aSearchString.indexOf(" ");
+      if (spaceIndex == -1) {
+        addTerm(aSearchString);
+        break;
+      }
+
+      addTerm(aSearchString.substring(0, spaceIndex));
+      aSearchString = aSearchString.substring(spaceIndex+1);
+    }
+
+    return terms;
+  },
+
+  buildFulltextQuery: function GlodaMsgSearcher_buildFulltextQuery() {
+    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
+      noMagic: true,
+      explicitSQL: NUEVO_FULLTEXT_SQL,
+      limitClauseAlreadyIncluded: true,
+      // osets is 0-based column number 14 (volatile to column changes)
+      // save the offset column for extra analysis
+      stashColumns: [14]
+    });
+
+    let fulltextQueryString = "";
+
+    for (let [iTerm, term] of this.fulltextTerms.entries()) {
+      if (iTerm)
+        fulltextQueryString += this.andTerms ? " " : " OR ";
+
+      // Put our term in quotes.  This is needed for the tokenizer to be able
+      //  to do useful things.  The exception is people clever enough to use
+      //  NEAR.
+      if (/^NEAR(\/\d+)?$/.test(term))
+        fulltextQueryString += term;
+      // Check if this is a single-character CJK search query.  If so, we want
+      //  to add a wildcard.
+      // Our tokenizer treats anything at/above 0x2000 as CJK for now.
+      else if (term.length == 1 && term.charCodeAt(0) >= 0x2000)
+        fulltextQueryString += term + "*";
+      else if (
+          term.length == 2 &&
+            term.charCodeAt(0) >= 0x2000 &&
+            term.charCodeAt(1) >= 0x2000
+          || term.length >= 3
+      )
+        fulltextQueryString += '"' + term + '"';
+
+    }
+
+    query.fulltextMatches(fulltextQueryString);
+    query.limit(this.retrievalLimit);
+
+    return query;
+  },
+
+  getCollection: function GlodaMsgSearcher_getCollection(
+      aListenerOverride, aData) {
+    if (aListenerOverride)
+      this.listener = aListenerOverride;
+
+    this.query = this.buildFulltextQuery();
+    this.collection = this.query.getCollection(this, aData);
+    this.completed = false;
+
+    return this.collection;
+  },
+
+  sortBy: '-dascore',
+
+  onItemsAdded: function GlodaMsgSearcher_onItemsAdded(aItems, aCollection) {
+    let newScores = Gloda.scoreNounItems(
+      aItems,
+      {
+        terms: this.fulltextTerms,
+        stashedColumns: aCollection.stashedColumns
+      },
+      [scoreOffsets]);
+    if (this.scores)
+      this.scores = this.scores.concat(newScores);
+    else
+      this.scores = newScores;
+
+    if (this.listener)
+      this.listener.onItemsAdded(aItems, aCollection);
+  },
+  onItemsModified: function GlodaMsgSearcher_onItemsModified(aItems,
+                                                             aCollection) {
+    if (this.listener)
+      this.listener.onItemsModified(aItems, aCollection);
+  },
+  onItemsRemoved: function GlodaMsgSearcher_onItemsRemoved(aItems,
+                                                           aCollection) {
+    if (this.listener)
+      this.listener.onItemsRemoved(aItems, aCollection);
+  },
+  onQueryCompleted: function GlodaMsgSearcher_onQueryCompleted(aCollection) {
+    this.completed = true;
+    if (this.listener)
+      this.listener.onQueryCompleted(aCollection);
+  },
+};

mailnews/db/gloda/modules/noun_freetag.js

@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['FreeTag', 'FreeTagNoun'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+function FreeTag(aTagName) {
+  this.name = aTagName;
+}
+
+FreeTag.prototype = {
+  toString: function () {
+    return this.name;
+  }
+};
+
+/**
+ * @namespace Tag noun provider.  Since the tag unique value is stored as a
+ *  parameter, we are an odd case and semantically confused.
+ */
+var FreeTagNoun = {
+  _log: Log4Moz.repository.getLogger("gloda.noun.freetag"),
+
+  name: "freetag",
+  clazz: FreeTag,
+  allowsArbitraryAttrs: false,
+  usesParameter: true,
+
+  _listeners: [],
+  addListener: function(aListener) {
+    this._listeners.push(aListener);
+  },
+  removeListener: function(aListener) {
+    let index = this._listeners.indexOf(aListener);
+    if (index >=0)
+      this._listeners.splice(index, 1);
+  },
+
+  populateKnownFreeTags: function() {
+    for (let attr of this.objectNounOfAttributes) {
+      let attrDB = attr.dbDef;
+      for (let param in attrDB.parameterBindings) {
+        this.getFreeTag(param);
+      }
+    }
+  },
+
+  knownFreeTags: {},
+  getFreeTag: function(aTagName) {
+    let tag = this.knownFreeTags[aTagName];
+    if (!tag) {
+      tag = this.knownFreeTags[aTagName] = new FreeTag(aTagName);
+      for (let listener of this._listeners)
+        listener.onFreeTagAdded(tag);
+    }
+    return tag;
+  },
+
+  comparator: function gloda_noun_freetag_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.name.localeCompare(b.name);
+  },
+
+  toParamAndValue: function gloda_noun_freetag_toParamAndValue(aTag) {
+    return [aTag.name, null];
+  },
+
+  toJSON: function gloda_noun_freetag_toJSON(aTag) {
+    return aTag.name;
+  },
+  fromJSON: function gloda_noun_freetag_fromJSON(aTagName) {
+    return this.getFreeTag(aTagName);
+  },
+};
+
+Gloda.defineNoun(FreeTagNoun);

mailnews/db/gloda/modules/noun_mimetype.js

@@ -0,0 +1,365 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['MimeType', 'MimeTypeNoun'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.noun.mimetype");
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+var CategoryStringMap = {};
+
+/**
+ * Mime type abstraction that exists primarily so we can map mime types to
+ *  integer id's.
+ *
+ * Instances of this class should only be retrieved via |MimeTypeNoun|; no one
+ *  should ever create an instance directly.
+ */
+function MimeType(aID, aType, aSubType, aFullType, aCategory) {
+  this._id = aID;
+  this._type = aType;
+  this._subType = aSubType;
+  this._fullType = aFullType;
+  this._category = aCategory;
+}
+
+MimeType.prototype = {
+  /**
+   * The integer id we have associated with the mime type.  This is stable for
+   *  the lifetime of the database, which means that anything in the Gloda
+   *  database can use this without fear.  Things not persisted in the database
+   *  should use the actual string mime type, retrieval via |fullType|.
+   */
+  get id() { return this._id; },
+  /**
+   * The first part of the MIME type; "text/plain" gets you "text".
+   */
+  get type() { return this._type; },
+  set fullType(aFullType) {
+    if (!this._fullType) {
+      this._fullType = aFullType;
+      [this._type, this._subType] = this._fullType.split("/");
+      this._category =
+        MimeTypeNoun._getCategoryForMimeType(aFullType, this._type);
+    }
+  },
+  /**
+   * If the |fullType| is "text/plain", subType is "plain".
+   */
+  get subType() { return this._subType; },
+  /**
+   * The full MIME type; "text/plain" returns "text/plain".
+   */
+  get fullType() { return this._fullType; },
+  toString: function () {
+    return this.fullType;
+  },
+
+  /**
+   * @return the category we believe this mime type belongs to.  This category
+   *     name should never be shown directly to the user.  Instead, use
+   *     |categoryLabel| to get the localized name for the category.  The
+   *     category mapping comes from mimeTypesCategories.js.
+   */
+  get category() {
+    return this._category;
+  },
+  /**
+   * @return The localized label for the category from gloda.properties in the
+   *     "gloda.mimetype.category.CATEGORY.label" definition using the value
+   *     from |category|.
+   */
+  get categoryLabel() {
+    return CategoryStringMap[this._category];
+  }
+};
+
+/**
+ * Mime type noun provider.
+ *
+ * The set of MIME Types is sufficiently limited that we can keep them all in
+ *  memory.  In theory it is also sufficiently limited that we could use the
+ *  parameter mechanism in the database.  However, it is more efficient, for
+ *  both space and performance reasons, to store the specific mime type as a
+ *  value.  For future-proofing reasons, we opt to use a database table to
+ *  persist the mapping rather than a hard-coded list.  A preferences file or
+ *  other text file would arguably suffice, but for consistency reasons, the
+ *  database is not a bad thing.
+ */
+var MimeTypeNoun = {
+  name: "mime-type",
+  clazz: MimeType, // gloda supports clazz as well as class
+  allowsArbitraryAttrs: false,
+
+  _strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+
+  // note! update test_noun_mimetype if you change our internals!
+  _mimeTypes: {},
+  _mimeTypesByID: {},
+  TYPE_BLOCK_SIZE: 16384,
+  _mimeTypeHighID: {},
+  _mimeTypeRangeDummyObjects: {},
+  _highID: 0,
+
+  // we now use the exciting 'schema' mechanism of defineNoun to get our table
+  //  created for us, plus some helper methods that we simply don't use.
+  schema: {
+    name: 'mimeTypes',
+    columns: [['id', 'INTEGER PRIMARY KEY', '_id'],
+              ['mimeType', 'TEXT', 'fullType']],
+  },
+
+  _init: function() {
+    LOG.debug("loading MIME types");
+    this._loadCategoryMapping();
+    this._loadMimeTypes();
+  },
+
+  /**
+   * A map from MIME type to category name.
+   */
+  _mimeTypeToCategory: {},
+  /**
+   * Load the contents of mimeTypeCategories.js and populate
+   */
+  _loadCategoryMapping: function MimeTypeNoun__loadCategoryMapping() {
+    let mimecatNS = {};
+    Cu.import("resource:///modules/gloda/mimeTypeCategories.js",
+              mimecatNS);
+    let mcm = mimecatNS.MimeCategoryMapping;
+
+    let mimeTypeToCategory = this._mimeTypeToCategory;
+
+    function procMapObj(aSubTree, aCategories) {
+      for (let key in aSubTree) {
+        let value = aSubTree[key];
+        // Add this category to our nested categories list.  Use concat since
+        //  the list will be long-lived and each list needs to be distinct.
+        let categories = aCategories.concat();
+        categories.push(key);
+
+        if (categories.length == 1) {
+          CategoryStringMap[key] =
+            MimeTypeNoun._strings.get(
+              "gloda.mimetype.category." + key + ".label");
+        }
+
+        // Is it an array? If so, just process this depth
+        if (Array.isArray(value)) {
+          for (let mimeTypeStr of value) {
+            mimeTypeToCategory[mimeTypeStr] = categories;
+          }
+        }
+        // it's yet another sub-tree branch
+        else {
+          procMapObj(value, categories);
+        }
+      }
+    }
+
+    procMapObj(mimecatNS.MimeCategoryMapping, []);
+  },
+
+  /**
+   * Lookup the category associated with a MIME type given its full type and
+   *  type.  (So, "foo/bar" and "foo" for "foo/bar".)
+   */
+  _getCategoryForMimeType:
+      function MimeTypeNoun__getCategoryForMimeType(aFullType, aType) {
+    if (aFullType in this._mimeTypeToCategory)
+      return this._mimeTypeToCategory[aFullType][0];
+    let wildType = aType + "/*";
+    if (wildType in this._mimeTypeToCategory)
+      return this._mimeTypeToCategory[wildType][0];
+    return this._mimeTypeToCategory["*"][0];
+  },
+
+  /**
+   * In order to allow the gloda query mechanism to avoid hitting the database,
+   *  we need to either define the noun type as cachable and have a super-large
+   *  cache or simply have a collection with every MIME type in it that stays
+   *  alive forever.
+   * This is that collection.  It is initialized by |_loadMimeTypes|.  As new
+   *  MIME types are created, we add them to the collection.
+   */
+  _universalCollection: null,
+
+  /**
+   * Kick off a query of all the mime types in our database, leaving
+   *  |_processMimeTypes| to actually do the legwork.
+   */
+  _loadMimeTypes: function MimeTypeNoun__loadMimeTypes() {
+    // get all the existing mime types!
+    let query = Gloda.newQuery(this.id);
+    let nullFunc = function() {};
+    this._universalCollection = query.getCollection({
+      onItemsAdded: nullFunc, onItemsModified: nullFunc,
+      onItemsRemoved: nullFunc,
+      onQueryCompleted: function (aCollection) {
+        MimeTypeNoun._processMimeTypes(aCollection.items);
+      }
+    }, null);
+  },
+
+  /**
+   * For the benefit of our Category queryHelper, we need dummy ranged objects
+   *  that cover the numerical address space allocated to the category.  We
+   *  can't use a real object for the upper-bound because the upper-bound is
+   *  constantly growing and there is the chance the query might get persisted,
+   *  which means these values need to be long-lived.  Unfortunately, our
+   *  solution to this problem (dummy objects) complicates the second case,
+   *  should it ever occur.  (Because the dummy objects cannot be persisted
+   *  on their own... but there are other issues that will come up that we will
+   *  just have to deal with then.)
+   */
+  _createCategoryDummies: function (aId, aCategory) {
+    let blockBottom = aId - (aId % this.TYPE_BLOCK_SIZE);
+    let blockTop = blockBottom + this.TYPE_BLOCK_SIZE - 1;
+    this._mimeTypeRangeDummyObjects[aCategory] = [
+      new MimeType(blockBottom, "!category-dummy!", aCategory,
+                   "!category-dummy!/" + aCategory, aCategory),
+      new MimeType(blockTop, "!category-dummy!", aCategory,
+                   "!category-dummy!/" + aCategory, aCategory)
+    ];
+  },
+
+  _processMimeTypes: function MimeTypeNoun__processMimeTypes(aMimeTypes) {
+    for (let mimeType of aMimeTypes) {
+      if (mimeType.id > this._highID)
+        this._highID = mimeType.id;
+      this._mimeTypes[mimeType] = mimeType;
+      this._mimeTypesByID[mimeType.id] = mimeType;
+
+      let typeBlock = mimeType.id - (mimeType.id % this.TYPE_BLOCK_SIZE);
+      let blockHighID = (mimeType.category in this._mimeTypeHighID) ?
+                          this._mimeTypeHighID[mimeType.category] : undefined;
+      // create the dummy range objects
+      if (blockHighID === undefined)
+        this._createCategoryDummies(mimeType.id, mimeType.category);
+      if ((blockHighID === undefined) || mimeType.id > blockHighID)
+        this._mimeTypeHighID[mimeType.category] = mimeType.id;
+    }
+  },
+
+  _addNewMimeType: function MimeTypeNoun__addNewMimeType(aMimeTypeName) {
+    let [typeName, subTypeName] = aMimeTypeName.split("/");
+    let category = this._getCategoryForMimeType(aMimeTypeName, typeName);
+
+    if (!(category in this._mimeTypeHighID)) {
+      let nextID = this._highID - (this._highID % this.TYPE_BLOCK_SIZE) +
+        this.TYPE_BLOCK_SIZE;
+      this._mimeTypeHighID[category] = nextID;
+      this._createCategoryDummies(nextID, category);
+    }
+
+    let nextID = ++this._mimeTypeHighID[category];
+
+    let mimeType = new MimeType(nextID, typeName, subTypeName, aMimeTypeName,
+                                category);
+    if (mimeType.id > this._highID)
+      this._highID = mimeType.id;
+
+    this._mimeTypes[aMimeTypeName] = mimeType;
+    this._mimeTypesByID[nextID] = mimeType;
+
+    // As great as the gloda extension mechanisms are, we don't think it makes
+    //  a lot of sense to use them in this case.  So we directly trigger object
+    //  insertion without any of the grokNounItem stuff.
+    this.objInsert.call(this.datastore, mimeType);
+    // Since we bypass grokNounItem and its fun, we need to explicitly add the
+    //  new MIME-type to _universalCollection ourselves.  Don't try this at
+    //  home, kids.
+    this._universalCollection._onItemsAdded([mimeType]);
+
+    return mimeType;
+  },
+
+  /**
+   * Map a mime type to a |MimeType| instance, creating it if necessary.
+   *
+   * @param aMimeTypeName The mime type.  It may optionally include parameters
+   *     (which will be ignored).  A mime type is of the form "type/subtype".
+   *     A type with parameters would look like 'type/subtype; param="value"'.
+   */
+  getMimeType: function MimeTypeNoun_getMimeType(aMimeTypeName) {
+    // first, lose any parameters
+    let semiIndex = aMimeTypeName.indexOf(";");
+    if (semiIndex >= 0)
+      aMimeTypeName = aMimeTypeName.substring(0, semiIndex);
+    aMimeTypeName = aMimeTypeName.trim().toLowerCase();
+
+    if (aMimeTypeName in this._mimeTypes)
+      return this._mimeTypes[aMimeTypeName];
+    else
+      return this._addNewMimeType(aMimeTypeName);
+  },
+
+  /**
+   * Query helpers contribute additional functions to the query object for the
+   *  attributes that use the noun type.  For example, we define Category, so
+   *  for the "attachmentTypes" attribute, "attachmentTypesCategory" would be
+   *  exposed.
+   */
+  queryHelpers: {
+    /**
+     * Query for MIME type categories based on one or more MIME type objects
+     *  passed in.  We want the range to span the entire block allocated to the
+     *  category.
+     *
+     * @param aAttrDef The attribute that is using us.
+     * @param aArguments The actual arguments object that
+     */
+    Category: function(aAttrDef, aArguments) {
+      let rangePairs = [];
+      // If there are no arguments then we want to fall back to the 'in'
+      //  constraint which matches on any attachment.
+      if (aArguments.length == 0)
+        return this._inConstraintHelper(aAttrDef, []);
+
+      for (let iArg = 0; iArg < aArguments.length; iArg++) {
+        let arg = aArguments[iArg];
+        rangePairs.push(MimeTypeNoun._mimeTypeRangeDummyObjects[arg.category]);
+      }
+      return this._rangedConstraintHelper(aAttrDef, rangePairs);
+    }
+  },
+
+  comparator: function gloda_noun_mimeType_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.fullType.localeCompare(b.fullType);
+  },
+
+  toParamAndValue: function gloda_noun_mimeType_toParamAndValue(aMimeType) {
+    return [null, aMimeType.id];
+  },
+  toJSON: function gloda_noun_mimeType_toJSON(aMimeType) {
+    return aMimeType.id;
+  },
+  fromJSON: function gloda_noun_mimeType_fromJSON(aMimeTypeID) {
+    return this._mimeTypesByID[aMimeTypeID];
+  },
+};
+Gloda.defineNoun(MimeTypeNoun, Gloda.NOUN_MIME_TYPE);
+try {
+MimeTypeNoun._init();
+} catch (ex) {
+  LOG.error("problem init-ing: " + ex.fileName + ":" + ex.lineNumber + ": " + ex);
+}

mailnews/db/gloda/modules/noun_tag.js

@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['TagNoun'];
+
+Components.utils.import("resource:///modules/mailServices.js");
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+/**
+ * @namespace Tag noun provider.
+ */
+var TagNoun = {
+  name: "tag",
+  clazz: Ci.nsIMsgTag,
+  usesParameter: true,
+  allowsArbitraryAttrs: false,
+  idAttr: "key",
+  _msgTagService: null,
+  _tagMap: null,
+  _tagList: null,
+
+  _init: function () {
+    this._msgTagService = MailServices.tags;
+    this._updateTagMap();
+  },
+
+  getAllTags: function gloda_noun_tag_getAllTags() {
+    if (this._tagList == null)
+      this._updateTagMap();
+    return this._tagList;
+  },
+
+  _updateTagMap: function gloda_noun_tag_updateTagMap() {
+    this._tagMap = {};
+    let tagArray = this._tagList = this._msgTagService.getAllTags({});
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      this._tagMap[tag.key] = tag;
+    }
+  },
+
+  comparator: function gloda_noun_tag_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.tag.localeCompare(b.tag);
+  },
+  userVisibleString: function gloda_noun_tag_userVisibleString(aTag) {
+    return aTag.tag;
+  },
+
+  // we cannot be an attribute value
+
+  toParamAndValue: function gloda_noun_tag_toParamAndValue(aTag) {
+    return [aTag.key, null];
+  },
+  toJSON: function gloda_noun_tag_toJSON(aTag) {
+    return aTag.key;
+  },
+  fromJSON: function gloda_noun_tag_fromJSON(aTagKey, aIgnored) {
+    let tag = this._tagMap.hasOwnProperty(aTagKey) ? this._tagMap[aTagKey]
+                : undefined;
+    // you will note that if a tag is removed, we are unable to aggressively
+    //  deal with this.  we are okay with this, but it would be nice to be able
+    //  to listen to the message tag service to know when we should rebuild.
+    if ((tag === undefined) && this._msgTagService.isValidKey(aTagKey)) {
+      this._updateTagMap();
+      tag = this._tagMap[aTagKey];
+    }
+    // we intentionally are returning undefined if the tag doesn't exist
+    return tag;
+  },
+  /**
+   * Convenience helper to turn a tag key into a tag name.
+   */
+  getTag: function gloda_noun_tag_getTag(aTagKey) {
+    return this.fromJSON(aTagKey);
+  }
+};
+
+TagNoun._init();
+Gloda.defineNoun(TagNoun, Gloda.NOUN_TAG);

mailnews/db/gloda/modules/public.js

@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["Gloda"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/everybody.js");
+Cu.import("resource:///modules/gloda/indexer.js");
+// initialize the indexer! (who was actually imported as a nested dep by the
+//  things everybody.js imported.)  We waited until now so it could know about
+//  its indexers.
+GlodaIndexer._init();
+Cu.import("resource:///modules/gloda/index_msg.js");
+
+/**
+ * Expose some junk
+ */
+function proxy(aSourceObj, aSourceAttr, aDestObj, aDestAttr) {
+  aDestObj[aDestAttr] = function() {
+    return aSourceObj[aSourceAttr].apply(aSourceObj, arguments);
+  };
+}
+
+proxy(GlodaIndexer, "addListener", Gloda, "addIndexerListener");
+proxy(GlodaIndexer, "removeListener", Gloda, "removeIndexerListener");
+proxy(GlodaMsgIndexer, "isMessageIndexed", Gloda, "isMessageIndexed");
+proxy(GlodaMsgIndexer, "setFolderIndexingPriority", Gloda,
+      "setFolderIndexingPriority");
+proxy(GlodaMsgIndexer, "resetFolderIndexingPriority", Gloda,
+      "resetFolderIndexingPriority");

mailnews/db/gloda/modules/query.js

@@ -0,0 +1,618 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaQueryClassFactory"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+// GlodaDatastore has some constants we need, and oddly enough, there was no
+//  load dependency preventing us from doing this.
+Cu.import("resource:///modules/gloda/datastore.js");
+
+/**
+ * @class Query class core; each noun gets its own sub-class where attributes
+ *  have helper methods bound.
+ *
+ * @param aOptions A dictionary of options.  Current legal options are:
+ *     - noMagic: Indicates that the noun's dbQueryJoinMagic should be ignored.
+ *                Currently, this means that messages will not have their
+ *                full-text indexed values re-attached.  This is planned to be
+ *                offset by having queries/cache lookups that do not request
+ *                noMagic to ensure that their data does get loaded.
+ *     - explicitSQL: A hand-rolled alternate representation for the core
+ *           SELECT portion of the SQL query.  The queryFromQuery logic still
+ *           generates its normal query, we just ignore its result in favor of
+ *           your provided value.  This means that the positional parameter
+ *           list is still built and you should/must rely on those bound
+ *           parameters (using '?').  The replacement occurs prior to the
+ *           outerWrapColumns, ORDER BY, and LIMIT contributions to the query.
+ *     - outerWrapColumns: If provided, wraps the query in a "SELECT *,blah
+ *           FROM (actual query)" where blah is your list of outerWrapColumns
+ *           made comma-delimited.  The idea is that this allows you to
+ *           reference the result of expressions inside the query using their
+ *           names rather than having to duplicate the logic.  In practice,
+ *           this makes things more readable but is unlikely to improve
+ *           performance.  (Namely, my use of 'offsets' for full-text stuff
+ *           ends up in the EXPLAIN plan twice despite this.)
+ *     - noDbQueryValidityConstraints: Indicates that any validity constraints
+ *           should be ignored. This should be used when you need to get every
+ *           match regardless of whether it's valid.
+ *
+ * @property _owner The query instance that holds the list of unions...
+ * @property _constraints A list of (lists of OR constraints) that are ANDed
+ *     together.  For example [[FROM bob, FROM jim], [DATE last week]] would
+ *     be requesting us to find all the messages from either bob or jim, and
+ *     sent in the last week.
+ * @property _unions A list of other queries whose results are unioned with our
+ *     own.  There is no concept of nesting or sub-queries apart from this
+ *     mechanism.
+ */
+function GlodaQueryClass(aOptions) {
+  this.options = (aOptions != null) ? aOptions : {};
+
+  // if we are an 'or' clause, who is our parent whom other 'or' clauses should
+  //  spawn from...
+  this._owner = null;
+  // our personal chain of and-ing.
+  this._constraints = [];
+  // the other instances we union with
+  this._unions = [];
+
+  this._order = [];
+  this._limit = 0;
+}
+
+GlodaQueryClass.prototype = {
+  WILDCARD: {},
+
+  get constraintCount() {
+    return this._constraints.length;
+  },
+
+  or: function gloda_query_or() {
+    let owner = this._owner || this;
+    let orQuery = new this._queryClass();
+    orQuery._owner = owner;
+    owner._unions.push(orQuery);
+    return orQuery;
+  },
+
+  orderBy: function gloda_query_orderBy() {
+    for (let iArg = 0; iArg < arguments.length; iArg++) {
+      let arg = arguments[iArg];
+      this._order.push(arg);
+    }
+    return this;
+  },
+
+  limit: function gloda_query_limit(aLimit) {
+    this._limit = aLimit;
+    return this;
+  },
+
+  /**
+   * Return a collection asynchronously populated by this collection.  You must
+   *  provide a listener to receive notifications from the collection as it
+   *  receives updates.  The listener object should implement onItemsAdded,
+   *  onItemsModified, and onItemsRemoved methods, all of which take a single
+   *  argument which is the list of items which have been added, modified, or
+   *  removed respectively.
+   *
+   * @param aListener The collection listener.
+   * @param [aData] The data attribute to set on the collection.
+   * @param [aArgs.becomeExplicit] Make the collection explicit so that the
+   *     collection will only ever contain results found from the database
+   *     query and the query will not be updated as new items are indexed that
+   *     also match the query.
+   * @param [aArgs.becomeNull] Change the collection's query to a null query so
+   *     that it will never receive any additional added/modified/removed events
+   *     apart from the underlying database query.  This is really only intended
+   *     for gloda internal use but may be acceptable for non-gloda use.  Please
+   *     ask on mozilla.dev.apps.thunderbird first to make sure there isn't a
+   *     better solution for your use-case.  (Note: removals will still happen
+   *     when things get fully deleted.)
+   */
+  getCollection: function gloda_query_getCollection(aListener, aData, aArgs) {
+    this.completed = false;
+    return this._nounDef.datastore.queryFromQuery(this, aListener, aData,
+             /* aExistingCollection */ null, /* aMasterCollection */ null,
+             aArgs);
+  },
+
+  /**
+   * Test whether the given first-class noun instance satisfies this query.
+   *
+   * @testpoint gloda.query.test
+   */
+  test: function gloda_query_test(aObj) {
+    // when changing this method, be sure that GlodaDatastore's queryFromQuery
+    //  method likewise has any required changes made.
+    let unionQueries = [this].concat(this._unions);
+
+    for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
+      let curQuery = unionQueries[iUnion];
+
+      // assume success until a specific (or) constraint proves us wrong
+      let querySatisfied = true;
+      for (let iConstraint = 0; iConstraint < curQuery._constraints.length;
+           iConstraint++) {
+        let constraint = curQuery._constraints[iConstraint];
+        let [constraintType, attrDef] = constraint;
+        let boundName = attrDef ? attrDef.boundName : "id";
+        if ((boundName in aObj) &&
+            aObj[boundName] === GlodaDatastore.IGNORE_FACET) {
+          querySatisfied = false;
+          break;
+        }
+
+        let constraintValues = constraint.slice(2);
+
+        if (constraintType === GlodaDatastore.kConstraintIdIn) {
+          if (constraintValues.indexOf(aObj.id) == -1) {
+            querySatisfied = false;
+            break;
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintIn
+        else if ((constraintType === GlodaDatastore.kConstraintIn) ||
+                 (constraintType === GlodaDatastore.kConstraintEquals)) {
+          let objectNounDef = attrDef.objectNounDef;
+
+          // if they provide an equals comparator, use that.
+          // (note: the next case has better optimization possibilities than
+          //  this mechanism, but of course has higher initialization costs or
+          //  code complexity costs...)
+          if (objectNounDef.equals) {
+            let testValues;
+            if (!(boundName in aObj))
+              testValues = [];
+            else if (attrDef.singular)
+              testValues = [aObj[boundName]];
+            else
+              testValues = aObj[boundName];
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (constraintValues.length == 0 && testValues.length &&
+                testValues[0] != null)
+              continue;
+
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier)
+                continue;
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              for (let value of constraintValues) {
+                if (objectNounDef.equals(testValue, value)) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch)
+                break;
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          }
+          // otherwise, we need to convert everyone to their param/value form
+          //  in order to test for equality
+          else {
+            // let's just do the simple, obvious thing for now.  which is
+            //  what we did in the prior case but exploding values using
+            //  toParamAndValue, and then comparing.
+            let testValues;
+            if (!(boundName in aObj))
+              testValues = [];
+            else if (attrDef.singular)
+              testValues = [aObj[boundName]];
+            else
+              testValues = aObj[boundName];
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (constraintValues.length == 0 && testValues.length &&
+                testValues[0] != null)
+              continue;
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier)
+                continue;
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              let [aParam, aValue] = objectNounDef.toParamAndValue(testValue);
+              for (let value of constraintValues) {
+                // skip empty set check sentinel values
+                if (value == null && attrDef.emptySetIsSignificant)
+                  continue;
+                let [bParam, bValue] = objectNounDef.toParamAndValue(value);
+                if (aParam == bParam && aValue == bValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch)
+                break;
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintRanges
+        else if (constraintType === GlodaDatastore.kConstraintRanges) {
+          let objectNounDef = attrDef.objectNounDef;
+
+          let testValues;
+          if (!(boundName in aObj))
+              testValues = [];
+          else if (attrDef.singular)
+            testValues = [aObj[boundName]];
+          else
+            testValues = aObj[boundName];
+
+          let foundMatch = false;
+          for (let testValue of testValues) {
+            let [tParam, tValue] = objectNounDef.toParamAndValue(testValue);
+            for (let rangeTuple of constraintValues) {
+              let [lowerRValue, upperRValue] = rangeTuple;
+              if (lowerRValue == null) {
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                if (tParam == upperParam && tValue <= upperValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              else if (upperRValue == null) {
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if (tParam == lowerParam && tValue >= lowerValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              else { // no one is null
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if ((tParam == lowerParam) && (tValue >= lowerValue) &&
+                    (tParam == upperParam) && (tValue <= upperValue)) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+            }
+            if (foundMatch)
+              break;
+          }
+          if (!foundMatch) {
+            querySatisfied = false;
+            break;
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintStringLike
+        else if (constraintType === GlodaDatastore.kConstraintStringLike) {
+          let curIndex = 0;
+          let value = (boundName in aObj) ? aObj[boundName] : "";
+          // the attribute must be singular, we don't support arrays of strings.
+          for (let valuePart of constraintValues) {
+            if (typeof valuePart == "string") {
+              let index = value.indexOf(valuePart);
+              // if curIndex is null, we just need any match
+              // if it's not null, it must match the offset of our found match
+              if (curIndex === null) {
+                if (index == -1)
+                  querySatisfied = false;
+                else
+                  curIndex = index + valuePart.length;
+              }
+              else {
+                if (index != curIndex)
+                  querySatisfied = false;
+                else
+                  curIndex = index + valuePart.length;
+              }
+              if (!querySatisfied)
+                break;
+            }
+            else // wild!
+              curIndex = null;
+          }
+          // curIndex must be null or equal to the length of the string
+          if (querySatisfied && curIndex !== null && curIndex != value.length)
+            querySatisfied = false;
+        }
+        // @testpoint gloda.query.test.kConstraintFulltext
+        else if (constraintType === GlodaDatastore.kConstraintFulltext) {
+          // this is beyond our powers. Even if we have the fulltext content in
+          //  memory, which we may not, the tokenization and such to perform
+          //  the testing gets very complicated in the face of i18n, etc.
+          // so, let's fail if the item is not already in the collection, and
+          //  let the testing continue if it is.  (some other constraint may no
+          //  longer apply...)
+          if (!(aObj.id in this.collection._idMap))
+            querySatisfied = false;
+        }
+
+        if (!querySatisfied)
+          break;
+      }
+
+      if (querySatisfied)
+        return true;
+    }
+    return false;
+  },
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  traditional in/equals constraint.  The goal is to let them build a range
+   *  without having to know how we structure |_constraints|.
+   *
+   * @protected
+   */
+  _inConstraintHelper:
+      function gloda_query__discreteConstraintHelper(aAttrDef, aValues) {
+    let constraint =
+      [GlodaDatastore.kConstraintIn, aAttrDef].concat(aValues);
+    this._constraints.push(constraint);
+    return this;
+  },
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  range.  The goal is to let them build a range without having to know how
+   *  we structure |_constraints| or requiring them to mark themselves as
+   *  continuous to get a "Range".
+   *
+   * @protected
+   */
+  _rangedConstraintHelper:
+      function gloda_query__rangedConstraintHelper(aAttrDef, aRanges) {
+    let constraint =
+      [GlodaDatastore.kConstraintRanges, aAttrDef].concat(aRanges);
+    this._constraints.push(constraint);
+    return this;
+  }
+};
+
+/**
+ * @class A query that never matches anything.
+ *
+ * Collections corresponding to this query are intentionally frozen in time and
+ *  do not want to be notified of any updates.  We need the collection to be
+ *  registered with the collection manager so that the noun instances in the
+ *  collection are always 'reachable' via the collection for as long as we might
+ *  be handing out references to the instances.  (The other way to avoid updates
+ *  would be to not register the collection, but then items might not be
+ *  reachable.)
+ * This is intended to be used in implementation details behind the gloda
+ *  abstraction barrier.  For example, the message indexer likes to be able
+ *  to represent 'ghost' and deleted messages, but these should never be exposed
+ *  to the user.  For code simplicity, it wants to be able to use the query
+ *  mechanism.  But it doesn't want updates that are effectively
+ *  nonsensical.  For example, a ghost message that is reused by message
+ *  indexing may already be present in a collection; when the collection manager
+ *  receives an itemsAdded event, a GlodaExplicitQueryClass would result in
+ *  an item added notification in that case, which would wildly not be desired.
+ */
+function GlodaNullQueryClass() {
+}
+
+GlodaNullQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for null queries.
+   */
+  options: {},
+
+  /**
+   * Provide a duck-typing way of indicating to GlodaCollectionManager that our
+   *  associated collection just doesn't want anything to change.  Our test
+   *  function is able to convey most of it, but special-casing has to happen
+   *  somewhere, so it happens here.
+   */
+  frozen: true,
+
+  /**
+   * Since our query never matches anything, it doesn't make sense to let
+   *  someone attempt to construct a boolean OR involving us.
+   *
+   * @returns null
+   */
+  or: function() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on a null query.  This method is normally used (on a normal query)
+   *  to return a collection populated by the constraints of the query.  We
+   *  match nothing, so we should return nothing.  More importantly, you are
+   *  currently doing something wrong if you try and do this, so null is
+   *  appropriate.  It may turn out that it makes sense for us to return an
+   *  empty collection in the future for sentinel value purposes, but we'll
+   *  cross that bridge when we come to it.
+   *
+   * @returns null
+   */
+  getCollection: function() {
+    return null;
+  },
+
+  /**
+   * Never matches anything.
+   *
+   * @param aObj The object someone wants us to test for relevance to our
+   *     associated collection.  But we don't care!  Not a fig!
+   * @returns false
+   */
+  test: function gloda_query_null_test(aObj) {
+    return false;
+  }
+};
+
+/**
+ * @class A query that only 'tests' for already belonging to the collection.
+ *
+ * This type of collection is useful for when you (or rather your listener)
+ *  are interested in hearing about modifications to your collection or removals
+ *  from your collection because of deletion, but do not want to be notified
+ *  about newly indexed items matching your normal query constraints.
+ *
+ * @param aCollection The collection this query belongs to.  This needs to be
+ *     passed-in here or the collection should set the attribute directly when
+ *     the query is passed in to a collection's constructor.
+ */
+function GlodaExplicitQueryClass(aCollection) {
+  this.collection = aCollection;
+}
+
+GlodaExplicitQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for explicit queries.
+   */
+  options: {},
+
+  /**
+   * Since our query is intended to only match the contents of our collection,
+   *  it doesn't make sense to let someone attempt to construct a boolean OR
+   *  involving us.
+   *
+   * @returns null
+   */
+  or: function() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on an explicit query.  This method is normally used (on a normal
+   *  query) to return a collection populated by the constraints of the query.
+   *  In the case of an explicit query, we expect it will be associated with
+   *  either a hand-created collection or the results of a normal query that is
+   *  immediately converted into an explicit query.  In all likelihood, calling
+   *  this method on an instance of this type is an error, so it is helpful to
+   *  return null because people will error hard.
+   *
+   * @returns null
+   */
+  getCollection: function() {
+    return null;
+  },
+
+  /**
+   * Matches only items that are already in the collection associated with this
+   *  query (by id).
+   *
+   * @param aObj The object/item to test for already being in the associated
+   *     collection.
+   * @returns true when the object is in the associated collection, otherwise
+   *     false.
+   */
+  test: function gloda_query_explicit_test(aObj) {
+    return (aObj.id in this.collection._idMap);
+  }
+};
+
+/**
+ * @class A query that 'tests' true for everything.  Intended for debugging purposes
+ *  only.
+ */
+function GlodaWildcardQueryClass() {
+}
+
+GlodaWildcardQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation.
+   */
+  options: {},
+
+  // don't let people try and mess with us
+  or: function() { return null; },
+  // don't let people try and query on us (until we have a real use case for
+  //  that...)
+  getCollection: function() { return null; },
+  /**
+   * Everybody wins!
+   */
+  test: function gloda_query_explicit_test(aObj) {
+    return true;
+  }
+};
+
+/**
+ * Factory method to effectively create per-noun subclasses of GlodaQueryClass,
+ *  GlodaNullQueryClass, GlodaExplicitQueryClass, and GlodaWildcardQueryClass.
+ *  For GlodaQueryClass this allows us to add per-noun helpers.  For the others,
+ *  this is merely a means of allowing us to attach the (per-noun) nounDef to
+ *  the 'class'.
+ */
+function GlodaQueryClassFactory(aNounDef) {
+  let newQueryClass = function(aOptions) {
+    GlodaQueryClass.call(this, aOptions);
+  };
+  newQueryClass.prototype = new GlodaQueryClass();
+  newQueryClass.prototype._queryClass = newQueryClass;
+  newQueryClass.prototype._nounDef = aNounDef;
+
+  let newNullClass = function(aCollection) {
+    GlodaNullQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newNullClass.prototype = new GlodaNullQueryClass();
+  newNullClass.prototype._queryClass = newNullClass;
+  newNullClass.prototype._nounDef = aNounDef;
+
+  let newExplicitClass = function(aCollection) {
+    GlodaExplicitQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newExplicitClass.prototype = new GlodaExplicitQueryClass();
+  newExplicitClass.prototype._queryClass = newExplicitClass;
+  newExplicitClass.prototype._nounDef = aNounDef;
+
+  let newWildcardClass = function(aCollection) {
+    GlodaWildcardQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newWildcardClass.prototype = new GlodaWildcardQueryClass();
+  newWildcardClass.prototype._queryClass = newWildcardClass;
+  newWildcardClass.prototype._nounDef = aNounDef;
+
+  return [newQueryClass, newNullClass, newExplicitClass, newWildcardClass];
+}

mailnews/db/gloda/modules/suffixtree.js

@@ -0,0 +1,340 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["SuffixTree", "MultiSuffixTree"];
+
+/**
+ * Given a list of strings and a corresponding map of items that those strings
+ *  correspond to, build a suffix tree.
+ */
+function MultiSuffixTree(aStrings, aItems) {
+  if (aStrings.length != aItems.length)
+    throw new Error("Array lengths need to be the same.");
+
+  let s = '';
+  let offsetsToItems = [];
+  let lastLength = 0;
+  for (let i = 0; i < aStrings.length; i++) {
+    s += aStrings[i];
+    offsetsToItems.push(lastLength, s.length, aItems[i]);
+    lastLength = s.length;
+  }
+  
+  this._construct(s);
+  this._offsetsToItems = offsetsToItems;
+  this._numItems = aItems.length;
+}
+
+/**
+ * @constructor
+ */
+function State(aStartIndex, aEndIndex, aSuffix) {
+  this.start = aStartIndex;
+  this.end = aEndIndex;
+  this.suffix = aSuffix;
+}
+
+var dump;
+if (dump === undefined) {
+  dump = function(a) {
+    print(a.slice(0, -1));
+  };
+}
+
+/**
+ * Since objects are basically hash-tables anyways, we simply create an
+ *  attribute whose name is the first letter of the edge string.  (So, the
+ *  edge string can conceptually be a multi-letter string, but since we would
+ *  split it were there any ambiguity, it's okay to just use the single letter.)
+ *  This avoids having to update the attribute name or worry about tripping our
+ *  implementation up.
+ */
+State.prototype = {
+  get isExplicit() {
+    // our end is not inclusive...
+    return (this.end <= this.start);
+  },
+  get isImplicit() {
+    // our end is not inclusive...
+    return (this.end > this.start);
+  },
+  
+  get length() {
+    return this.end - this.start;
+  },
+  
+  toString: function State_toString() {
+    return "[Start: " + this.start + " End: " + this.end +
+           (this.suffix ? " non-null suffix]" : " null suffix]"); 
+  }
+};
+
+/**
+ * Suffix tree implemented using Ukkonen's algorithm.
+ * @constructor
+ */
+function SuffixTree(aStr) {
+  this._construct(aStr);
+}
+
+/**
+ * States are 
+ */
+SuffixTree.prototype = {
+  /**
+   * Find all items matching the provided substring.
+   */
+  findMatches: function findMatches(aSubstring) {
+    let results = [];
+    let state = this._root;
+    let index=0;
+    let end = aSubstring.length;
+    while(index < end) {
+      state = state[aSubstring[index]];
+      // bail if there was no edge
+      if (state === undefined)
+        return results;
+      // bail if the portion of the edge we traversed is not equal to that
+      //  portion of our pattern
+      let actualTraverseLength = Math.min(state.length,
+                                          end - index);
+      if (this._str.substring(state.start,
+                              state.start + actualTraverseLength) !=
+          aSubstring.substring(index, index + actualTraverseLength))
+        return results;
+      index += state.length;
+    }
+    
+    // state should now be the node which itself and all its children match...
+    // The delta is to adjust us to the offset of the last letter of our match;
+    //  the edge we traversed to get here may have found us traversing more
+    //  than we wanted.
+    // index - end captures the over-shoot of the edge traversal,
+    // index - end + 1 captures the fact that we want to find the last letter
+    //  that matched, not just the first letter beyond it
+    // However, if this state is a leaf node (end == 'infinity'), then 'end'
+    //  isn't describing an edge at all and we want to avoid accounting for it.
+    let delta;
+    /*
+    if (state.end != this._infinity)
+      //delta = index - end + 1;
+      delta = end - (index - state.length); 
+    else */
+    delta = index - state.length - end + 1;
+ 
+    this._resultGather(state, results, {}, end, delta, true);
+    return results;
+  },
+  
+  _resultGather: function resultGather(aState, aResults, aPresence,
+                                       aPatLength, aDelta, alreadyAdjusted) {
+    // find the item that this state originated from based on the state's
+    //  start character.  offsetToItem holds [string start index, string end
+    //  index (exclusive), item reference].  So we want to binary search to
+    //  find the string whose start/end index contains the state's start index.
+    let low = 0;
+    let high = this._numItems-1;
+    let mid, stringStart, stringEnd;
+    
+    let patternLast = aState.start - aDelta;
+    while (low <= high) {
+      mid = low + Math.floor((high - low) / 2); // excessive, especially with js nums
+      stringStart = this._offsetsToItems[mid*3];
+      let startDelta = stringStart - patternLast;
+      stringEnd = this._offsetsToItems[mid*3+1];
+      let endDelta = stringEnd - patternLast;
+      if (startDelta > 0)
+        high = mid - 1;
+      else if (endDelta <= 0)
+        low = mid + 1;
+      else {
+        break;
+      }
+    }
+    
+    // - The match occurred completely inside a source string.  Success.
+    // - The match spans more than one source strings, and is therefore not
+    //   a match.
+    
+    // at this point, we have located the origin string that corresponds to the
+    //  start index of this state.
+    // - The match terminated with the end of the preceding string, and does
+    //   not match us at all.  We, and potentially our children, are merely
+    //   serving as a unique terminal.
+    // - The 
+
+  let patternFirst = patternLast - (aPatLength - 1);
+
+  if (patternFirst >= stringStart) {
+    if (!(stringStart in aPresence)) {
+      aPresence[stringStart] = true;
+      aResults.push(this._offsetsToItems[mid*3+2]);
+    }
+  }
+    
+    // bail if we had it coming OR
+    // if the result terminates at/part-way through this state, meaning any
+    //  of its children are not going to be actual results, just hangers
+    //  on.
+/*
+    if (bail || (end <= aState.end)) {
+dump("  bailing! (bail was: " + bail + ")\n");
+      return;
+    }
+*/    
+    // process our children...
+    for (let key in aState) {
+      // edges have attributes of length 1...
+      if (key.length == 1) {
+        let statePrime = aState[key];
+        this._resultGather(statePrime, aResults, aPresence, aPatLength,
+                           aDelta + aState.length, //(alreadyAdjusted ? 0 : aState.length),
+                           false);
+      }
+    }
+  },
+
+  /**
+   * Given a reference 'pair' of a state and a string (may be 'empty'=explicit,
+   *  which means no work to do and we return immediately) follow that state
+   *  (and then the successive states)'s transitions until we run out of
+   *  transitions.  This happens either when we find an explicit state, or 
+   *  find ourselves partially along an edge (conceptually speaking).  In
+   *  the partial case, we return the state prior to the edge traversal.
+   * (The information about the 'edge' is contained on its target State;
+   *  we can do this because a state is only referenced by one other state.) 
+   */
+  _canonize: function canonize(aState, aStart, aEnd) {
+    if (aEnd <= aStart) {
+      return [aState, aStart];
+    }
+  
+    let statePrime;
+    // we treat an aState of null as 'bottom', which has transitions for every
+    //  letter in the alphabet to 'root'.  rather than create all those
+    //  transitions, we special-case here.
+    if (aState === null)
+      statePrime = this._root;
+    else
+      statePrime = aState[this._str[aStart]];
+    while (statePrime.length <= aEnd - aStart) { // (no 1 adjustment required)
+      aStart += statePrime.length;
+      aState = statePrime;
+      if (aStart < aEnd) {
+        statePrime = aState[this._str[aStart]];
+      }
+    }
+    return [aState, aStart]; 
+  },
+
+  /**
+   * Given a reference 'pair' whose state may or may not be explicit (and for
+   *  which we will perform the required splitting to make it explicit), test
+   *  whether it already possesses a transition corresponding to the provided
+   *  character.
+   * @return A list of: whether we had to make it explicit, the (potentially)
+   *    new explicit state.
+   */
+  _testAndSplit: function testAndSplit(aState, aStart, aEnd, aChar) {
+    if (aStart < aEnd) { // it's not explicit
+      let statePrime = aState[this._str[aStart]];
+      let length = aEnd - aStart;
+      if (aChar == this._str[statePrime.start + length]) {
+        return [true, aState];
+      }
+      else {
+        // do splitting... aState -> rState -> statePrime
+        let rState = new State(statePrime.start, statePrime.start + length);
+        aState[this._str[statePrime.start]] = rState;
+        statePrime.start += length;
+        rState[this._str[statePrime.start]] = statePrime;
+        return [false, rState];
+      }
+    }
+    else { // it's already explicit
+      if (aState === null) { // bottom case... shouldn't happen, but hey. 
+        return [true, aState];
+      }
+      return [(aChar in aState), aState];
+    }
+      
+  },
+
+  _update: function update(aState, aStart, aIndex) {
+    let oldR = this._root;
+    let textAtIndex = this._str[aIndex]; // T sub i (0-based corrected...)
+    // because of the way we store the 'end' value as a one-past form, we do
+    //  not need to subtract 1 off of aIndex.
+    let [endPoint, rState] = this._testAndSplit(aState, aStart, aIndex, //no -1
+                                                textAtIndex);
+    while (!endPoint) {
+      let rPrime = new State(aIndex, this._infinity);
+      rState[textAtIndex] = rPrime;
+      if (oldR !== this._root)
+        oldR.suffix = rState;
+      oldR = rState;
+      [aState, aStart] = this._canonize(aState.suffix, aStart, aIndex); // no -1
+      [endPoint, rState] = this._testAndSplit(aState, aStart, aIndex, // no -1
+                                              textAtIndex);
+    }
+    if (oldR !== this._root)
+      oldR.suffix = aState;
+    
+    return [aState, aStart];
+  },
+  
+  _construct: function construct(aStr) {
+    this._str = aStr;
+    // just needs to be longer than the string.
+    this._infinity = aStr.length + 1;
+    
+    //this._bottom = new State(0, -1, null);
+    this._root = new State(-1, 0, null); // null === bottom
+    let state = this._root;
+    let start = 0;
+  
+    for (let i = 0; i < aStr.length; i++) {
+      [state, start] = this._update(state, start, i); // treat as flowing -1...
+      [state, start] = this._canonize(state, start, i+1); // 1-length string
+    }
+  },
+  
+  dump: function SuffixTree_show(aState, aIndent, aKey) {
+    if (aState === undefined)
+      aState = this._root;
+    if (aIndent === undefined) {
+      aIndent = "";
+      aKey = ".";
+    }
+    
+    if (aState.isImplicit) {
+      let snip;
+      if (aState.length > 10)
+        snip = this._str.slice(aState.start,
+                           Math.min(aState.start+10, this._str.length)) + "...";
+      else
+        snip =  this._str.slice(aState.start,
+                                Math.min(aState.end, this._str.length)); 
+      dump(aIndent + aKey + ":" + snip + "(" +
+           aState.start + ":" + aState.end + ")\n");
+    }
+    else
+      dump(aIndent + aKey + ": (explicit:" + aState.start + ":" + aState.end +")\n");
+    let nextIndent = aIndent + "  ";
+    let keys = Object.keys(aState).filter(c => c.length == 1);
+    for (let key of keys) {
+      this.dump(aState[key], nextIndent, key);
+    }
+  }
+};
+MultiSuffixTree.prototype = SuffixTree.prototype;
+
+function examplar() {
+  let names = ["AndrewSmith", "AndrewJones", "MarkSmith", "BryanClark",
+               "MarthaJones", "DavidAscher", "DanMosedale", "DavidBienvenu",
+               "JanetDavis", "JosephBryant"];
+  let b = new MultiSuffixTree(names, names);
+  b.dump();
+  dump(b.findMatches("rya") + "\n");
+}

mailnews/db/gloda/modules/utils.js

@@ -0,0 +1,155 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaUtils'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/mailServices.js");
+
+/**
+ * @namespace A holding place for logic that is not gloda-specific and should
+ *  reside elsewhere.
+ */
+var GlodaUtils = {
+
+  /**
+   * This Regexp is super-complicated and used at least in two different parts of
+   * the code, so let's expose it from one single location.
+   */
+  PART_RE: new RegExp("^[^?]+\\?(?:/;section=\\d+\\?)?(?:[^&]+&)*part=([^&]+)(?:&[^&]+)*$"),
+
+  deMime: function gloda_utils_deMime(aString) {
+    return MailServices.mimeConverter.decodeMimeHeader(aString, null, false, true);
+  },
+
+  _headerParser: MailServices.headerParser,
+
+  /**
+   * Parses an RFC 2822 list of e-mail addresses and returns an object with
+   *  4 attributes, as described below.  We will use the example of the user
+   *  passing an argument of '"Bob Smith" <bob@example.com>'.
+   *
+   * This method (by way of nsIMsgHeaderParser) takes care of decoding mime
+   *  headers, but is not aware of folder-level character set overrides.
+   *
+   * count: the number of addresses parsed. (ex: 1)
+   * addresses: a list of e-mail addresses (ex: ["bob@example.com"])
+   * names: a list of names (ex: ["Bob Smith"])
+   * fullAddresses: aka the list of name and e-mail together (ex: ['"Bob Smith"
+   *  <bob@example.com>']).
+   *
+   * This method is a convenience wrapper around nsIMsgHeaderParser.
+   */
+  parseMailAddresses: function gloda_utils_parseMailAddresses(aMailAddresses) {
+    let addresses = {}, names = {}, fullAddresses = {};
+    this._headerParser.parseHeadersWithArray(aMailAddresses, addresses,
+                                             names, fullAddresses);
+    return {names: names.value, addresses: addresses.value,
+            fullAddresses: fullAddresses.value,
+            count: names.value.length};
+  },
+
+  /**
+   * MD5 hash a string and return the hex-string result. Impl from nsICryptoHash
+   *  docs.
+   */
+  md5HashString: function gloda_utils_md5hash(aString) {
+    let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+                    createInstance(Ci.nsIScriptableUnicodeConverter);
+    let trash = {};
+    converter.charset = "UTF-8";
+    let data = converter.convertToByteArray(aString, trash);
+
+    let hasher = Cc['@mozilla.org/security/hash;1'].
+                 createInstance(Ci.nsICryptoHash);
+    hasher.init(Ci.nsICryptoHash.MD5);
+    hasher.update(data, data.length);
+    let hash = hasher.finish(false);
+
+     // return the two-digit hexadecimal code for a byte
+    function toHexString(charCode) {
+      return ("0" + charCode.toString(16)).slice(-2);
+    }
+
+    // convert the binary hash data to a hex string.
+    let hex = Object.keys(hash).map(i => toHexString(hash.charCodeAt(i)));
+    return hex.join("");
+  },
+
+  getCardForEmail: function gloda_utils_getCardForEmail(aAddress) {
+    // search through all of our local address books looking for a match.
+    let enumerator = MailServices.ab.directories;
+    let cardForEmailAddress;
+    let addrbook;
+    while (!cardForEmailAddress && enumerator.hasMoreElements())
+    {
+      addrbook = enumerator.getNext().QueryInterface(Ci.nsIAbDirectory);
+      try
+      {
+        cardForEmailAddress = addrbook.cardForEmailAddress(aAddress);
+        if (cardForEmailAddress)
+          return cardForEmailAddress;
+      } catch (ex) {}
+    }
+
+    return null;
+  },
+
+  _FORCE_GC_AFTER_NUM_HEADERS: 4096,
+  _headersSeen: 0,
+  /**
+   * As |forceGarbageCollection| says, once XPConnect sees a header, it likes
+   *  to hold onto that reference.  This method is used to track the number of
+   *  headers we have seen and force a GC when we have to.
+   *
+   * Ideally the indexer's idle-biased GC mechanism would take care of all the
+   *  GC; we are just a failsafe to make sure that our memory usage is bounded
+   *  based on the number of headers we have seen rather than just time.
+   *  Since holding onto headers can keep databases around too, this also
+   *  helps avoid keeping file handles open, etc.
+   *
+   * |forceGarbageCollection| will zero our tracking variable when a GC happens
+   *  so we are informed by the indexer's GC triggering.
+   *
+   * And of course, we don't want to trigger collections willy nilly because
+   *  they have a cost even if there is no garbage.
+   *
+   * @param aNumHeadersSeen The number of headers code has seen.  A granularity
+   *     of hundreds of messages should be fine.
+   */
+  considerHeaderBasedGC: function(aNumHeadersSeen) {
+    this._headersSeen += aNumHeadersSeen;
+    if (this._headersSeen >= this._FORCE_GC_AFTER_NUM_HEADERS)
+      this.forceGarbageCollection();
+  },
+
+  /**
+   * Force a garbage-collection sweep.  Gloda has to force garbage collection
+   *  periodically because XPConnect's XPCJSRuntime::DeferredRelease mechanism
+   *  can end up holding onto a ridiculously high number of XPConnect objects in
+   *  between normal garbage collections.  This has mainly posed a problem
+   *  because nsAutolock is a jerk in DEBUG builds in 1.9.1, but in theory this
+   *  also helps us even out our memory usage.
+   * We also are starting to do this more to try and keep the garbage collection
+   *  durations acceptable.  We intentionally avoid triggering the cycle
+   *  collector in those cases, as we do presume a non-trivial fixed cost for
+   *  cycle collection.  (And really all we want is XPConnect to not be a jerk.)
+   * This method exists mainly to centralize our GC activities and because if
+   *  we do start involving the cycle collector, that is a non-trivial block of
+   *  code to copy-and-paste all over the place (at least in a module).
+   *
+   * @param aCycleCollecting Do we need the cycle collector to run?  Currently
+   *     unused / unimplemented, but we would use
+   *     nsIDOMWindowUtils.garbageCollect() to do so.
+   */
+  forceGarbageCollection:
+      function gloda_utils_garbageCollection(aCycleCollecting) {
+    Cu.forceGC();
+    this._headersSeen = 0;
+  }
+};

mailnews/db/gloda/moz.build

@@ -0,0 +1,11 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'modules',
+    'components',
+]
+
+JAR_MANIFESTS += ['jar.mn']

mailnews/db/moz.build

@@ -0,0 +1,10 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'msgdb',
+    'gloda',
+]
+

mailnews/db/msgdb/moz.build

@@ -0,0 +1,9 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += [
+    'public',
+    'src',
+]

mailnews/db/msgdb/public/moz.build

@@ -0,0 +1,27 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPIDL_SOURCES += [
+    'nsIDBChangeAnnouncer.idl',
+    'nsIDBChangeListener.idl',
+    'nsIDBFolderInfo.idl',
+    'nsIMsgDatabase.idl',
+    'nsIMsgOfflineImapOperation.idl',
+    'nsINewsDatabase.idl',
+]
+
+XPIDL_MODULE = 'msgdb'
+
+EXPORTS += [
+    'nsDBFolderInfo.h',
+    'nsImapMailDatabase.h',
+    'nsMailDatabase.h',
+    'nsMsgDatabase.h',
+    'nsMsgDBCID.h',
+    'nsMsgHdr.h',
+    'nsMsgThread.h',
+    'nsNewsDatabase.h',
+]
+

mailnews/db/msgdb/public/nsDBFolderInfo.h

@@ -0,0 +1,135 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* This class encapsulates the global information about a folder stored in the
+	summary file.
+*/
+#ifndef _nsDBFolderInfo_H
+#define _nsDBFolderInfo_H
+
+#include "mozilla/MemoryReporting.h"
+#include "nsStringGlue.h"
+#include "MailNewsTypes.h"
+#include "mdb.h"
+#include "nsTArray.h"
+#include "nsIDBFolderInfo.h"
+#include <time.h>
+
+class nsMsgDatabase;
+
+// again, this could inherit from nsISupports, but I don't see the need as of yet.
+// I'm not sure it needs to be ref-counted (but I think it does).
+
+// I think these getters and setters really need to go through mdb and not rely on the object
+// caching the values. If this somehow turns out to be prohibitively expensive, we can invent
+// some sort of dirty mechanism, but I think it turns out that these values will be cached by 
+// the MSG_FolderInfo's anyway.
+class nsDBFolderInfo : public nsIDBFolderInfo
+{
+public:
+  friend class nsMsgDatabase;
+  
+  nsDBFolderInfo(nsMsgDatabase *mdb);
+  
+  NS_DECL_ISUPPORTS
+    // interface methods.
+    NS_DECL_NSIDBFOLDERINFO
+    // create the appropriate table and row in a new db.
+    nsresult			AddToNewMDB();
+  // accessor methods.
+  
+  bool      TestFlag(int32_t flags);
+  int16_t   GetIMAPHierarchySeparator() ;
+  void      SetIMAPHierarchySeparator(int16_t hierarchyDelimiter) ;
+  void      ChangeImapTotalPendingMessages(int32_t delta);
+  void      ChangeImapUnreadPendingMessages(int32_t delta) ;
+  
+  nsresult      InitFromExistingDB();
+  // get and set arbitrary property, aka row cell value.
+  nsresult SetPropertyWithToken(mdb_token aProperty, const nsAString &propertyStr);
+  nsresult SetUint32PropertyWithToken(mdb_token aProperty, uint32_t propertyValue);
+  nsresult SetInt64PropertyWithToken(mdb_token aProperty, int64_t propertyValue);
+  nsresult SetInt32PropertyWithToken(mdb_token aProperty, int32_t propertyValue);
+  nsresult GetPropertyWithToken(mdb_token aProperty, nsAString &propertyValue);
+  nsresult GetUint32PropertyWithToken(mdb_token aProperty, uint32_t &propertyValue, uint32_t defaultValue = 0);
+  nsresult GetInt32PropertyWithToken(mdb_token aProperty, int32_t &propertyValue, int32_t defaultValue = 0);
+  nsresult GetInt64PropertyWithToken(mdb_token aProperty,
+                                     int64_t &propertyValue, int64_t defaultValue = 0);
+
+  nsTArray<nsMsgKey> m_lateredKeys; // list of latered messages
+
+  virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+  {
+    return m_lateredKeys.ShallowSizeOfExcludingThis(aMallocSizeOf);
+  }
+  virtual size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+  {
+    return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+  }
+protected:
+  virtual ~nsDBFolderInfo();
+
+  // initialize from appropriate table and row in existing db.
+  nsresult InitMDBInfo();
+  nsresult LoadMemberVariables();
+
+  nsresult AdjustHighWater(nsMsgKey highWater, bool force);
+
+  void ReleaseExternalReferences(); // let go of any references to other objects.
+
+  int64_t   m_folderSize;
+  int64_t   m_expungedBytes; // sum of size of deleted messages in folder
+  uint32_t  m_folderDate;
+  nsMsgKey  m_highWaterMessageKey; // largest news article number or imap uid whose header we've seen
+
+  //  m_numUnreadMessages and m_numMessages can never be negative. 0 means 'no msgs'.
+  int32_t   m_numUnreadMessages;
+  int32_t   m_numMessages;    // includes expunged and ignored messages
+
+  int32_t   m_flags;  // folder specific flags. This holds things like re-use thread pane,
+  // configured for off-line use, use default retrieval, purge article/header options
+
+  uint16_t    m_version;                // for upgrading...
+  int16_t     m_IMAPHierarchySeparator;	// imap path separator
+  
+  // mail only (for now)
+  
+  // IMAP only
+  int32_t     m_ImapUidValidity;
+  int32_t     m_totalPendingMessages;
+  int32_t     m_unreadPendingMessages;
+  
+  // news only (for now)
+  nsMsgKey    m_expiredMark;		// Highest invalid article number in group - for expiring
+  // the db folder info will have to know what db and row it belongs to, since it is really
+  // just a wrapper around the singleton folder info row in the mdb. 
+  nsMsgDatabase *m_mdb;
+  nsIMdbTable   *m_mdbTable;	// singleton table in db
+  nsIMdbRow     *m_mdbRow;	// singleton row in table;
+
+  nsCString     m_charSet;
+  bool          m_charSetOverride;
+  bool          m_mdbTokensInitialized;
+  
+  mdb_token     m_rowScopeToken;
+  mdb_token     m_tableKindToken;
+  // tokens for the pre-set columns - we cache these for speed, which may be silly
+  mdb_token     m_mailboxNameColumnToken;
+  mdb_token     m_numMessagesColumnToken;
+  mdb_token     m_numUnreadMessagesColumnToken;
+  mdb_token     m_flagsColumnToken;
+  mdb_token     m_folderSizeColumnToken;
+  mdb_token     m_expungedBytesColumnToken;
+  mdb_token     m_folderDateColumnToken;
+  mdb_token     m_highWaterMessageKeyColumnToken;
+  
+  mdb_token     m_imapUidValidityColumnToken;
+  mdb_token     m_totalPendingMessagesColumnToken;
+  mdb_token     m_unreadPendingMessagesColumnToken;
+  mdb_token     m_expiredMarkColumnToken;
+  mdb_token     m_versionColumnToken;
+};
+
+#endif

mailnews/db/msgdb/public/nsIDBChangeAnnouncer.idl

@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+#include "MailNewsTypes2.idl"
+
+interface nsIDBChangeListener;
+interface nsIMsgDBHdr;
+
+[scriptable, uuid(22baf00b-939d-42c3-ac51-21d99dfa1f05)]
+
+interface nsIDBChangeAnnouncer  : nsISupports {
+ /* these 2 calls return NS_OK on success, NS_COMFALSE on failure */
+  void AddListener(in nsIDBChangeListener listener);
+  void RemoveListener(in nsIDBChangeListener listener);
+
+  void NotifyHdrChangeAll(in nsIMsgDBHdr aHdrChanged, in unsigned long aOldFlags, in unsigned long aNewFlags, 
+                                in nsIDBChangeListener instigator);
+  void NotifyHdrAddedAll(in nsIMsgDBHdr aHdrAdded, in nsMsgKey parentKey, in long flags, 
+                               in nsIDBChangeListener instigator);
+  void NotifyHdrDeletedAll(in nsIMsgDBHdr aHdrDeleted, in nsMsgKey parentKey, in long flags, 
+                                 in nsIDBChangeListener instigator);
+  void NotifyParentChangedAll(in nsMsgKey keyReparented, in nsMsgKey oldParent, in nsMsgKey newParent, in nsIDBChangeListener instigator); 
+
+  void NotifyReadChanged(in nsIDBChangeListener instigator);
+
+  void NotifyJunkScoreChanged(in nsIDBChangeListener aInstigator);
+
+  void NotifyAnnouncerGoingAway();
+};
+

mailnews/db/msgdb/public/nsIDBChangeListener.idl

@@ -0,0 +1,115 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+#include "MailNewsTypes2.idl"
+
+interface nsIDBChangeAnnouncer;
+interface nsIMsgDBHdr;
+interface nsIMsgDatabase;
+
+/**
+ * These callbacks are provided to allow listeners to the message database
+ * to update their status when changes occur.
+ */
+[scriptable, uuid(21c56d34-71b9-42bb-9606-331a6a5f8210)]
+
+interface  nsIDBChangeListener : nsISupports {
+  /**
+   * Callback when message flags are changed.
+   *
+   * @param aHdrChanged The changed header.
+   * @param aOldFlags   Message flags prior to change.
+   * @param aNewFlags   Message flags after change.
+   * @param aInstigator Object that initiated the change.
+   */
+  void onHdrFlagsChanged(in nsIMsgDBHdr aHdrChanged, in unsigned long aOldFlags,
+                   in unsigned long aNewFlags, in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback when message is marked as deleted.
+   *
+   * @param aHdrChanged The message header that is going to be deleted.
+   * @param aParentKey  Key of parent.
+   * @param aFlags      Flags that message has before delete.
+   * @param aInstigator Object that initiated the change. Can be null.
+   */
+  void onHdrDeleted(in nsIMsgDBHdr aHdrChanged, in nsMsgKey aParentKey, in long aFlags,
+                    in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback when message is added.
+   *
+   * @param aHdrChanged The message header that is added.
+   * @param aParentKey  Parent key of message.
+   * @param aFlags      Flags that new message will have.
+   * @param aInstigator Object that initiated the change. Can be null.
+   */
+  void onHdrAdded(in nsIMsgDBHdr aHdrChanged, in nsMsgKey aParentKey, in long aFlags,
+                  in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback when message parrent is changed. Parent is changed when message is deleted or moved.
+   *
+   * @param aKeyChanged The message key that parent key was changed.
+   * @param oldParent   Old parent key.
+   * @param newParent   New parent key.
+   * @param aInstigator Object that initiated the change. Can be null.
+   */
+  void onParentChanged(in nsMsgKey aKeyChanged, in nsMsgKey oldParent, in nsMsgKey newParent,
+                       in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback when announcer is going away. This is good place to release strong pointers to announcer.
+   *
+   * @param instigator Object that initiated the change. Can be null.
+   */
+  void onAnnouncerGoingAway(in nsIDBChangeAnnouncer instigator);
+
+  /**
+   * Callback when read flag is changed.
+   *
+   * @param aInstigator Object that initiated the change. Can be null.
+   */
+  void onReadChanged(in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback used in case when "junkscore" property is changed.
+   *
+   * @param aInstigator Object that initiated the change. Can be null.
+   */
+  void onJunkScoreChanged(in nsIDBChangeListener aInstigator);
+
+  /**
+   * Callback used in the general case where any field may have changed.
+   * OnHdrPropertyChanged is called twice per change. On the first call, aPreChange
+   * is true, and aStatus is undefined. OnHdrPropertyChanged saves any required status in aStatus
+   * (such as a filter match). The calling function stores the value of aStatus, changes the
+   * header aHdrToChange, then calls OnHdrPropertyChanged again with aPreChange false. On this
+   * second call, the stored value of aStatus is provided, so that any changes may be noted.
+   *
+   * @param aHdrToChange the message header that is changing.
+   * @param aPreChange   true on first call before change, false on second call after change
+   * @param aStatus      storage location provided by calling routine for status
+   * @param aInstigator  object that initiated the change
+   */   
+  void onHdrPropertyChanged(in nsIMsgDBHdr aHdrToChange, in boolean aPreChange, inout uint32_t aStatus,
+                       in nsIDBChangeListener aInstigator);
+
+  /**
+   * Generic notification for extensibility. Common events should be documented
+   * here so we have a hope of keeping the documentation up to date.
+   * Current events are:
+   * "DBOpened" - When a pending listener becomes real. This can happen when
+   *              the existing db is force closed and a new one opened. Only
+   *              registered pending listeners are notified.
+   *
+   * @param aDB      the db for this event.
+   * @param aEvent   type of event. 
+   *
+   */
+  void onEvent(in nsIMsgDatabase aDB, in string aEvent);
+};
+

mailnews/db/msgdb/public/nsIDBFolderInfo.idl

@@ -0,0 +1,108 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+#include "MailNewsTypes2.idl"
+
+[scriptable, uuid(a72dab4b-b3bd-471e-9a38-1b242b385459)]
+interface  nsIDBFolderInfo : nsISupports {
+  attribute long flags;
+
+  /**
+   * Or's aFlags into flags.
+   *
+   * @param - the flags(s) to set
+   *
+   * @return - the resulting flags.
+   */
+  long  orFlags(in long aFlags);
+  /**
+   * And's aFlags with flags, set flags to the result
+   *
+   * @param             the flags(s) to AND
+   *
+   * @return            the resulting flags.
+   */
+  long  andFlags(in long aFlags);
+
+  /**
+   * Allows us to keep track of the highwater mark
+   *
+   * @param aNewKey     If larger than the current highwater
+   *                    mark, sets the highwater mark to aNewKey.
+   */
+  void  onKeyAdded(in nsMsgKey aNewKey);
+
+  attribute nsMsgKey highWater;
+  attribute nsMsgKey expiredMark;
+  attribute long long folderSize;
+  attribute unsigned long folderDate;
+  void changeNumUnreadMessages(in long aDelta);
+  void changeNumMessages(in long aDelta);
+
+  // numUnreadMessages and numMessages will never return negative numbers. 0 means 'no msgs'.
+  attribute long numUnreadMessages;
+  attribute long numMessages;
+
+  attribute long long expungedBytes;
+  attribute long imapUidValidity;
+  attribute unsigned long version;
+  attribute long imapTotalPendingMessages;
+  attribute long imapUnreadPendingMessages;
+
+  attribute nsMsgViewTypeValue viewType;
+  attribute nsMsgViewFlagsTypeValue viewFlags;
+  attribute nsMsgViewSortTypeValue sortType;
+  attribute nsMsgViewSortOrderValue sortOrder;
+
+  void changeExpungedBytes(in long aDelta);
+
+  /**
+   * Gets a string property from the folder.
+   *
+   * @param propertyName The name of the property for the value to retrieve.
+   */
+  ACString getCharProperty(in string propertyName);
+
+  /**
+   * Sets a string property from the folder.
+   *
+   * @param propertyName   The name of the property for which to set a value
+   * @param propertyValue  The new value of the property.
+   */
+  void setCharProperty(in string aPropertyName, in ACString aPropertyValue);
+  void setUint32Property(in string propertyName, in unsigned long propertyValue);
+  void setInt64Property(in string propertyName, in long long propertyValue);
+  unsigned long getUint32Property(in string propertyName, in unsigned long defaultValue);
+  long long     getInt64Property(in string propertyName, in long long defaultValue);
+  boolean getBooleanProperty(in string propertyName, in boolean defaultValue);
+  void setBooleanProperty(in string propertyName, in boolean aPropertyValue);
+  nsIDBFolderInfo	GetTransferInfo();
+  void	initFromTransferInfo(in nsIDBFolderInfo transferInfo);
+
+  /**
+   * Gets/Sets the current character set for the folder. If there is no
+   * specific character set for the folder, it will return an empty string.
+   */
+  attribute ACString characterSet;
+
+  /**
+   * Returns the effective character set on the folder. If there is no specific
+   * set defined for the folder, it will return the default character set.
+   */
+  readonly attribute ACString effectiveCharacterSet;
+
+  attribute boolean characterSetOverride;
+
+  attribute AString locale;
+  attribute AString mailboxName;
+
+
+  AString getProperty(in string propertyName);
+  void setProperty(in string propertyName, in AString propertyStr);
+
+  attribute string knownArtsSet;
+  attribute ACString folderName;
+};

mailnews/db/msgdb/public/nsIMsgDatabase.idl

@@ -0,0 +1,570 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * @defgroup msgdb Mailnews message database
+ * This module is the access point to locally-stored databases.
+ *
+ * These databases are stored in .msf files. Each file contains useful cached
+ * information, like the message id or references, as well as the cc header or
+ * tag information. This cached information is encapsulated in nsIMsgDBHdr.
+ *
+ * Also included is threading information, mostly encapsulated in nsIMsgThread.
+ * The final component is the database folder info, which contains information
+ * on the view and basic information also stored in the folder cache such as the
+ * name or most recent update.
+ *
+ * What this module does not do is access individual messages. Access is
+ * strictly controlled by the nsIMsgFolder objects and their backends.
+ * @{
+ */
+#include "nsISupports.idl"
+#include "nsIDBChangeAnnouncer.idl"
+
+%{C++
+#include "nsTArray.h"
+%}
+
+interface nsIMutableArray;
+interface nsIMsgDatabase;
+interface nsIMsgDBView;
+interface nsIDBChangeListener;
+interface nsIMsgDBHdr;
+interface nsISimpleEnumerator;
+interface nsIMsgThread;
+interface nsIDBFolderInfo;
+interface nsIMsgOfflineImapOperation;
+interface nsIMsgFolder;
+interface nsIMsgKeyArray;
+interface nsIFile;
+interface nsIArray;
+
+typedef unsigned long nsMsgRetainByPreference;
+
+
+[scriptable, uuid(fe8b7cec-eec8-4bcd-82ff-d8bb23cef3da)]
+
+interface nsIMsgRetentionSettings : nsISupports
+{
+  const unsigned long nsMsgRetainAll = 1;
+  const unsigned long nsMsgRetainByAge = 2;
+  const unsigned long nsMsgRetainByNumHeaders = 3;
+
+  attribute boolean useServerDefaults;
+  attribute nsMsgRetainByPreference retainByPreference;
+  attribute unsigned long daysToKeepHdrs;
+  attribute unsigned long numHeadersToKeep;
+
+  // this is for keeping offline bodies.
+  attribute boolean cleanupBodiesByDays;
+  attribute unsigned long daysToKeepBodies;
+
+  /**
+   * Should retention settings be applied to flagged/starred messages?
+   * If false, flagged messages are never automatically deleted.
+   */
+  attribute boolean applyToFlaggedMessages;
+};
+
+[scriptable, uuid(86a9da90-14f1-11d5-a5c0-0060b0fc04b7)]
+interface nsIMsgDownloadSettings : nsISupports
+{
+  attribute boolean useServerDefaults;
+  attribute boolean downloadByDate;
+  attribute boolean downloadUnreadOnly;
+  attribute unsigned long ageLimitOfMsgsToDownload;
+};
+
+typedef long nsMsgDBCommit;
+
+[scriptable, uuid(15431853-e448-45dc-8978-9958bf74d9b7)]
+
+interface nsMsgDBCommitType 
+{
+  const long kLargeCommit = 1;
+  const long kSessionCommit = 2;
+  const long kCompressCommit = 3;
+};
+
+[ref] native nsMsgKeyArrayRef(nsTArray<nsMsgKey>);
+[ptr] native nsMsgKeyArrayPtr(nsTArray<nsMsgKey>);
+
+/**
+ * A service to open mail databases and manipulate listeners automatically.
+ *
+ * The contract ID for this component is
+ * <tt>\@mozilla.org/msgDatabase/msgDBService;1</tt>.
+ */
+[scriptable, uuid(4cbbf024-3760-402d-89f3-6ababafeb07d)]
+interface nsIMsgDBService : nsISupports
+{
+  /**
+   * Opens a database for a given folder.
+   *
+   * This method is preferred over nsIMsgDBService::openMailDBFromFile if the
+   * caller has an actual nsIMsgFolder around. If the database detects that it
+   * is unreadable or out of date (using nsIMsgDatabase::outOfDate) it will
+   * destroy itself and prepare to be rebuilt, unless aLeaveInvalidDB is true.
+   *
+   * If one gets a NS_MSG_ERROR_FOLDER_SUMMARY_MISSING message, then one
+   * should call nsIMsgDBService::createNewDB to create the new database.
+   *
+   * @param aFolder         The folder whose database should be returned.
+   * @param aLeaveInvalidDB Whether or not the database should be deleted if it
+   *                        is invalid.
+   * @return                A new nsIMsgDatabase object representing the folder
+   *                        database that was opened.
+   * @exception NS_ERROR_FILE_TARGET_DOES_NOT_EXIST
+   *                        The file could not be created.
+   * @exception NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE
+   *                        The database is present (and was opened), but the
+   *                        summary file is out of date.
+   * @exception NS_MSG_ERROR_FOLDER_SUMMARY_MISSING
+   *                        The database is present, but the summary file is
+   *                        missing.
+   * @see nsIMsgDatabase::Open
+   * @see nsIMsgDBService::createNewDB
+   */
+  nsIMsgDatabase openFolderDB(in nsIMsgFolder aFolder,
+                              in boolean aLeaveInvalidDB);
+
+  /**
+   * This is the same as a synchronous open in terms of params and errors.
+   * But to finish opening the db, the caller must call
+   * nsIMsgDBService::OpenMore repeatedly until the open is finished.
+   * @see nsIMsgDBService::openFolderDB
+   * @see nsIMsgDBService::openMore
+   */
+  nsIMsgDatabase asyncOpenFolderDB(in nsIMsgFolder aFolder,
+                                   in boolean aLeaveInvalidDB);
+
+  /**
+   * Continues the open process for a db opened with
+   * nsIMsgDBService::asyncOpenFolderDB. Returns true if the db is ready
+   * to use, false if openMore needs to be called again.
+   * This will throw the same kinds of exceptions as openFolderDB.
+   * @param aTimeHint approximate number of milliseconds to spend
+   *                  before returning. This is more of a floor than
+                      a ceiling, since we can't guarantee that there
+                      won't be one big chunk that we can't interrupt.
+   * @return true if db is ready to use, false if openMore needs to
+   *         be called again.
+   * @see nsIMsgDBService::openFolderDB
+   */
+  boolean openMore(in nsIMsgDatabase aDB, in unsigned long aTimeHint);
+
+  /**
+   * Creates a new database for the given folder.
+   *
+   * If the database already exists, it will return the database, emit a
+   * warning, but not fully initialize it. For this reason, it should only be
+   * used when it is known that the database does not exist, such as when
+   * nsIMsgDBService::openFolderDB throws an error.
+   *
+   * @see nsIMsgDBService::openFolderDB
+   */
+  nsIMsgDatabase createNewDB(in nsIMsgFolder aFolder);
+
+  /**
+   * Opens or creates a database for a given file.
+   *
+   * This method should only be used if the caller does not have a folder
+   * instance, because the resulting db and message headers retrieved from the
+   * database would not know their owning folder, which limits their usefulness.
+   * For this reason, one should use nsIMsgDBService::openFolderDB instead
+   * except under special circumstances.
+   *
+   * Unlike nsIMsgDBService::openFolderDB, there is no corresponding method to
+   * create a new database if opening the database failed. However, this method
+   * will never throw NS_MSG_ERROR_FOLDER_SUMMARY_MISSING, so no corresponding
+   * method is needed.
+   *
+   * @param aFile           The file for which the database should be returned.
+   * @param aFolder         Folder the db corresponds to (may be null)
+   * @param aCreate         Whether or not the file should be created.
+   * @param aLeaveInvalidDB Whether or not the database should be deleted if it
+   *                        is invalid.
+   * @return                A new nsIMsgDatabase object encapsulating the file
+   *                        passed in.
+   * @exception NS_ERROR_FILE_TARGET_DOES_NOT_EXIST
+   *                        The file could not be created.
+   * @see nsIMsgDBService::openFolderDB
+   * @see nsIMsgDatabase::Open
+   */
+  nsIMsgDatabase openMailDBFromFile(in nsIFile aFile,
+                                    in nsIMsgFolder aFolder,
+                                    in boolean aCreate,
+                                    in boolean aLeaveInvalidDB);
+  /**
+   * Adds the given listener to the listener set for the folder.
+   *
+   * Since the message database will likely be opened and closed many times, by
+   * registering using this method, one will be guaranteed to see all subsequent
+   * modifications. This will also add the listener to the database if it is
+   * already opened.
+   *
+   * @param aFolder         The folder to add a listener to.
+   * @param aListener       The listener to add the folder to.
+   */
+  void registerPendingListener(in nsIMsgFolder aFolder,
+                               in nsIDBChangeListener aListener);
+  /**
+   * Removes the listener from all folder listener sets.
+   *
+   * @param aListener       The listener to remove.
+   * @exception NS_ERROR_FAILURE
+   *                        The listener is not registered.
+   */
+  void unregisterPendingListener(in nsIDBChangeListener aListener);
+
+  /**
+   * Get the db for a folder, if already open.
+   *
+   * @param aFolder   The folder to get the cached (open) db for.
+   *
+   * @returns         null if the db isn't open, otherwise the db.
+   */
+  nsIMsgDatabase cachedDBForFolder(in nsIMsgFolder aFolder);
+
+  /**
+   * Close the db for a folder, if already open.
+   *
+   * @param aFolder   The folder to close the cached (open) db for.
+   */
+  void forceFolderDBClosed(in nsIMsgFolder aFolder);
+
+  /// an enumerator to iterate over the open dbs.
+  readonly attribute nsIArray openDBs;
+};
+
+[scriptable, uuid(b64e66f8-4717-423a-be42-482658fb2199)]
+interface nsIMsgDatabase : nsIDBChangeAnnouncer {
+  void Close(in boolean aForceCommit);
+
+  void Commit(in nsMsgDBCommit commitType);
+  // Force closed is evil, and we should see if we can do without it.
+  // In 4.x, it was mainly used to remove corrupted databases.
+  void ForceClosed();
+  void clearCachedHdrs();
+  void resetHdrCacheSize(in unsigned long size);
+
+  readonly attribute nsIDBFolderInfo  dBFolderInfo;
+
+  /// Size of the database file in bytes.
+  readonly attribute long long databaseSize;
+
+  /// Folder this db was opened on.
+  readonly attribute nsIMsgFolder folder;
+
+  /**
+   * This is used when deciding which db's to close to free up memory
+   * and other resources in an LRU manner. It doesn't track every operation
+   * on every object from the db, but high level things like open, commit,
+   * and perhaps some of the list methods. Commit should be a proxy for all
+   * the mutation methods.
+   *
+   * I'm allowing clients to set the last use time as well, so that
+   * nsIMsgFolder.msgDatabase can set the last use time.
+   */
+  attribute PRTime lastUseTime;
+
+  // get a message header for the given key. Caller must release()!
+
+  nsIMsgDBHdr GetMsgHdrForKey(in nsMsgKey key);
+  nsIMsgDBHdr getMsgHdrForMessageID(in string messageID);
+
+  /**
+   * get a message header for a Gmail message with the given X-GM-MSGID.
+   */
+  nsIMsgDBHdr GetMsgHdrForGMMsgID(in string aGmailMessageID);
+  //Returns whether or not this database contains the given key
+  boolean ContainsKey(in nsMsgKey key);
+
+/**
+ * Must call AddNewHdrToDB after creating. The idea is that you create
+ * a new header, fill in its properties, and then call AddNewHdrToDB.
+ * AddNewHdrToDB will send notifications to any listeners.
+ *
+ * @param aKey msgKey for the new header. If aKey is nsMsgKey_None,
+ *             we will auto-assign a new key.
+ */
+  nsIMsgDBHdr CreateNewHdr(in nsMsgKey aKey);
+
+  void AddNewHdrToDB(in nsIMsgDBHdr newHdr, in boolean notify);
+
+  nsIMsgDBHdr CopyHdrFromExistingHdr(in nsMsgKey key, in nsIMsgDBHdr existingHdr, in boolean addHdrToDB);
+
+  /**
+   * Returns all message keys stored in the database.
+   * Keys are returned in the order as stored in the database.
+   * The caller should sort them if it needs to.
+   */
+  void ListAllKeys(in nsIMsgKeyArray array);
+
+  nsISimpleEnumerator EnumerateMessages();
+  nsISimpleEnumerator ReverseEnumerateMessages();
+  nsISimpleEnumerator EnumerateThreads();
+
+  /**
+   * Get an enumerator for use with nextMatchingHdrs. The enumerator
+   * will only return messages that match the passed-in search terms.
+   *
+   * @param     searchTerms       array of search terms to evaluate.
+   * @param     reverse           start at the end, defaults to false.
+   *
+   * @returns   an enumerator for passing into nextMatchingHdrs
+   */
+  nsISimpleEnumerator getFilterEnumerator(in nsIArray searchTerms,
+                                          [optional] in boolean reverse);
+
+  /**
+   * Get the next N matching headers using a filter enumerator
+   * obtained by calling getFilterEnumerator.
+   *
+   * @param     enumerator -      This *must* be a filter enumerator
+   * @param     numHdrsToLookAt   if non 0, the number of hdrs to advance the
+   *                              enumerator before returning.
+   * @param     maxResults        if non 0, the max results to return.
+   * @param     matchingHdrs      if non null, array of matching hdrs.
+   * @param     numMatches        if non null, the number of matching hdrs.
+   *
+   * @returns   false, if done, true if more hdrs to look at.
+   */
+  boolean nextMatchingHdrs(in nsISimpleEnumerator enumerator,
+                           in long numHdrsToLookAt,
+                           in long maxResults,
+                           in nsIMutableArray matchingHdrs,
+                           out long numMatches);
+
+
+  // count the total and unread msgs, and adjust global count if needed
+  void syncCounts();
+
+  nsIMsgThread GetThreadContainingMsgHdr(in nsIMsgDBHdr msgHdr) ;
+
+  // helpers for user command functions like delete, mark read, etc.
+
+  void MarkHdrRead(in nsIMsgDBHdr msgHdr, in boolean bRead,
+                         in nsIDBChangeListener instigator);
+
+  void MarkHdrReplied(in nsIMsgDBHdr msgHdr, in boolean bReplied,
+                         in nsIDBChangeListener instigator);
+
+  void MarkHdrMarked(in nsIMsgDBHdr msgHdr, in boolean mark,
+                         in nsIDBChangeListener instigator);
+  /**
+   * Remove the new status from a message.
+   *
+   * @param aMsgHdr        The database reference header for the message
+   * @param aInstigator    Reference to original calling object
+   */
+  void MarkHdrNotNew(in nsIMsgDBHdr aMsgHdr,
+                     in nsIDBChangeListener aInstigator);
+
+  // MDN support
+  void MarkMDNNeeded(in nsMsgKey key, in boolean bNeeded,
+                           in nsIDBChangeListener instigator);
+
+  // MarkMDNneeded only used when mail server is a POP3 server
+  // or when the IMAP server does not support user defined
+  // PERMANENTFLAGS
+  boolean IsMDNNeeded(in nsMsgKey key);
+
+  void MarkMDNSent(in nsMsgKey key, in boolean bNeeded,
+                         in nsIDBChangeListener instigator);
+  boolean IsMDNSent(in nsMsgKey key);
+
+// methods to get and set docsets for ids.
+  void MarkRead(in nsMsgKey key, in boolean bRead, 
+                      in nsIDBChangeListener instigator);
+
+  void MarkReplied(in nsMsgKey key, in boolean bReplied, 
+                         in nsIDBChangeListener instigator);
+
+  void MarkForwarded(in nsMsgKey key, in boolean bForwarded, 
+                           in nsIDBChangeListener instigator);
+
+  void MarkHasAttachments(in nsMsgKey key, in boolean bHasAttachments, 
+                                in nsIDBChangeListener instigator);
+
+  void MarkThreadRead(in nsIMsgThread thread, in nsIDBChangeListener instigator,
+                      out unsigned long aCount,
+                      [array, size_is(aCount)] out nsMsgKey aKeys);
+
+  /// Mark the specified thread ignored.
+  void MarkThreadIgnored(in nsIMsgThread thread, in nsMsgKey threadKey,
+                         in boolean bIgnored,
+                         in nsIDBChangeListener instigator);
+
+  /// Mark the specified thread watched.
+  void MarkThreadWatched(in nsIMsgThread thread, in nsMsgKey threadKey,
+                         in boolean bWatched,
+                         in nsIDBChangeListener instigator);
+
+  /// Mark the specified subthread ignored.
+  void MarkHeaderKilled(in nsIMsgDBHdr msg, in boolean bIgnored,
+                        in nsIDBChangeListener instigator);
+
+  /// Is the message read.
+  boolean IsRead(in nsMsgKey key);
+  /// Is the message part of an ignored thread.
+  boolean IsIgnored(in nsMsgKey key);
+  /// Is the message part of a watched thread.
+  boolean IsWatched(in nsMsgKey key);
+  /// Is the message flagged/starred.
+  boolean IsMarked(in nsMsgKey key);
+  /// Does the message have attachments.
+  boolean HasAttachments(in nsMsgKey key);
+
+  void MarkAllRead(out unsigned long aCount,
+                   [array, size_is(aCount)] out nsMsgKey aKeys);
+
+  void deleteMessages(in unsigned long aNumKeys,
+                      [array, size_is(aNumKeys)] in nsMsgKey nsMsgKeys,
+                      in nsIDBChangeListener instigator);
+  void DeleteMessage(in nsMsgKey key, 
+                           in nsIDBChangeListener instigator,
+                           in boolean commit);
+  void DeleteHeader(in nsIMsgDBHdr msgHdr, in nsIDBChangeListener instigator,
+                          in boolean commit, in boolean notify);
+
+  // lower level routine that doesn't remove hdr from thread or adjust counts
+  void RemoveHeaderMdbRow(in nsIMsgDBHdr msgHdr);
+
+  void UndoDelete(in nsIMsgDBHdr msgHdr);
+
+  void MarkMarked(in nsMsgKey key, in boolean mark,
+                        in nsIDBChangeListener instigator);
+  void MarkOffline(in nsMsgKey key, in boolean offline,
+                         in nsIDBChangeListener instigator);
+  void SetLabel(in nsMsgKey key, in nsMsgLabelValue label);
+  void setStringProperty(in nsMsgKey aKey, in string aProperty, in string aValue);
+  /**
+   * Set the value of a string property in a message header
+   *
+   * @param msgHdr    Header of the message whose property will be changed
+   * @param aProperty the property to change
+   * @param aValue    new value for the property
+   */
+  void setStringPropertyByHdr(in nsIMsgDBHdr msgHdr, in string aProperty, in string aValue);
+
+  /**
+   * Set the value of a uint32 property in a message header.
+   *
+   * @param aMsgHdr   header of the message whose property will be changed
+   * @param aProperty the property to change
+   * @param aValue    new value for the property
+   */
+  void setUint32PropertyByHdr(in nsIMsgDBHdr aMsgHdr,
+                              in string aProperty, in unsigned long aValue);
+
+  void MarkImapDeleted(in nsMsgKey key, in boolean deleted,
+                             in nsIDBChangeListener instigator);
+
+  readonly attribute nsMsgKey FirstNew;
+
+  attribute nsIMsgRetentionSettings msgRetentionSettings;
+  // purge unwanted message headers and/or bodies. If deleteViaFolder is
+  // true, we'll call nsIMsgFolder::DeleteMessages to delete the messages.
+  // Otherwise, we'll just delete them from the db.
+  void applyRetentionSettings(in nsIMsgRetentionSettings aMsgRetentionSettings, 
+                              in boolean aDeleteViaFolder);
+
+  attribute nsIMsgDownloadSettings msgDownloadSettings;
+
+  boolean HasNew();
+  void ClearNewList(in boolean notify);
+  void AddToNewList(in nsMsgKey key);
+
+  // used mainly to force the timestamp of a local mail folder db to
+  // match the time stamp of the corresponding berkeley mail folder,
+  // but also useful to tell the summary to mark itself invalid
+  // Also, if a local folder is being reparsed, summary will be invalid
+  // until the reparsing is done.
+  attribute boolean summaryValid;
+
+  // batching - can be used to cache file stream for local mail,
+  // and perhaps to use the mdb batching mechanism as well.
+  void StartBatch();
+  void EndBatch();
+  // offline operations - we could move these into an offline operation interface
+  // but it would have to be in nsMailDatabase, since local folders can be move destinations
+  nsIMsgOfflineImapOperation GetOfflineOpForKey(in nsMsgKey messageKey, in boolean create);
+  void  RemoveOfflineOp(in nsIMsgOfflineImapOperation op);
+  nsISimpleEnumerator EnumerateOfflineOps();
+  [noscript] void ListAllOfflineOpIds(in nsMsgKeyArrayPtr offlineOpIds);
+  [noscript] void ListAllOfflineDeletes(in nsMsgKeyArrayPtr offlineDeletes);
+  void ListAllOfflineMsgs(in nsIMsgKeyArray aKeys);
+
+  void setAttributeOnPendingHdr(in nsIMsgDBHdr pendingHdr, in string property,
+                                  in string propertyVal);
+
+  void setUint32AttributeOnPendingHdr(in nsIMsgDBHdr pendingHdr, in string property,
+                                  in unsigned long propertyVal);
+
+  /**
+   * Sets a pending 64 bit attribute, which tells the DB that when a message
+   * which looks like the pendingHdr (e.g., same message-id) is added to the
+   * db, set the passed in property and value on the new header. This is
+   * usually because we've copied an imap message to a different folder, and
+   * want to carry forward attributes from the original message to the copy,
+   * but don't have the message hdr for the copy yet so we can't set
+   * attributes directly.
+   *
+   * @param aPendingHdr usually the source of the copy.
+   * @param aProperty name of property to set.
+   * @param aPropertyVal 64 bit value of property to set.
+   */
+  void setUint64AttributeOnPendingHdr(in nsIMsgDBHdr aPendingHdr,
+                                      in string aProperty,
+                                      in unsigned long long aPropertyVal);
+
+  /**
+   * Given a message header with its message-id set, update any pending
+   *  attributes on the header.
+   *
+   * @param aNewHdr a new header that may have pending attributes.
+   */
+  void updatePendingAttributes(in nsIMsgDBHdr aNewHdr);
+
+  readonly attribute nsMsgKey lowWaterArticleNum;
+  readonly attribute nsMsgKey highWaterArticleNum;
+  attribute nsMsgKey nextPseudoMsgKey;   //for undo-redo of move pop->imap
+  readonly attribute nsMsgKey nextFakeOfflineMsgKey; // for saving "fake" offline msg hdrs
+  // for sorting
+  void createCollationKey(in AString sourceString, out unsigned long aCount,
+                          [array, size_is(aCount)] out octet aKey);
+  long compareCollationKeys(in unsigned long aLen1,
+                            [array, size_is(aLen1)] in octet key1,
+                            in unsigned long aLen2,
+                            [array, size_is(aLen2)] in octet key2);
+
+  // when creating a view, the default sort order and view flags 
+  // use these for the default.  (this allows news to override, so that
+  // news can be threaded by default)
+  readonly attribute nsMsgViewFlagsTypeValue defaultViewFlags;
+  readonly attribute nsMsgViewSortTypeValue  defaultSortType;
+  readonly attribute nsMsgViewSortOrderValue defaultSortOrder;
+
+  // for msg hdr hash table allocation. controllable by caller to improve folder loading preformance.
+  attribute unsigned long msgHdrCacheSize;
+
+  /**
+   * The list of messages currently in the NEW state.
+   * 
+   * If there are no such messages, a null pointer may be returned.
+   * the caller should free when done using free.
+   */
+  void getNewList(out unsigned long count, [array, size_is(count)] out nsMsgKey newKeys);
+  
+  // These are used for caching search hits in a db, to speed up saved search folders.
+  nsISimpleEnumerator getCachedHits(in string aSearchFolderUri);
+  void refreshCache(in string aSearchFolderUri, in unsigned long aNumKeys, [array, size_is (aNumKeys)] in nsMsgKey aNewHits,
+     out unsigned long aNumBadHits, [array, size_is(aNumBadHits)] out nsMsgKey aStaleHits);
+  void updateHdrInCache(in string aSearchFolderUri, in nsIMsgDBHdr aHdr, in boolean aAdd);
+  boolean hdrIsInCache(in string aSearchFolderUri, in nsIMsgDBHdr aHdr);
+
+};
+/** @} */

mailnews/db/msgdb/public/nsIMsgOfflineImapOperation.idl

@@ -0,0 +1,53 @@
+/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+#include "MailNewsTypes2.idl"
+// #include "nsIImapUrl.idl" // for imapMessageFlagsType
+
+typedef unsigned short imapMessageFlagsType;
+
+typedef long nsOfflineImapOperationType;
+
+[scriptable, uuid(b5229a55-22bb-444b-be92-13d719353828)]
+
+interface nsIMsgOfflineImapOperation : nsISupports
+{
+// type of stored imap operations
+  const long kFlagsChanged      = 0x1;
+  const long kMsgMoved          = 0x2;
+  const long kMsgCopy           = 0x4;
+  const long kMoveResult        = 0x8;
+  const long kAppendDraft       = 0x10;
+  const long kAddedHeader       = 0x20;
+  const long kDeletedMsg        = 0x40;
+  const long kMsgMarkedDeleted	= 0x80;
+  const long kAppendTemplate    = 0x100;
+  const long kDeleteAllMsgs     = 0x200;
+  const long kAddKeywords       = 0x400;
+  const long kRemoveKeywords    = 0x800;
+
+  attribute nsOfflineImapOperationType operation;
+  void clearOperation(in nsOfflineImapOperationType operation);
+  attribute nsMsgKey messageKey;
+
+  // for move/copy operations, the msg key of the source msg.
+  attribute nsMsgKey srcMessageKey;
+
+  attribute imapMessageFlagsType flagOperation;
+  attribute imapMessageFlagsType newFlags; // for kFlagsChanged
+  attribute string destinationFolderURI; // for move or copy
+  attribute string sourceFolderURI;
+  void addKeywordToAdd(in string aKeyword);
+  void addKeywordToRemove(in string aKeyword);
+  readonly attribute string keywordsToAdd; 
+  readonly attribute string keywordsToRemove;
+  readonly attribute long numberOfCopies;
+  void addMessageCopyOperation(in string destinationBox);
+  string getCopyDestination(in long copyIndex);
+  attribute unsigned long msgSize;
+  attribute boolean playingBack;
+};
+

mailnews/db/msgdb/public/nsINewsDatabase.idl

@@ -0,0 +1,18 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+%{C++
+#include "nsMsgKeySet.h"
+%}
+
+[ptr] native nsMsgKeySetPtr(nsMsgKeySet);
+
+[scriptable, uuid(f700208a-1dd1-11b2-b947-e4e1e4fdf278)]
+
+interface nsINewsDatabase : nsISupports {
+  [noscript] attribute nsMsgKeySetPtr readSet;
+};

mailnews/db/msgdb/public/nsImapMailDatabase.h

@@ -0,0 +1,52 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef _nsImapMailDatabase_H_
+#define _nsImapMailDatabase_H_
+
+#include "mozilla/Attributes.h"
+#include "nsMailDatabase.h"
+
+class nsImapMailDatabase : public nsMailDatabase
+{
+public:
+  // OK, it's dumb that this should require a fileSpec, since there is no file
+  // for the folder. This is mainly because we're deriving from nsMailDatabase;
+  // Perhaps we shouldn't...
+  nsImapMailDatabase();
+  virtual ~nsImapMailDatabase();
+  
+  NS_IMETHOD    StartBatch() override;
+  NS_IMETHOD    EndBatch() override;
+  NS_IMETHOD    GetSummaryValid(bool *aResult) override;
+  NS_IMETHOD    SetSummaryValid(bool valid = true) override;
+  virtual nsresult AdjustExpungedBytesOnDelete(nsIMsgDBHdr *msgHdr) override;
+
+  NS_IMETHOD    ForceClosed() override;
+  NS_IMETHOD    AddNewHdrToDB(nsIMsgDBHdr *newHdr, bool notify) override;
+  NS_IMETHOD    SetAttributeOnPendingHdr(nsIMsgDBHdr *pendingHdr, const char *property,
+                                         const char *propertyVal) override;
+  NS_IMETHOD    SetUint32AttributeOnPendingHdr(nsIMsgDBHdr *pendingHdr, const char *property,
+                                               uint32_t propertyVal) override;
+  NS_IMETHOD    SetUint64AttributeOnPendingHdr(nsIMsgDBHdr *aPendingHdr,
+                                               const char *aProperty,
+                                               uint64_t aPropertyVal) override;
+  NS_IMETHOD    DeleteMessages(uint32_t aNumKeys, nsMsgKey* nsMsgKeys,
+                               nsIDBChangeListener *instigator) override;
+  NS_IMETHOD    UpdatePendingAttributes(nsIMsgDBHdr* aNewHdr) override;
+
+protected:
+  // IMAP does not set local file flags, override does nothing
+  virtual void UpdateFolderFlag(nsIMsgDBHdr *msgHdr, bool bSet,
+                                nsMsgMessageFlagType flag, nsIOutputStream **ppFileStream);
+
+  nsresult      GetRowForPendingHdr(nsIMsgDBHdr *pendingHdr, nsIMdbRow **row);
+  nsresult     GetAllPendingHdrsTable();
+  mdb_token    m_pendingHdrsRowScopeToken;
+  mdb_token    m_pendingHdrsTableKindToken;
+  nsCOMPtr<nsIMdbTable> m_mdbAllPendingHdrsTable;
+};
+
+
+#endif

mailnews/db/msgdb/public/nsMailDatabase.h

@@ -0,0 +1,67 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMailDatabase_H_
+#define _nsMailDatabase_H_
+
+#include "mozilla/Attributes.h"
+#include "nsMsgDatabase.h"
+#include "nsMsgMessageFlags.h"
+#include "nsIFile.h"
+#include "nsTArray.h"
+
+// This is the subclass of nsMsgDatabase that handles local mail messages.
+class nsIOFileStream;
+class nsIFile;
+class nsOfflineImapOperation;
+
+class nsMailDatabase : public nsMsgDatabase
+{
+public:
+  nsMailDatabase();
+  virtual ~nsMailDatabase();
+  NS_IMETHOD  ForceClosed() override;
+  NS_IMETHOD DeleteMessages(uint32_t aNumKeys, nsMsgKey* nsMsgKeys,
+                            nsIDBChangeListener *instigator) override;
+
+  NS_IMETHOD StartBatch() override;
+  NS_IMETHOD EndBatch() override;
+
+  nsresult  Open(nsMsgDBService* aDBService, nsIFile *aSummaryFile, bool create, bool upgrading) override;
+  virtual nsMailDatabase  *GetMailDB() {return this;}
+
+  virtual uint32_t  GetCurVersion() override {return kMsgDBVersion;}
+  
+  NS_IMETHOD  GetOfflineOpForKey(nsMsgKey opKey, bool create,
+                                 nsIMsgOfflineImapOperation **op) override;
+  NS_IMETHOD  RemoveOfflineOp(nsIMsgOfflineImapOperation *op) override;
+
+  NS_IMETHOD  SetSummaryValid(bool valid) override;
+  NS_IMETHOD  GetSummaryValid(bool *valid) override;
+	
+  NS_IMETHOD    EnumerateOfflineOps(nsISimpleEnumerator **enumerator) override;
+  NS_IMETHOD    ListAllOfflineOpIds(nsTArray<nsMsgKey> *offlineOpIds) override;
+  NS_IMETHOD    ListAllOfflineDeletes(nsTArray<nsMsgKey> *offlineDeletes) override;
+
+  friend class nsMsgOfflineOpEnumerator;
+protected:
+
+  nsresult        GetAllOfflineOpsTable(); // get this on demand
+
+  // get the time and date of the mailbox file
+  void            GetMailboxModProperties(int64_t *aSize, uint32_t *aDate); 
+
+  nsCOMPtr <nsIMdbTable>  m_mdbAllOfflineOpsTable;
+  mdb_token       m_offlineOpsRowScopeToken;
+  mdb_token       m_offlineOpsTableKindToken;
+
+  virtual void    SetReparse(bool reparse);
+  
+protected:
+  
+  bool            m_reparse;
+};
+
+#endif

mailnews/db/msgdb/public/nsMsgDBCID.h

@@ -0,0 +1,63 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsMsgDBCID_h__
+#define nsMsgDBCID_h__
+
+#include "nsISupports.h"
+#include "nsIFactory.h"
+#include "nsIComponentManager.h"
+
+// 03223c50-1e88-45e8-ba1a-7ce792dc3fc3
+#define NS_MSGDB_SERVICE_CID \
+{  0x03223c50, 0x1e88, 0x45e8, \
+    { 0xba, 0x1a, 0x7c, 0xe7, 0x92, 0xdc, 0x3f, 0xc3 } }
+
+#define NS_MSGDB_SERVICE_CONTRACTID \
+  "@mozilla.org/msgDatabase/msgDBService;1"
+
+#define NS_MSGDB_CONTRACTID \
+  "@mozilla.org/nsMsgDatabase/msgDB-"
+
+#define NS_MAILBOXDB_CONTRACTID \
+  NS_MSGDB_CONTRACTID"mailbox"
+
+//	a86c86ae-e97f-11d2-a506-0060b0fc04b7
+#define NS_MAILDB_CID                      \
+{ 0xa86c86ae, 0xe97f, 0x11d2,                   \
+    { 0xa5, 0x06, 0x00, 0x60, 0xb0, 0xfc, 0x04, 0xb7 } }
+
+#define NS_NEWSDB_CONTRACTID \
+  NS_MSGDB_CONTRACTID"news"
+
+// 36414aa0-e980-11d2-a506-0060b0fc04b7
+#define NS_NEWSDB_CID                      \
+{ 0x36414aa0, 0xe980, 0x11d2,                  \
+    { 0xa5, 0x06, 0x00, 0x60, 0xb0, 0xfc, 0x04, 0xb7 } }
+
+#define NS_IMAPDB_CONTRACTID \
+  NS_MSGDB_CONTRACTID"imap"
+
+// 9e4b07ee-e980-11d2-a506-0060b0fc04b7
+#define NS_IMAPDB_CID                      \
+{ 0x9e4b07ee, 0xe980, 0x11d2,                  \
+    { 0xa5, 0x06, 0x00, 0x60, 0xb0, 0xfc, 0x04, 0xb7 } }
+
+#define NS_MSG_RETENTIONSETTINGS_CID \
+{ 0x1bd976d6, 0xdf44, 0x11d4,       \
+  {0xa5, 0xb6, 0x00, 0x60, 0xb0, 0xfc, 0x04, 0xb7} }
+
+#define NS_MSG_RETENTIONSETTINGS_CONTRACTID \
+  "@mozilla.org/msgDatabase/retentionSettings;1"
+
+// 4e3dae5a-157a-11d5-a5c0-0060b0fc04b7
+#define NS_MSG_DOWNLOADSETTINGS_CID \
+{ 0x4e3dae5a, 0x157a, 0x11d5,       \
+  {0xa5, 0xc0, 0x00, 0x60, 0xb0, 0xfc, 0x04, 0xb7} }
+
+#define NS_MSG_DOWNLOADSETTINGS_CONTRACTID \
+  "@mozilla.org/msgDatabase/downloadSettings;1"
+
+#endif

mailnews/db/msgdb/public/nsMsgDatabase.h

@@ -0,0 +1,462 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMsgDatabase_H_
+#define _nsMsgDatabase_H_
+
+#include "mozilla/Attributes.h"
+#include "mozilla/MemoryReporting.h"
+#include "nsIMsgDatabase.h"
+#include "nsMsgHdr.h"
+#include "nsStringGlue.h"
+#include "nsAutoPtr.h"
+#include "nsIDBChangeListener.h"
+#include "nsIDBChangeAnnouncer.h"
+#include "nsMsgMessageFlags.h"
+#include "nsIMsgFolder.h"
+#include "nsIMutableArray.h"
+#include "nsDBFolderInfo.h"
+#include "nsICollation.h"
+#include "nsIMsgSearchSession.h"
+#include "nsIMimeConverter.h"
+#include "nsCOMPtr.h"
+#include "nsCOMArray.h"
+#include "PLDHashTable.h"
+#include "nsTArray.h"
+#include "nsTObserverArray.h"
+class ListContext;
+class nsMsgKeySet;
+class nsMsgThread;
+class nsMsgDatabase;
+class nsIMsgThread;
+class nsIDBFolderInfo;
+
+const int32_t kMsgDBVersion = 1;
+
+// Hopefully we're not opening up lots of databases at the same time, however
+// this will give us a buffer before we need to start reallocating the cache
+// array.
+const uint32_t kInitialMsgDBCacheSize = 20;
+
+class nsMsgDBService final : public nsIMsgDBService
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGDBSERVICE
+
+  nsMsgDBService();
+
+  void AddToCache(nsMsgDatabase* pMessageDB);
+  void DumpCache();
+  void EnsureCached(nsMsgDatabase* pMessageDB)
+  {
+    if (!m_dbCache.Contains(pMessageDB))
+      m_dbCache.AppendElement(pMessageDB);
+  }
+  void RemoveFromCache(nsMsgDatabase* pMessageDB)
+  {
+    m_dbCache.RemoveElement(pMessageDB);
+  }
+
+protected:
+  ~nsMsgDBService();
+  void HookupPendingListeners(nsIMsgDatabase *db, nsIMsgFolder *folder);
+  void FinishDBOpen(nsIMsgFolder *aFolder, nsMsgDatabase *aMsgDB);
+  nsMsgDatabase* FindInCache(nsIFile *dbName);
+
+  nsCOMArray <nsIMsgFolder> m_foldersPendingListeners;
+  nsCOMArray <nsIDBChangeListener> m_pendingListeners;
+  AutoTArray<nsMsgDatabase*, kInitialMsgDBCacheSize> m_dbCache;
+};
+
+class nsMsgDBEnumerator : public nsISimpleEnumerator {
+public:
+    NS_DECL_ISUPPORTS
+
+    // nsISimpleEnumerator methods:
+    NS_DECL_NSISIMPLEENUMERATOR
+
+    // nsMsgDBEnumerator methods:
+    typedef nsresult (*nsMsgDBEnumeratorFilter)(nsIMsgDBHdr* hdr, void* closure);
+
+    nsMsgDBEnumerator(nsMsgDatabase* db, nsIMdbTable *table,
+                      nsMsgDBEnumeratorFilter filter, void* closure,
+                      bool iterateForwards = true);
+    void Clear();
+
+    nsresult                        GetRowCursor();
+    virtual nsresult                PrefetchNext();
+    RefPtr<nsMsgDatabase>         mDB;
+    nsCOMPtr<nsIMdbTableRowCursor>  mRowCursor;
+    mdb_pos                         mRowPos;
+    nsCOMPtr<nsIMsgDBHdr>           mResultHdr;
+    bool                            mDone;
+    bool                            mNextPrefetched;
+    bool                            mIterateForwards;
+    nsMsgDBEnumeratorFilter         mFilter;
+    nsCOMPtr <nsIMdbTable>          mTable;
+    void*                           mClosure;
+    // This is used when the caller wants to limit how many headers the
+    // enumerator looks at in any given time slice.
+    mdb_pos                         mStopPos;
+
+protected:
+    virtual ~nsMsgDBEnumerator();
+};
+
+class nsMsgFilteredDBEnumerator : public nsMsgDBEnumerator
+{
+public:
+  nsMsgFilteredDBEnumerator(nsMsgDatabase* db, nsIMdbTable *table,
+                            bool reverse, nsIArray *searchTerms);
+  virtual ~nsMsgFilteredDBEnumerator();
+  nsresult InitSearchSession(nsIArray *searchTerms, nsIMsgFolder *folder);
+
+protected:
+  virtual nsresult               PrefetchNext() override;
+
+  nsCOMPtr <nsIMsgSearchSession> m_searchSession;
+
+};
+
+namespace mozilla {
+namespace mailnews {
+class MsgDBReporter;
+}
+}
+
+class nsMsgDatabase : public nsIMsgDatabase
+{
+public:
+  friend class nsMsgDBService;
+  friend class nsMsgPropertyEnumerator; // accesses m_mdbEnv and m_mdbStore
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIDBCHANGEANNOUNCER
+  NS_DECL_NSIMSGDATABASE
+
+  /**
+   * Opens a database folder.
+   *
+   * @param aFolderName     The name of the folder to create.
+   * @param aCreate         Whether or not the file should be created.
+   * @param aLeaveInvalidDB Set to true if you do not want the database to be
+   *                        deleted if it is invalid.
+   * @exception NS_ERROR_FILE_TARGET_DOES_NOT_EXIST
+   *                        The file could not be created.
+   * @exception NS_MSG_ERROR_FOLDER_SUMMARY_OUT_OF_DATE
+   *                        The database is present (and was opened), but the
+   *                        summary file is out of date.
+   * @exception NS_MSG_ERROR_FOLDER_SUMMARY_MISSING
+   *                        The database is present (and was opened), but the
+   *                        summary file is missing.
+   */
+  virtual nsresult Open(nsMsgDBService *aDBService, nsIFile *aFolderName,
+                        bool aCreate, bool aLeaveInvalidDB);
+  virtual nsresult IsHeaderRead(nsIMsgDBHdr *hdr, bool *pRead);
+  virtual nsresult MarkHdrReadInDB(nsIMsgDBHdr *msgHdr, bool bRead,
+                               nsIDBChangeListener *instigator);
+  nsresult OpenInternal(nsMsgDBService *aDBService, nsIFile *aFolderName,
+                        bool aCreate, bool aLeaveInvalidDB, bool sync);
+  nsresult CheckForErrors(nsresult err, bool sync, nsMsgDBService *aDBService, nsIFile *summaryFile);
+  virtual nsresult OpenMDB(const char *dbName, bool create, bool sync);
+  virtual nsresult CloseMDB(bool commit);
+  virtual nsresult CreateMsgHdr(nsIMdbRow* hdrRow, nsMsgKey key, nsIMsgDBHdr **result);
+  virtual nsresult GetThreadForMsgKey(nsMsgKey msgKey, nsIMsgThread **result);
+  virtual nsresult EnumerateMessagesWithFlag(nsISimpleEnumerator* *result, uint32_t *pFlag);
+  nsresult         GetSearchResultsTable(const char *searchFolderUri, bool createIfMissing, nsIMdbTable **table);
+
+  // this might just be for debugging - we'll see.
+  nsresult ListAllThreads(nsTArray<nsMsgKey> *threadIds);
+  //////////////////////////////////////////////////////////////////////////////
+  // nsMsgDatabase methods:
+  nsMsgDatabase();
+
+  void GetMDBFactory(nsIMdbFactory ** aMdbFactory);
+  nsIMdbEnv             *GetEnv() {return m_mdbEnv;}
+  nsIMdbStore           *GetStore() {return m_mdbStore;}
+  virtual uint32_t      GetCurVersion();
+  nsresult              GetCollationKeyGenerator();
+  nsIMimeConverter *    GetMimeConverter();
+
+  nsresult GetTableCreateIfMissing(const char *scope, const char *kind, nsIMdbTable **table,
+                                   mdb_token &scopeToken, mdb_token &kindToken);
+
+  //helper function to fill in nsStrings from hdr row cell contents.
+  nsresult RowCellColumnTonsString(nsIMdbRow *row, mdb_token columnToken, nsAString &resultStr);
+  nsresult RowCellColumnToUInt32(nsIMdbRow *row, mdb_token columnToken, uint32_t *uint32Result, uint32_t defaultValue = 0);
+  nsresult RowCellColumnToUInt32(nsIMdbRow *row, mdb_token columnToken, uint32_t &uint32Result, uint32_t defaultValue = 0);
+  nsresult RowCellColumnToUInt64(nsIMdbRow *row, mdb_token columnToken, uint64_t *uint64Result, uint64_t defaultValue = 0);
+  nsresult RowCellColumnToMime2DecodedString(nsIMdbRow *row, mdb_token columnToken, nsAString &resultStr);
+  nsresult RowCellColumnToCollationKey(nsIMdbRow *row, mdb_token columnToken, uint8_t **result, uint32_t *len);
+  nsresult RowCellColumnToConstCharPtr(nsIMdbRow *row, mdb_token columnToken, const char **ptr);
+  nsresult RowCellColumnToAddressCollationKey(nsIMdbRow *row, mdb_token colToken, uint8_t **result, uint32_t *len);
+
+  nsresult GetEffectiveCharset(nsIMdbRow *row, nsACString &resultCharset);
+
+  // these methods take the property name as a string, not a token.
+  // they should be used when the properties aren't accessed a lot
+  nsresult        GetProperty(nsIMdbRow *row, const char *propertyName, char **result);
+  nsresult        SetProperty(nsIMdbRow *row, const char *propertyName, const char *propertyVal);
+  nsresult        GetPropertyAsNSString(nsIMdbRow *row, const char *propertyName, nsAString &result);
+  nsresult        SetPropertyFromNSString(nsIMdbRow *row, const char *propertyName, const nsAString &propertyVal);
+  nsresult        GetUint32Property(nsIMdbRow *row, const char *propertyName, uint32_t *result, uint32_t defaultValue = 0);
+  nsresult        GetUint64Property(nsIMdbRow *row, const char *propertyName, uint64_t *result, uint64_t defaultValue = 0);
+  nsresult        SetUint32Property(nsIMdbRow *row, const char *propertyName, uint32_t propertyVal);
+  nsresult        SetUint64Property(nsIMdbRow *row, const char *propertyName, uint64_t propertyVal);
+  nsresult        GetBooleanProperty(nsIMdbRow *row, const char *propertyName,
+                                     bool *result, bool defaultValue = false);
+  nsresult        SetBooleanProperty(nsIMdbRow *row, const char *propertyName,
+                                    bool propertyVal);
+  // helper function for once we have the token.
+  nsresult        SetNSStringPropertyWithToken(nsIMdbRow *row, mdb_token aProperty, const nsAString &propertyStr);
+
+  // helper functions to put values in cells for the passed-in row
+  nsresult        UInt32ToRowCellColumn(nsIMdbRow *row, mdb_token columnToken, uint32_t value);
+  nsresult        CharPtrToRowCellColumn(nsIMdbRow *row, mdb_token columnToken, const char *charPtr);
+  nsresult        RowCellColumnToCharPtr(nsIMdbRow *row, mdb_token columnToken, char **result);
+  nsresult        UInt64ToRowCellColumn(nsIMdbRow *row, mdb_token columnToken, uint64_t value);
+
+  // helper functions to copy an nsString to a yarn, int32 to yarn, and vice versa.
+  static struct mdbYarn *nsStringToYarn(struct mdbYarn *yarn, const nsAString &str);
+  static struct mdbYarn *UInt32ToYarn(struct mdbYarn *yarn, uint32_t i);
+  static struct mdbYarn *UInt64ToYarn(struct mdbYarn *yarn, uint64_t i);
+  static void YarnTonsString(struct mdbYarn *yarn, nsAString &str);
+  static void YarnTonsCString(struct mdbYarn *yarn, nsACString &str);
+  static void YarnToUInt32(struct mdbYarn *yarn, uint32_t *i);
+  static void YarnToUInt64(struct mdbYarn *yarn, uint64_t *i);
+
+#ifdef DEBUG
+  virtual nsresult DumpContents();
+  nsresult DumpThread(nsMsgKey threadId);
+  nsresult DumpMsgChildren(nsIMsgDBHdr *msgHdr);
+#endif
+
+  friend class nsMsgHdr;  // use this to get access to cached tokens for hdr fields
+  friend class nsMsgThread;  // use this to get access to cached tokens for hdr fields
+  friend class nsMsgDBEnumerator;
+  friend class nsMsgDBThreadEnumerator;
+protected:
+  virtual ~nsMsgDatabase();
+
+  // prefs stuff - in future, we might want to cache the prefs interface
+  nsresult        GetBoolPref(const char *prefName, bool *result);
+  nsresult        GetIntPref(const char *prefName, int32_t *result);
+  virtual void    GetGlobalPrefs();
+    // retrieval methods
+  nsIMsgThread *  GetThreadForReference(nsCString &msgID, nsIMsgDBHdr **pMsgHdr);
+  nsIMsgThread *  GetThreadForSubject(nsCString &subject);
+  nsIMsgThread *  GetThreadForMessageId(nsCString &msgId);
+  nsIMsgThread *  GetThreadForThreadId(nsMsgKey threadId);
+  nsMsgHdr     *  GetMsgHdrForReference(nsCString &reference);
+  nsIMsgDBHdr  *  GetMsgHdrForSubject(nsCString &subject);
+  // threading interfaces
+  virtual nsresult CreateNewThread(nsMsgKey key, const char *subject, nsMsgThread **newThread);
+  virtual bool    ThreadBySubjectWithoutRe();
+  virtual bool    UseStrictThreading();
+  virtual bool    UseCorrectThreading();
+  virtual nsresult ThreadNewHdr(nsMsgHdr* hdr, bool &newThread);
+  virtual nsresult AddNewThread(nsMsgHdr *msgHdr);
+  virtual nsresult AddToThread(nsMsgHdr *newHdr, nsIMsgThread *thread, nsIMsgDBHdr *pMsgHdr, bool threadInThread);
+
+  static PRTime gLastUseTime; // global last use time
+  PRTime m_lastUseTime;       // last use time for this db
+  // inline to make instrumentation as cheap as possible
+  inline void RememberLastUseTime() {gLastUseTime = m_lastUseTime = PR_Now();}
+
+  bool    MatchDbName(nsIFile *dbName);  // returns TRUE if they match
+
+  // Flag handling routines
+  virtual nsresult SetKeyFlag(nsMsgKey key, bool set, uint32_t flag,
+                              nsIDBChangeListener *instigator = NULL);
+  virtual nsresult SetMsgHdrFlag(nsIMsgDBHdr *msgHdr, bool set, uint32_t flag,
+                                 nsIDBChangeListener *instigator);
+
+  virtual bool    SetHdrFlag(nsIMsgDBHdr *, bool bSet, nsMsgMessageFlagType flag);
+  virtual bool    SetHdrReadFlag(nsIMsgDBHdr *, bool pRead);
+  virtual uint32_t GetStatusFlags(nsIMsgDBHdr *msgHdr, uint32_t origFlags);
+  // helper function which doesn't involve thread object
+
+  virtual nsresult RemoveHeaderFromDB(nsMsgHdr *msgHdr);
+  virtual nsresult RemoveHeaderFromThread(nsMsgHdr *msgHdr);
+  virtual nsresult AdjustExpungedBytesOnDelete(nsIMsgDBHdr *msgHdr);
+
+  nsCOMPtr <nsICollation> m_collationKeyGenerator;
+  nsCOMPtr <nsIMimeConverter> m_mimeConverter;
+  nsCOMPtr <nsIMsgRetentionSettings> m_retentionSettings;
+  nsCOMPtr <nsIMsgDownloadSettings> m_downloadSettings;
+
+  nsresult PurgeMessagesOlderThan(uint32_t daysToKeepHdrs,
+                                  bool applyToFlaggedMessages,
+                                  nsIMutableArray *hdrsToDelete);
+  nsresult PurgeExcessMessages(uint32_t numHeadersToKeep,
+                               bool applyToFlaggedMessages,
+                               nsIMutableArray *hdrsToDelete);
+
+  // mdb bookkeeping stuff
+  virtual nsresult      InitExistingDB();
+  virtual nsresult      InitNewDB();
+  virtual nsresult      InitMDBInfo();
+
+  nsCOMPtr <nsIMsgFolder> m_folder;
+  nsDBFolderInfo      *m_dbFolderInfo;
+  nsMsgKey      m_nextPseudoMsgKey;
+  nsIMdbEnv     *m_mdbEnv;  // to be used in all the db calls.
+  nsIMdbStore   *m_mdbStore;
+  nsIMdbTable   *m_mdbAllMsgHeadersTable;
+  nsIMdbTable   *m_mdbAllThreadsTable;
+
+  // Used for asynchronous db opens. If non-null, we're still opening
+  // the underlying mork database. If null, the db has been completely opened.
+  nsCOMPtr<nsIMdbThumb> m_thumb;
+  // used to remember the args to Open for async open.
+  bool m_create;
+  bool m_leaveInvalidDB;
+
+  nsCString     m_dbName;
+  nsTArray<nsMsgKey> m_newSet;  // new messages since last open.
+  bool          m_mdbTokensInitialized;
+  nsTObserverArray<nsCOMPtr<nsIDBChangeListener> > m_ChangeListeners;
+  mdb_token     m_hdrRowScopeToken;
+  mdb_token     m_threadRowScopeToken;
+  mdb_token     m_hdrTableKindToken;
+  mdb_token     m_threadTableKindToken;
+  mdb_token     m_allThreadsTableKindToken;
+  mdb_token     m_subjectColumnToken;
+  mdb_token     m_senderColumnToken;
+  mdb_token     m_messageIdColumnToken;
+  mdb_token     m_referencesColumnToken;
+  mdb_token     m_recipientsColumnToken;
+  mdb_token     m_dateColumnToken;
+  mdb_token     m_messageSizeColumnToken;
+  mdb_token     m_flagsColumnToken;
+  mdb_token     m_priorityColumnToken;
+  mdb_token     m_labelColumnToken;
+  mdb_token     m_statusOffsetColumnToken;
+  mdb_token     m_numLinesColumnToken;
+  mdb_token     m_ccListColumnToken;
+  mdb_token     m_bccListColumnToken;
+  mdb_token     m_threadFlagsColumnToken;
+  mdb_token     m_threadIdColumnToken;
+  mdb_token     m_threadChildrenColumnToken;
+  mdb_token     m_threadUnreadChildrenColumnToken;
+  mdb_token     m_messageThreadIdColumnToken;
+  mdb_token     m_threadSubjectColumnToken;
+  mdb_token     m_messageCharSetColumnToken;
+  mdb_token     m_threadParentColumnToken;
+  mdb_token     m_threadRootKeyColumnToken;
+  mdb_token     m_threadNewestMsgDateColumnToken;
+  mdb_token     m_offlineMsgOffsetColumnToken;
+  mdb_token     m_offlineMessageSizeColumnToken;
+
+  // header caching stuff - MRU headers, keeps them around in memory
+  nsresult      AddHdrToCache(nsIMsgDBHdr *hdr, nsMsgKey key);
+  nsresult      ClearHdrCache(bool reInit);
+  nsresult      RemoveHdrFromCache(nsIMsgDBHdr *hdr, nsMsgKey key);
+  // all headers currently instantiated, doesn't hold refs
+  // these get added when msg hdrs get constructed, and removed when they get destroyed.
+  nsresult      GetHdrFromUseCache(nsMsgKey key, nsIMsgDBHdr* *result);
+  nsresult      AddHdrToUseCache(nsIMsgDBHdr *hdr, nsMsgKey key);
+  nsresult      ClearUseHdrCache();
+  nsresult      RemoveHdrFromUseCache(nsIMsgDBHdr *hdr, nsMsgKey key);
+
+  // not-reference holding array of threads we've handed out.
+  // If a db goes away, it will clean up the outstanding threads.
+  // We use an nsTArray because we don't expect to ever have very many
+  // of these, rarely more than 5.
+  nsTArray<nsMsgThread *> m_threads;
+  // Clear outstanding thread objects
+  void ClearThreads();
+  nsMsgThread *FindExistingThread(nsMsgKey threadId);
+
+  mdb_pos       FindInsertIndexInSortedTable(nsIMdbTable *table, mdb_id idToInsert);
+
+  void          ClearCachedObjects(bool dbGoingAway);
+  void          ClearEnumerators();
+  // all instantiated headers, but doesn't hold refs.
+  PLDHashTable  *m_headersInUse;
+  static PLDHashNumber HashKey(const void* aKey);
+  static bool MatchEntry(const PLDHashEntryHdr* aEntry, const void* aKey);
+  static void MoveEntry(PLDHashTable* aTable, const PLDHashEntryHdr* aFrom, PLDHashEntryHdr* aTo);
+  static void ClearEntry(PLDHashTable* aTable, PLDHashEntryHdr* aEntry);
+  static PLDHashTableOps gMsgDBHashTableOps;
+  struct MsgHdrHashElement : public PLDHashEntryHdr {
+    nsMsgKey       mKey;
+    nsIMsgDBHdr     *mHdr;
+  };
+  PLDHashTable  *m_cachedHeaders;
+  bool          m_bCacheHeaders;
+  nsMsgKey  m_cachedThreadId;
+  nsCOMPtr <nsIMsgThread> m_cachedThread;
+  nsCOMPtr<nsIMdbFactory> mMdbFactory;
+
+  // Message reference hash table
+  static PLDHashTableOps gRefHashTableOps;
+  struct RefHashElement : public PLDHashEntryHdr {
+    const char     *mRef;       // Hash entry key, must come first
+    nsMsgKey        mThreadId;
+    uint32_t        mCount;
+  };
+  PLDHashTable *m_msgReferences;
+  nsresult GetRefFromHash(nsCString &reference, nsMsgKey *threadId);
+  nsresult AddRefToHash(nsCString &reference, nsMsgKey threadId);
+  nsresult AddMsgRefsToHash(nsIMsgDBHdr *msgHdr);
+  nsresult RemoveRefFromHash(nsCString &reference);
+  nsresult RemoveMsgRefsFromHash(nsIMsgDBHdr *msgHdr);
+  nsresult InitRefHash();
+
+  // not-reference holding array of enumerators we've handed out.
+  // If a db goes away, it will clean up the outstanding enumerators.
+  nsTArray<nsMsgDBEnumerator *> m_enumerators;
+
+  // Memory reporter details
+public:
+  static size_t HeaderHashSizeOf(PLDHashEntryHdr *hdr,
+                                 mozilla::MallocSizeOf aMallocSizeOf,
+                                 void *arg);
+  virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
+  virtual size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
+  {
+    return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+  }
+private:
+  uint32_t m_cacheSize;
+  RefPtr<mozilla::mailnews::MsgDBReporter> mMemReporter;
+};
+
+class nsMsgRetentionSettings : public nsIMsgRetentionSettings
+{
+public:
+  nsMsgRetentionSettings();
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGRETENTIONSETTINGS
+protected:
+  virtual ~nsMsgRetentionSettings();
+  nsMsgRetainByPreference m_retainByPreference;
+  uint32_t                m_daysToKeepHdrs;
+  uint32_t                m_numHeadersToKeep;
+  bool                    m_useServerDefaults;
+  bool                    m_cleanupBodiesByDays;
+  uint32_t                m_daysToKeepBodies;
+  bool                    m_applyToFlaggedMessages;
+};
+
+class nsMsgDownloadSettings : public nsIMsgDownloadSettings
+{
+public:
+  nsMsgDownloadSettings();
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGDOWNLOADSETTINGS
+protected:
+  virtual ~nsMsgDownloadSettings();
+  bool m_useServerDefaults;
+  bool m_downloadUnreadOnly;
+  bool m_downloadByDate;
+  int32_t m_ageLimitOfMsgsToDownload;
+};
+
+#endif

mailnews/db/msgdb/public/nsMsgHdr.h

@@ -0,0 +1,86 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMsgHdr_H
+#define _nsMsgHdr_H
+
+#include "mozilla/MemoryReporting.h"
+#include "nsIMsgHdr.h"
+#include "nsStringGlue.h"
+#include "MailNewsTypes.h"
+#include "mdb.h"
+#include "nsTArray.h"
+
+class nsMsgDatabase;
+class nsCString;
+class nsIMsgThread;
+
+class nsMsgHdr : public nsIMsgDBHdr {
+public:
+    NS_DECL_NSIMSGDBHDR
+	friend class nsMsgDatabase;
+    friend class nsMsgPropertyEnumerator; // accesses m_mdb
+    ////////////////////////////////////////////////////////////////////////////
+    ////////////////////////////////////////////////////////////////////////////
+    // nsMsgHdr methods:
+    nsMsgHdr(nsMsgDatabase *db, nsIMdbRow *dbRow);
+
+    virtual nsresult    GetRawFlags(uint32_t *result);
+    void                Init();
+    virtual nsresult    InitCachedValues();
+    virtual nsresult    InitFlags();
+    void                ClearCachedValues() {m_initedValues = 0;}
+
+    NS_DECL_ISUPPORTS
+
+    nsIMdbRow   *GetMDBRow() {return m_mdbRow;}
+    bool        IsParentOf(nsIMsgDBHdr *possibleChild);
+    bool        IsAncestorOf(nsIMsgDBHdr *possibleChild);
+    bool        IsAncestorKilled(uint32_t ancestorsToCheck);
+    void        ReparentInThread(nsIMsgThread *thread);
+
+    size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOfFun) const
+    {
+      return m_references.ShallowSizeOfExcludingThis(aMallocSizeOfFun);
+    }
+    size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOfFun) const
+    {
+      return aMallocSizeOfFun(this) + SizeOfExcludingThis(aMallocSizeOfFun);
+    }
+
+protected:
+  virtual ~nsMsgHdr();
+    nsresult SetStringColumn(const char *str, mdb_token token);
+    nsresult SetUInt32Column(uint32_t value, mdb_token token);
+    nsresult GetUInt32Column(mdb_token token, uint32_t *pvalue, uint32_t defaultValue = 0);
+    nsresult SetUInt64Column(uint64_t value, mdb_token token);
+    nsresult GetUInt64Column(mdb_token token, uint64_t *pvalue, uint64_t defaultValue = 0);
+
+    // reference and threading stuff.
+    nsresult	ParseReferences(const char *references);
+    const char* GetNextReference(const char *startNextRef, nsCString &reference,
+                                 bool acceptNonDelimitedReferences);
+
+    nsMsgKey	m_threadId; 
+    nsMsgKey	m_messageKey; 	//news: article number, mail mbox offset, imap uid...
+    nsMsgKey	m_threadParent;	// message this is a reply to, in thread.
+    PRTime      m_date;                         
+    uint32_t    m_messageSize;	// lines for news articles, bytes for mail messages
+    uint32_t    m_statusOffset;	// offset in a local mail message of the mozilla status hdr
+    uint32_t    m_flags;
+    // avoid parsing references every time we want one
+    nsTArray<nsCString> m_references;
+    nsMsgPriorityValue  m_priority;
+
+    // nsMsgHdrs will have to know what db and row they belong to, since they are really
+    // just a wrapper around the msg row in the mdb. This could cause problems,
+    // though I hope not.
+    nsMsgDatabase *m_mdb;
+    nsIMdbRow     *m_mdbRow;
+    uint32_t      m_initedValues;
+};
+
+#endif
+

mailnews/db/msgdb/public/nsMsgThread.h

@@ -0,0 +1,63 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsMsgThread_H
+#define _nsMsgThread_H
+
+#include "nsAutoPtr.h"
+#include "nsIMsgThread.h"
+#include "nsStringGlue.h"
+#include "MailNewsTypes.h"
+#include "mdb.h"
+
+class nsIMdbTable;
+class nsIMsgDBHdr;
+class nsMsgDatabase;
+
+class nsMsgThread : public nsIMsgThread {
+public:
+  nsMsgThread();
+  nsMsgThread(nsMsgDatabase *db, nsIMdbTable *table);
+
+  friend class nsMsgThreadEnumerator;
+  friend class nsMsgDatabase;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGTHREAD
+
+  RefPtr<nsMsgDatabase> m_mdbDB;
+
+protected:
+  virtual ~nsMsgThread();
+
+  void                  Init();
+  void                  Clear();
+  virtual nsresult      InitCachedValues();
+  nsresult              ChangeChildCount(int32_t delta);
+  nsresult              ChangeUnreadChildCount(int32_t delta);
+  nsresult              RemoveChild(nsMsgKey msgKey);
+  nsresult              SetThreadRootKey(nsMsgKey threadRootKey);
+  nsresult              GetChildHdrForKey(nsMsgKey desiredKey, 
+                                          nsIMsgDBHdr **result, int32_t *resultIndex); 
+  nsresult              RerootThread(nsIMsgDBHdr *newParentOfOldRoot, nsIMsgDBHdr *oldRoot, nsIDBChangeAnnouncer *announcer);
+  nsresult              ReparentChildrenOf(nsMsgKey oldParent, nsMsgKey newParent, nsIDBChangeAnnouncer *announcer);
+  
+  nsresult              ReparentNonReferenceChildrenOf(nsIMsgDBHdr *topLevelHdr, nsMsgKey newParentKey,
+                                                       nsIDBChangeAnnouncer *announcer);
+  nsresult              ReparentMsgsWithInvalidParent(uint32_t numChildren, nsMsgKey threadParentKey);
+
+  nsMsgKey              m_threadKey; 
+  uint32_t              m_numChildren;
+  uint32_t              m_numUnreadChildren;
+  uint32_t              m_flags;
+  nsCOMPtr<nsIMdbTable> m_mdbTable;
+  nsCOMPtr<nsIMdbRow>   m_metaRow;
+  bool                  m_cachedValuesInitialized;
+  nsMsgKey              m_threadRootKey;
+  uint32_t              m_newestMsgDate;
+};
+
+#endif
+

mailnews/db/msgdb/public/nsNewsDatabase.h

@@ -0,0 +1,57 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef _nsNewsDatabase_H_
+#define _nsNewsDatabase_H_
+
+#include "mozilla/Attributes.h"
+#include "nsMsgDatabase.h"
+#include "nsINewsDatabase.h"
+#include "nsTArray.h"
+
+class nsIDBChangeListener;
+class MSG_RetrieveArtInfo;
+class MSG_PurgeInfo;
+// news group database
+
+class nsNewsDatabase : public nsMsgDatabase , public nsINewsDatabase
+{
+public:
+  nsNewsDatabase();
+
+  NS_DECL_ISUPPORTS_INHERITED 
+  NS_DECL_NSINEWSDATABASE
+
+  NS_IMETHOD Close(bool forceCommit) override;
+  NS_IMETHOD ForceClosed() override;
+  NS_IMETHOD Commit(nsMsgDBCommit commitType) override;
+  virtual uint32_t GetCurVersion() override;
+
+  // methods to get and set docsets for ids.
+  NS_IMETHOD  IsRead(nsMsgKey key, bool *pRead) override;
+  virtual nsresult  IsHeaderRead(nsIMsgDBHdr *msgHdr, bool *pRead) override;
+
+  NS_IMETHOD         GetHighWaterArticleNum(nsMsgKey *key) override;
+  NS_IMETHOD         GetLowWaterArticleNum(nsMsgKey *key) override;
+  NS_IMETHOD         MarkAllRead(uint32_t *aNumMarked, nsMsgKey **thoseMarked) override;
+
+  virtual nsresult    ExpireUpTo(nsMsgKey expireKey);
+  virtual nsresult    ExpireRange(nsMsgKey startRange, nsMsgKey endRange);
+ 
+  virtual bool        SetHdrReadFlag(nsIMsgDBHdr *msgHdr, bool bRead) override;
+ 
+  virtual nsresult  AdjustExpungedBytesOnDelete(nsIMsgDBHdr *msgHdr) override;
+  nsresult          SyncWithReadSet();
+  
+  NS_IMETHOD GetDefaultViewFlags(nsMsgViewFlagsTypeValue *aDefaultViewFlags) override;
+  NS_IMETHOD GetDefaultSortType(nsMsgViewSortTypeValue *aDefaultSortType) override;
+  NS_IMETHOD GetDefaultSortOrder(nsMsgViewSortOrderValue *aDefaultSortOrder) override;
+
+protected:
+  virtual ~nsNewsDatabase();
+  // this is owned by the nsNewsFolder, which lives longer than the db.
+  nsMsgKeySet           *m_readSet;
+};
+
+#endif

mailnews/db/msgdb/src/moz.build

@@ -0,0 +1,18 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SOURCES += [
+    'nsDBFolderInfo.cpp',
+    'nsImapMailDatabase.cpp',
+    'nsMailDatabase.cpp',
+    'nsMsgDatabase.cpp',
+    'nsMsgHdr.cpp',
+    'nsMsgOfflineImapOperation.cpp',
+    'nsMsgThread.cpp',
+    'nsNewsDatabase.cpp',
+]
+
+FINAL_LIBRARY = 'mail'
+

mailnews/db/msgdb/src/nsDBFolderInfo.cpp

@@ -0,0 +1,977 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsDBFolderInfo.h"
+#include "nsMsgDatabase.h"
+#include "nsMsgFolderFlags.h"
+#include "nsIPrefService.h"
+#include "nsIPrefBranch.h"
+#include "nsIPrefLocalizedString.h"
+#include "nsIObserver.h"
+#include "nsIObserverService.h"
+#include "nsIMsgDBView.h"
+#include "nsServiceManagerUtils.h"
+#include "nsImapCore.h"
+#include "mozilla/Services.h"
+
+static const char *kDBFolderInfoScope = "ns:msg:db:row:scope:dbfolderinfo:all";
+static const char *kDBFolderInfoTableKind = "ns:msg:db:table:kind:dbfolderinfo";
+
+struct mdbOid gDBFolderInfoOID;
+
+static const char * kNumMessagesColumnName ="numMsgs";
+// have to leave this as numNewMsgs even though it's numUnread Msgs
+static const char * kNumUnreadMessagesColumnName = "numNewMsgs";
+static const char * kFlagsColumnName = "flags";
+static const char * kFolderSizeColumnName = "folderSize";
+static const char * kExpungedBytesColumnName = "expungedBytes";
+static const char * kFolderDateColumnName = "folderDate";
+static const char * kHighWaterMessageKeyColumnName = "highWaterKey";
+
+static const char * kImapUidValidityColumnName = "UIDValidity";
+static const char * kTotalPendingMessagesColumnName = "totPendingMsgs";
+static const char * kUnreadPendingMessagesColumnName = "unreadPendingMsgs";
+static const char * kMailboxNameColumnName = "mailboxName";
+static const char * kKnownArtsSetColumnName = "knownArts";
+static const char * kExpiredMarkColumnName = "expiredMark";
+static const char * kVersionColumnName = "version";
+static const char * kCharacterSetColumnName = "charSet";
+static const char * kCharacterSetOverrideColumnName = "charSetOverride";
+static const char * kLocaleColumnName = "locale";
+
+
+#define kMAILNEWS_VIEW_DEFAULT_CHARSET        "mailnews.view_default_charset"
+#define kMAILNEWS_DEFAULT_CHARSET_OVERRIDE    "mailnews.force_charset_override"
+static nsCString* gDefaultCharacterSet = nullptr;
+static bool       gDefaultCharacterOverride;
+static nsIObserver *gFolderCharsetObserver = nullptr;
+
+// observer for charset related preference notification
+class nsFolderCharsetObserver : public nsIObserver {
+
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIOBSERVER
+
+  nsFolderCharsetObserver() { }
+private:
+  virtual ~nsFolderCharsetObserver() {}
+};
+
+NS_IMPL_ISUPPORTS(nsFolderCharsetObserver, nsIObserver)
+
+NS_IMETHODIMP nsFolderCharsetObserver::Observe(nsISupports *aSubject, const char *aTopic, const char16_t *someData)
+{
+  nsresult rv;
+
+  nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv);
+  if (NS_FAILED(rv)) return rv;
+
+  nsCOMPtr<nsIPrefBranch> prefBranch;
+  rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID))
+  {
+    nsDependentString prefName(someData);
+
+    if (prefName.EqualsLiteral(kMAILNEWS_VIEW_DEFAULT_CHARSET))
+    {
+      nsCOMPtr<nsIPrefLocalizedString> pls;
+      rv = prefBranch->GetComplexValue(kMAILNEWS_VIEW_DEFAULT_CHARSET,
+                      NS_GET_IID(nsIPrefLocalizedString), getter_AddRefs(pls));
+      if (NS_SUCCEEDED(rv))
+      {
+        nsString ucsval;
+        pls->ToString(getter_Copies(ucsval));
+        if (!ucsval.IsEmpty())
+        {
+          if (gDefaultCharacterSet)
+            CopyUTF16toUTF8(ucsval, *gDefaultCharacterSet);
+        }
+      }
+    }
+    else if (prefName.EqualsLiteral(kMAILNEWS_DEFAULT_CHARSET_OVERRIDE))
+    {
+      rv = prefBranch->GetBoolPref(kMAILNEWS_DEFAULT_CHARSET_OVERRIDE, &gDefaultCharacterOverride);
+    }
+  }
+  else if (!strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID))
+  {
+    rv = prefBranch->RemoveObserver(kMAILNEWS_VIEW_DEFAULT_CHARSET, this);
+    rv = prefBranch->RemoveObserver(kMAILNEWS_DEFAULT_CHARSET_OVERRIDE, this);
+    NS_IF_RELEASE(gFolderCharsetObserver);
+    delete gDefaultCharacterSet;
+    gDefaultCharacterSet = nullptr;
+  }
+  return rv;
+}
+
+
+NS_IMPL_ADDREF(nsDBFolderInfo)
+NS_IMPL_RELEASE(nsDBFolderInfo)
+
+NS_IMETHODIMP
+nsDBFolderInfo::QueryInterface(REFNSIID iid, void** result)
+{
+  if (! result)
+    return NS_ERROR_NULL_POINTER;
+
+  *result = nullptr;
+  if(iid.Equals(NS_GET_IID(nsIDBFolderInfo)) ||
+    iid.Equals(NS_GET_IID(nsISupports)))
+  {
+    *result = static_cast<nsIDBFolderInfo*>(this);
+    AddRef();
+    return NS_OK;
+  }
+  return NS_NOINTERFACE;
+}
+
+
+nsDBFolderInfo::nsDBFolderInfo(nsMsgDatabase *mdb)
+                  : m_flags(0),
+                    m_expiredMark(0),
+                    m_expiredMarkColumnToken(0)
+{
+  m_mdbTable = NULL;
+  m_mdbRow = NULL;
+  m_version = 1; // for upgrading...
+  m_IMAPHierarchySeparator = 0; // imap path separator
+  // mail only (for now)
+  m_folderSize = 0;
+  m_folderDate = 0;
+  m_expungedBytes = 0; // sum of size of deleted messages in folder
+  m_highWaterMessageKey = 0;
+
+  m_numUnreadMessages = 0;
+  m_numMessages = 0;
+  // IMAP only
+  m_ImapUidValidity = kUidUnknown;
+  m_totalPendingMessages =0;
+  m_unreadPendingMessages = 0;
+
+  m_mdbTokensInitialized = false;
+  m_charSetOverride = false;
+
+  if (!gFolderCharsetObserver)
+  {
+    nsresult rv;
+    nsCOMPtr<nsIPrefService> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID, &rv);
+    nsCOMPtr<nsIPrefBranch> prefBranch;
+    if (NS_SUCCEEDED(rv))
+    {
+      rv = prefs->GetBranch(nullptr, getter_AddRefs(prefBranch));
+    }
+    if (NS_SUCCEEDED(rv))
+    {
+      nsCOMPtr<nsIPrefLocalizedString> pls;
+      rv = prefBranch->GetComplexValue(kMAILNEWS_VIEW_DEFAULT_CHARSET,
+        NS_GET_IID(nsIPrefLocalizedString), getter_AddRefs(pls));
+      if (NS_SUCCEEDED(rv))
+      {
+        nsString ucsval;
+        pls->ToString(getter_Copies(ucsval));
+        if (!ucsval.IsEmpty())
+        {
+          if (!gDefaultCharacterSet)
+            gDefaultCharacterSet = new nsCString;
+
+          if (gDefaultCharacterSet)
+            CopyUTF16toUTF8(ucsval, *gDefaultCharacterSet);
+        }
+      }
+      rv = prefBranch->GetBoolPref(kMAILNEWS_DEFAULT_CHARSET_OVERRIDE, &gDefaultCharacterOverride);
+
+      gFolderCharsetObserver = new nsFolderCharsetObserver();
+      NS_ASSERTION(gFolderCharsetObserver, "failed to create observer");
+
+      // register prefs callbacks
+      if (gFolderCharsetObserver)
+      {
+        NS_ADDREF(gFolderCharsetObserver);
+        rv = prefBranch->AddObserver(kMAILNEWS_VIEW_DEFAULT_CHARSET, gFolderCharsetObserver, false);
+        rv = prefBranch->AddObserver(kMAILNEWS_DEFAULT_CHARSET_OVERRIDE, gFolderCharsetObserver, false);
+
+        // also register for shutdown
+        nsCOMPtr<nsIObserverService> observerService =
+          mozilla::services::GetObserverService();
+        if (observerService)
+        {
+          rv = observerService->AddObserver(gFolderCharsetObserver, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false);
+        }
+      }
+    }
+  }
+
+  m_mdb = mdb;
+  if (mdb)
+  {
+    nsresult err;
+
+    //		mdb->AddRef();
+    err = m_mdb->GetStore()->StringToToken(mdb->GetEnv(), kDBFolderInfoScope, &m_rowScopeToken);
+    if (NS_SUCCEEDED(err))
+    {
+      err = m_mdb->GetStore()->StringToToken(mdb->GetEnv(), kDBFolderInfoTableKind, &m_tableKindToken);
+      if (NS_SUCCEEDED(err))
+      {
+        gDBFolderInfoOID.mOid_Scope = m_rowScopeToken;
+        gDBFolderInfoOID.mOid_Id = 1;
+      }
+    }
+    InitMDBInfo();
+  }
+}
+
+nsDBFolderInfo::~nsDBFolderInfo()
+{
+  // nsMsgDatabase strictly owns nsDBFolderInfo, so don't ref-count db.
+  ReleaseExternalReferences();
+}
+
+// Release any objects we're holding onto. This needs to be safe
+// to call multiple times.
+void nsDBFolderInfo::ReleaseExternalReferences()
+{
+  if (m_mdb)
+  {
+    if (m_mdbTable)
+    {
+      NS_RELEASE(m_mdbTable);
+      m_mdbTable = nullptr;
+    }
+    if (m_mdbRow)
+    {
+      NS_RELEASE(m_mdbRow);
+      m_mdbRow = nullptr;
+    }
+    m_mdb = nullptr;
+  }
+}
+
+// this routine sets up a new db to know about the dbFolderInfo stuff...
+nsresult nsDBFolderInfo::AddToNewMDB()
+{
+  nsresult ret = NS_OK;
+  if (m_mdb && m_mdb->GetStore())
+  {
+    nsIMdbStore *store = m_mdb->GetStore();
+    // create the unique table for the dbFolderInfo.
+    nsresult err = store->NewTable(m_mdb->GetEnv(), m_rowScopeToken,
+      m_tableKindToken, true, nullptr, &m_mdbTable);
+
+    // create the singleton row for the dbFolderInfo.
+    err  = store->NewRowWithOid(m_mdb->GetEnv(),
+      &gDBFolderInfoOID, &m_mdbRow);
+
+    // add the row to the singleton table.
+    if (m_mdbRow && NS_SUCCEEDED(err))
+      err = m_mdbTable->AddRow(m_mdb->GetEnv(), m_mdbRow);
+
+    ret = err;	// what are we going to do about nsresult's?
+  }
+  return ret;
+}
+
+nsresult nsDBFolderInfo::InitFromExistingDB()
+{
+  nsresult ret = NS_OK;
+  if (m_mdb && m_mdb->GetStore())
+  {
+    nsIMdbStore *store = m_mdb->GetStore();
+    if (store)
+    {
+      mdb_pos		rowPos;
+      mdb_count outTableCount; // current number of such tables
+      mdb_bool mustBeUnique; // whether port can hold only one of these
+      mdb_bool hasOid;
+      ret = store->GetTableKind(m_mdb->GetEnv(), m_rowScopeToken, m_tableKindToken, &outTableCount,
+        &mustBeUnique, &m_mdbTable);
+      //			NS_ASSERTION(mustBeUnique && outTableCount == 1, "only one global db info allowed");
+
+      if (m_mdbTable)
+      {
+        // find singleton row for global info.
+        ret = m_mdbTable->HasOid(m_mdb->GetEnv(), &gDBFolderInfoOID, &hasOid);
+        if (NS_SUCCEEDED(ret))
+        {
+          nsIMdbTableRowCursor *rowCursor;
+          rowPos = -1;
+          ret= m_mdbTable->GetTableRowCursor(m_mdb->GetEnv(), rowPos, &rowCursor);
+          if (NS_SUCCEEDED(ret))
+          {
+            ret = rowCursor->NextRow(m_mdb->GetEnv(), &m_mdbRow, &rowPos);
+            NS_RELEASE(rowCursor);
+            if (!m_mdbRow)
+              ret = NS_ERROR_FAILURE;
+            if (NS_SUCCEEDED(ret))
+              LoadMemberVariables();
+          }
+        }
+      }
+      else
+        ret = NS_ERROR_FAILURE;
+    }
+  }
+  return ret;
+}
+
+nsresult nsDBFolderInfo::InitMDBInfo()
+{
+  nsresult ret = NS_OK;
+  if (!m_mdbTokensInitialized && m_mdb && m_mdb->GetStore())
+  {
+    nsIMdbStore *store = m_mdb->GetStore();
+    nsIMdbEnv	*env = m_mdb->GetEnv();
+
+    store->StringToToken(env,  kNumMessagesColumnName, &m_numMessagesColumnToken);
+    store->StringToToken(env,  kNumUnreadMessagesColumnName, &m_numUnreadMessagesColumnToken);
+    store->StringToToken(env,  kFlagsColumnName, &m_flagsColumnToken);
+    store->StringToToken(env,  kFolderSizeColumnName, &m_folderSizeColumnToken);
+    store->StringToToken(env,  kExpungedBytesColumnName, &m_expungedBytesColumnToken);
+    store->StringToToken(env,  kFolderDateColumnName, &m_folderDateColumnToken);
+
+    store->StringToToken(env,  kHighWaterMessageKeyColumnName, &m_highWaterMessageKeyColumnToken);
+    store->StringToToken(env,  kMailboxNameColumnName, &m_mailboxNameColumnToken);
+
+    store->StringToToken(env,  kImapUidValidityColumnName, &m_imapUidValidityColumnToken);
+    store->StringToToken(env,  kTotalPendingMessagesColumnName, &m_totalPendingMessagesColumnToken);
+    store->StringToToken(env,  kUnreadPendingMessagesColumnName, &m_unreadPendingMessagesColumnToken);
+    store->StringToToken(env,  kExpiredMarkColumnName, &m_expiredMarkColumnToken);
+    store->StringToToken(env,  kVersionColumnName, &m_versionColumnToken);
+    m_mdbTokensInitialized  = true;
+  }
+
+  return ret;
+}
+
+nsresult nsDBFolderInfo::LoadMemberVariables()
+{
+  // it's really not an error for these properties to not exist...
+  GetInt32PropertyWithToken(m_numMessagesColumnToken, m_numMessages);
+  GetInt32PropertyWithToken(m_numUnreadMessagesColumnToken, m_numUnreadMessages);
+  GetInt32PropertyWithToken(m_flagsColumnToken, m_flags);
+  GetInt64PropertyWithToken(m_folderSizeColumnToken, m_folderSize);
+  GetUint32PropertyWithToken(m_folderDateColumnToken, m_folderDate);
+  GetInt32PropertyWithToken(m_imapUidValidityColumnToken, m_ImapUidValidity, kUidUnknown);
+  GetUint32PropertyWithToken(m_expiredMarkColumnToken, m_expiredMark);
+  GetInt64PropertyWithToken(m_expungedBytesColumnToken, m_expungedBytes);
+  GetUint32PropertyWithToken(m_highWaterMessageKeyColumnToken, m_highWaterMessageKey);
+  int32_t version;
+
+  GetInt32PropertyWithToken(m_versionColumnToken, version);
+  m_version = (uint16_t) version;
+  m_charSetOverride = gDefaultCharacterOverride;
+  uint32_t propertyValue;
+  nsresult rv = GetUint32Property(kCharacterSetOverrideColumnName, gDefaultCharacterOverride, &propertyValue);
+  if (NS_SUCCEEDED(rv))
+    m_charSetOverride = propertyValue;
+
+  m_mdb->GetProperty(m_mdbRow, kCharacterSetColumnName, getter_Copies(m_charSet));
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetVersion(uint32_t version)
+{
+  m_version = version;
+  return SetUint32PropertyWithToken(m_versionColumnToken, (uint32_t) m_version);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetVersion(uint32_t *version)
+{
+  *version = m_version;
+  return NS_OK;
+}
+
+
+nsresult nsDBFolderInfo::AdjustHighWater(nsMsgKey highWater, bool force)
+{
+  if (force || m_highWaterMessageKey < highWater)
+  {
+    m_highWaterMessageKey = highWater;
+    SetUint32PropertyWithToken(m_highWaterMessageKeyColumnToken, highWater);
+  }
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetHighWater(nsMsgKey highWater)
+{
+  return AdjustHighWater(highWater, true);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::OnKeyAdded(nsMsgKey aNewKey)
+{
+  return AdjustHighWater(aNewKey, false);
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetFolderSize(int64_t *size)
+{
+  NS_ENSURE_ARG_POINTER(size);
+  *size = m_folderSize;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetFolderSize(int64_t size)
+{
+  m_folderSize = size;
+  return SetInt64Property(kFolderSizeColumnName, m_folderSize);
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetFolderDate(uint32_t *folderDate)
+{
+  NS_ENSURE_ARG_POINTER(folderDate);
+  *folderDate = m_folderDate;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetFolderDate(uint32_t folderDate)
+{
+  m_folderDate = folderDate;
+  return SetUint32PropertyWithToken(m_folderDateColumnToken, folderDate);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetHighWater(nsMsgKey *result)
+{
+  // Sanity check highwater - if it gets too big, other code
+  // can fail. Look through last 100 messages to recalculate
+  // the highwater mark.
+  *result = m_highWaterMessageKey;
+  if (m_highWaterMessageKey > 0xFFFFFF00 && m_mdb)
+  {
+    nsCOMPtr <nsISimpleEnumerator> hdrs;
+    nsresult rv = m_mdb->ReverseEnumerateMessages(getter_AddRefs(hdrs));
+    if (NS_FAILED(rv))
+      return rv;
+    bool hasMore = false;
+    nsCOMPtr<nsIMsgDBHdr> pHeader;
+    nsMsgKey recalculatedHighWater = 1;
+    int32_t i = 0;
+    while(i++ < 100 && NS_SUCCEEDED(rv = hdrs->HasMoreElements(&hasMore))
+              && hasMore)
+    {
+      nsCOMPtr<nsISupports> supports;
+      (void) hdrs->GetNext(getter_AddRefs(supports));
+      pHeader = do_QueryInterface(supports);
+      if (pHeader)
+      {
+        nsMsgKey msgKey;
+        pHeader->GetMessageKey(&msgKey);
+        if (msgKey > recalculatedHighWater)
+          recalculatedHighWater = msgKey;
+      }
+    }
+    NS_ASSERTION(m_highWaterMessageKey >= recalculatedHighWater,
+                 "highwater incorrect");
+    m_highWaterMessageKey = recalculatedHighWater;
+  }
+  *result = m_highWaterMessageKey;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetExpiredMark(nsMsgKey expiredKey)
+{
+  m_expiredMark = expiredKey;
+  return SetUint32PropertyWithToken(m_expiredMarkColumnToken, expiredKey);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetExpiredMark(nsMsgKey *result)
+{
+  *result = m_expiredMark;
+  return NS_OK;
+}
+
+// The size of the argument depends on the maximum size of a single message
+NS_IMETHODIMP nsDBFolderInfo::ChangeExpungedBytes(int32_t delta)
+{
+  return SetExpungedBytes(m_expungedBytes + delta);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetMailboxName(const nsAString &newBoxName)
+{
+  return SetPropertyWithToken(m_mailboxNameColumnToken, newBoxName);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetMailboxName(nsAString &boxName)
+{
+  return GetPropertyWithToken(m_mailboxNameColumnToken, boxName);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::ChangeNumUnreadMessages(int32_t delta)
+{
+  m_numUnreadMessages += delta;
+  // m_numUnreadMessages can never be set to negative.
+  if (m_numUnreadMessages < 0)
+  {
+#ifdef DEBUG_bienvenu1
+     NS_ASSERTION(false, "Hardcoded assertion");
+#endif
+      m_numUnreadMessages = 0;
+  }
+  return SetUint32PropertyWithToken(m_numUnreadMessagesColumnToken, m_numUnreadMessages);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::ChangeNumMessages(int32_t delta)
+{
+  m_numMessages += delta;
+  // m_numMessages can never be set to negative.
+  if (m_numMessages < 0)
+  {
+#ifdef DEBUG_bienvenu
+    NS_ASSERTION(false, "num messages can't be < 0");
+#endif
+    m_numMessages = 0;
+  }
+  return SetUint32PropertyWithToken(m_numMessagesColumnToken, m_numMessages);
+}
+
+
+NS_IMETHODIMP nsDBFolderInfo::GetNumUnreadMessages(int32_t *result)
+{
+  *result = m_numUnreadMessages;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetNumUnreadMessages(int32_t numUnreadMessages)
+{
+  m_numUnreadMessages = numUnreadMessages;
+  return SetUint32PropertyWithToken(m_numUnreadMessagesColumnToken, m_numUnreadMessages);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetNumMessages(int32_t *result)
+{
+  *result = m_numMessages;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetNumMessages(int32_t numMessages)
+{
+  m_numMessages = numMessages;
+  return SetUint32PropertyWithToken(m_numMessagesColumnToken, m_numMessages);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetExpungedBytes(int64_t *result)
+{
+  *result = m_expungedBytes;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetExpungedBytes(int64_t expungedBytes)
+{
+  m_expungedBytes = expungedBytes;
+  return SetInt64PropertyWithToken(m_expungedBytesColumnToken, m_expungedBytes);
+}
+
+
+NS_IMETHODIMP nsDBFolderInfo::GetFlags(int32_t *result)
+{
+  *result = m_flags;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetFlags(int32_t flags)
+{
+  nsresult ret = NS_OK;
+
+  if (m_flags != flags)
+  {
+    NS_ASSERTION((m_flags & nsMsgFolderFlags::Inbox) == 0 || (flags & nsMsgFolderFlags::Inbox) != 0, "lost inbox flag");
+    m_flags = flags;
+    ret = SetInt32PropertyWithToken(m_flagsColumnToken, m_flags);
+  }
+  return ret;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::OrFlags(int32_t flags, int32_t *result)
+{
+  m_flags |= flags;
+  *result = m_flags;
+  return SetInt32PropertyWithToken(m_flagsColumnToken, m_flags);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::AndFlags(int32_t flags, int32_t *result)
+{
+  m_flags &= flags;
+  *result = m_flags;
+  return SetInt32PropertyWithToken(m_flagsColumnToken, m_flags);
+}
+
+NS_IMETHODIMP	nsDBFolderInfo::GetImapUidValidity(int32_t *result)
+{
+  *result = m_ImapUidValidity;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetImapUidValidity(int32_t uidValidity)
+{
+  m_ImapUidValidity = uidValidity;
+  return SetUint32PropertyWithToken(m_imapUidValidityColumnToken, m_ImapUidValidity);
+}
+
+bool nsDBFolderInfo::TestFlag(int32_t flags)
+{
+  return (m_flags & flags) != 0;
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetCharacterSet(nsACString &result)
+{
+  if (!m_charSet.IsEmpty())
+    result.Assign(m_charSet);
+  else if (gDefaultCharacterSet)
+    result.Assign(*gDefaultCharacterSet);
+  else
+    result.Truncate();
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetEffectiveCharacterSet(nsACString &result)
+{
+  result.Truncate();
+  if (NS_FAILED(GetCharProperty(kCharacterSetColumnName, result)) ||
+      (result.IsEmpty() && gDefaultCharacterSet))
+    result = *gDefaultCharacterSet;
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetCharacterSet(const nsACString &charSet)
+{
+  m_charSet.Assign(charSet);
+  return SetCharProperty(kCharacterSetColumnName, charSet);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetCharacterSetOverride(bool *characterSetOverride)
+{
+  NS_ENSURE_ARG_POINTER(characterSetOverride);
+  *characterSetOverride = m_charSetOverride;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetCharacterSetOverride(bool characterSetOverride)
+{
+  m_charSetOverride = characterSetOverride;
+  return SetUint32Property(kCharacterSetOverrideColumnName, characterSetOverride);
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetLocale(nsAString &result)
+{
+  GetProperty(kLocaleColumnName, result);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetLocale(const nsAString &locale)
+{
+  return SetProperty(kLocaleColumnName, locale);
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetImapTotalPendingMessages(int32_t *result)
+{
+  NS_ENSURE_ARG_POINTER(result);
+  *result = m_totalPendingMessages;
+  return NS_OK;
+}
+
+void nsDBFolderInfo::ChangeImapTotalPendingMessages(int32_t delta)
+{
+  m_totalPendingMessages+=delta;
+  SetInt32PropertyWithToken(m_totalPendingMessagesColumnToken, m_totalPendingMessages);
+}
+
+NS_IMETHODIMP
+nsDBFolderInfo::GetImapUnreadPendingMessages(int32_t *result)
+{
+  NS_ENSURE_ARG_POINTER(result);
+  *result = m_unreadPendingMessages;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetImapUnreadPendingMessages(int32_t numUnreadPendingMessages)
+{
+  m_unreadPendingMessages = numUnreadPendingMessages;
+  return SetUint32PropertyWithToken(m_unreadPendingMessagesColumnToken, m_unreadPendingMessages);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetImapTotalPendingMessages(int32_t numTotalPendingMessages)
+{
+  m_totalPendingMessages = numTotalPendingMessages;
+  return SetUint32PropertyWithToken(m_totalPendingMessagesColumnToken, m_totalPendingMessages);
+}
+
+void nsDBFolderInfo::ChangeImapUnreadPendingMessages(int32_t delta)
+{
+  m_unreadPendingMessages+=delta;
+  SetInt32PropertyWithToken(m_unreadPendingMessagesColumnToken, m_unreadPendingMessages);
+}
+
+/* attribute nsMsgViewTypeValue viewType; */
+NS_IMETHODIMP nsDBFolderInfo::GetViewType(nsMsgViewTypeValue *aViewType)
+{
+  uint32_t viewTypeValue;
+  nsresult rv = GetUint32Property("viewType", nsMsgViewType::eShowAllThreads, &viewTypeValue);
+  *aViewType = viewTypeValue;
+  return rv;
+}
+NS_IMETHODIMP nsDBFolderInfo::SetViewType(nsMsgViewTypeValue aViewType)
+{
+  return SetUint32Property("viewType", aViewType);
+}
+
+/* attribute nsMsgViewFlagsTypeValue viewFlags; */
+NS_IMETHODIMP nsDBFolderInfo::GetViewFlags(nsMsgViewFlagsTypeValue *aViewFlags)
+{
+  nsMsgViewFlagsTypeValue defaultViewFlags;
+  nsresult rv = m_mdb->GetDefaultViewFlags(&defaultViewFlags);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  uint32_t viewFlagsValue;
+  rv = GetUint32Property("viewFlags", defaultViewFlags, &viewFlagsValue);
+  *aViewFlags = viewFlagsValue;
+  return rv;
+}
+NS_IMETHODIMP nsDBFolderInfo::SetViewFlags(nsMsgViewFlagsTypeValue aViewFlags)
+{
+  return SetUint32Property("viewFlags", aViewFlags);
+}
+
+/* attribute nsMsgViewSortTypeValue sortType; */
+NS_IMETHODIMP nsDBFolderInfo::GetSortType(nsMsgViewSortTypeValue *aSortType)
+{
+  nsMsgViewSortTypeValue defaultSortType;
+  nsresult rv = m_mdb->GetDefaultSortType(&defaultSortType);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  uint32_t sortTypeValue;
+  rv = GetUint32Property("sortType", defaultSortType, &sortTypeValue);
+  *aSortType = sortTypeValue;
+  return rv;
+}
+NS_IMETHODIMP nsDBFolderInfo::SetSortType(nsMsgViewSortTypeValue aSortType)
+{
+  return SetUint32Property("sortType", aSortType);
+}
+
+/* attribute nsMsgViewSortOrderValue sortOrder; */
+NS_IMETHODIMP nsDBFolderInfo::GetSortOrder(nsMsgViewSortOrderValue *aSortOrder)
+{
+  nsMsgViewSortOrderValue defaultSortOrder;
+  nsresult rv = m_mdb->GetDefaultSortOrder(&defaultSortOrder);
+  NS_ENSURE_SUCCESS(rv,rv);
+
+  uint32_t sortOrderValue;
+  rv = GetUint32Property("sortOrder", defaultSortOrder, &sortOrderValue);
+  *aSortOrder = sortOrderValue;
+  return rv;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetSortOrder(nsMsgViewSortOrderValue aSortOrder)
+{
+  return SetUint32Property("sortOrder", aSortOrder);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetKnownArtsSet(const char *newsArtSet)
+{
+  return m_mdb->SetProperty(m_mdbRow, kKnownArtsSetColumnName, newsArtSet);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetKnownArtsSet(char **newsArtSet)
+{
+  return m_mdb->GetProperty(m_mdbRow, kKnownArtsSetColumnName, newsArtSet);
+}
+
+// get arbitrary property, aka row cell value.
+NS_IMETHODIMP nsDBFolderInfo::GetProperty(const char *propertyName, nsAString &resultProperty)
+{
+  return m_mdb->GetPropertyAsNSString(m_mdbRow, propertyName, resultProperty);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetCharProperty(const char *aPropertyName,
+                                              const nsACString &aPropertyValue)
+{
+  return m_mdb->SetProperty(m_mdbRow, aPropertyName,
+                            nsCString(aPropertyValue).get());
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetCharProperty(const char *propertyName,
+                                              nsACString &resultProperty)
+{
+  nsCString result;
+  nsresult rv = m_mdb->GetProperty(m_mdbRow, propertyName, getter_Copies(result));
+  if (NS_SUCCEEDED(rv))
+    resultProperty.Assign(result);
+  return rv;
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetUint32Property(const char *propertyName, uint32_t propertyValue)
+{
+  return m_mdb->SetUint32Property(m_mdbRow, propertyName, propertyValue);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetInt64Property(const char *propertyName, int64_t propertyValue)
+{
+  return m_mdb->SetUint64Property(m_mdbRow, propertyName, (uint64_t) propertyValue);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetProperty(const char *propertyName, const nsAString &propertyStr)
+{
+  return m_mdb->SetPropertyFromNSString(m_mdbRow, propertyName, propertyStr);
+}
+
+nsresult nsDBFolderInfo::SetPropertyWithToken(mdb_token aProperty, const nsAString &propertyStr)
+{
+  return m_mdb->SetNSStringPropertyWithToken(m_mdbRow, aProperty, propertyStr);
+}
+
+nsresult nsDBFolderInfo::SetUint32PropertyWithToken(mdb_token aProperty, uint32_t propertyValue)
+{
+  return m_mdb->UInt32ToRowCellColumn(m_mdbRow, aProperty, propertyValue);
+}
+
+nsresult nsDBFolderInfo::SetInt64PropertyWithToken(mdb_token aProperty, int64_t propertyValue)
+{
+  return m_mdb->UInt64ToRowCellColumn(m_mdbRow, aProperty, (uint64_t) propertyValue);
+}
+
+nsresult nsDBFolderInfo::SetInt32PropertyWithToken(mdb_token aProperty, int32_t propertyValue)
+{
+  nsAutoString propertyStr;
+  propertyStr.AppendInt(propertyValue, 16);
+  return SetPropertyWithToken(aProperty, propertyStr);
+}
+
+nsresult nsDBFolderInfo::GetPropertyWithToken(mdb_token aProperty, nsAString &resultProperty)
+{
+  return m_mdb->RowCellColumnTonsString(m_mdbRow, aProperty, resultProperty);
+}
+
+nsresult nsDBFolderInfo::GetUint32PropertyWithToken(mdb_token aProperty, uint32_t &propertyValue, uint32_t defaultValue)
+{
+  return m_mdb->RowCellColumnToUInt32(m_mdbRow, aProperty, propertyValue, defaultValue);
+}
+
+nsresult nsDBFolderInfo::GetInt32PropertyWithToken(mdb_token aProperty, int32_t &propertyValue, int32_t defaultValue)
+{
+  return m_mdb->RowCellColumnToUInt32(m_mdbRow, aProperty, (uint32_t &) propertyValue, defaultValue);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetUint32Property(const char *propertyName, uint32_t defaultValue, uint32_t *propertyValue)
+{
+  return m_mdb->GetUint32Property(m_mdbRow, propertyName, propertyValue, defaultValue);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetInt64Property(const char *propertyName, int64_t defaultValue, int64_t *propertyValue)
+{
+  return m_mdb->GetUint64Property(m_mdbRow, propertyName, (uint64_t *) &propertyValue, defaultValue);
+}
+
+nsresult nsDBFolderInfo::GetInt64PropertyWithToken(mdb_token aProperty,
+                                                   int64_t &propertyValue,
+                                                   int64_t defaultValue)
+{
+  return m_mdb->RowCellColumnToUInt64(m_mdbRow, aProperty, (uint64_t *) &propertyValue, defaultValue);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetBooleanProperty(const char *propertyName, bool defaultValue, bool *propertyValue)
+{
+  uint32_t defaultUint32Value = (defaultValue) ? 1 : 0;
+  uint32_t returnValue;
+  nsresult rv = m_mdb->GetUint32Property(m_mdbRow, propertyName, &returnValue, defaultUint32Value);
+  *propertyValue = (returnValue != 0);
+  return rv;
+}
+NS_IMETHODIMP	nsDBFolderInfo::SetBooleanProperty(const char *propertyName, bool propertyValue)
+{
+  return m_mdb->SetUint32Property(m_mdbRow, propertyName, propertyValue ? 1 : 0);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::GetFolderName(nsACString &folderName)
+{
+  return GetCharProperty("folderName", folderName);
+}
+
+NS_IMETHODIMP nsDBFolderInfo::SetFolderName(const nsACString &folderName)
+{
+  return SetCharProperty("folderName", folderName);
+}
+
+class nsTransferDBFolderInfo : public nsDBFolderInfo
+{
+public:
+  nsTransferDBFolderInfo();
+  virtual ~nsTransferDBFolderInfo();
+  // parallel arrays of properties and values
+  nsTArray<nsCString> m_properties;
+  nsTArray<nsCString> m_values;
+};
+
+nsTransferDBFolderInfo::nsTransferDBFolderInfo() : nsDBFolderInfo(nullptr)
+{
+}
+
+nsTransferDBFolderInfo::~nsTransferDBFolderInfo()
+{
+}
+
+/* void GetTransferInfo (out nsIDBFolderInfo transferInfo); */
+NS_IMETHODIMP nsDBFolderInfo::GetTransferInfo(nsIDBFolderInfo **transferInfo)
+{
+  NS_ENSURE_ARG_POINTER(transferInfo);
+
+  nsTransferDBFolderInfo *newInfo = new nsTransferDBFolderInfo;
+  *transferInfo = newInfo;
+  NS_ADDREF(newInfo);
+
+  mdb_count numCells;
+  mdbYarn cellYarn;
+  mdb_column cellColumn;
+  char columnName[100];
+  mdbYarn cellName = { columnName, 0, sizeof(columnName), 0, 0, nullptr };
+
+  NS_ASSERTION(m_mdbRow, "null row in getTransferInfo");
+  m_mdbRow->GetCount(m_mdb->GetEnv(), &numCells);
+  // iterate over the cells in the dbfolderinfo remembering attribute names and values.
+  for (mdb_count cellIndex = 0; cellIndex < numCells; cellIndex++)
+  {
+    nsresult err = m_mdbRow->SeekCellYarn(m_mdb->GetEnv(), cellIndex, &cellColumn, nullptr);
+    if (NS_SUCCEEDED(err))
+    {
+      err = m_mdbRow->AliasCellYarn(m_mdb->GetEnv(), cellColumn, &cellYarn);
+      if (NS_SUCCEEDED(err))
+      {
+        m_mdb->GetStore()->TokenToString(m_mdb->GetEnv(), cellColumn, &cellName);
+        newInfo->m_values.AppendElement(Substring((const char *)cellYarn.mYarn_Buf,
+                                        (const char *) cellYarn.mYarn_Buf + cellYarn.mYarn_Fill));
+        newInfo->m_properties.AppendElement(Substring((const char *) cellName.mYarn_Buf,
+                                            (const char *) cellName.mYarn_Buf + cellName.mYarn_Fill));
+      }
+    }
+  }
+
+  return NS_OK;
+}
+
+
+/* void InitFromTransferInfo (in nsIDBFolderInfo transferInfo); */
+NS_IMETHODIMP nsDBFolderInfo::InitFromTransferInfo(nsIDBFolderInfo *aTransferInfo)
+{
+  NS_ENSURE_ARG(aTransferInfo);
+
+  nsTransferDBFolderInfo *transferInfo = static_cast<nsTransferDBFolderInfo *>(aTransferInfo);
+
+  for (uint32_t i = 0; i < transferInfo->m_values.Length(); i++)
+    SetCharProperty(transferInfo->m_properties[i].get(), transferInfo->m_values[i]);
+
+  LoadMemberVariables();
+  return NS_OK;
+}
+

mailnews/db/msgdb/src/nsImapMailDatabase.cpp

@@ -0,0 +1,249 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <sys/stat.h>
+
+#include "msgCore.h"
+#include "nsImapMailDatabase.h"
+#include "nsDBFolderInfo.h"
+
+const char *kPendingHdrsScope = "ns:msg:db:row:scope:pending:all";	// scope for all offine ops table
+const char *kPendingHdrsTableKind = "ns:msg:db:table:kind:pending";
+struct mdbOid gAllPendingHdrsTableOID;
+
+nsImapMailDatabase::nsImapMailDatabase()
+{
+  m_mdbAllPendingHdrsTable = nullptr;
+}
+
+nsImapMailDatabase::~nsImapMailDatabase()
+{
+}
+
+NS_IMETHODIMP	nsImapMailDatabase::GetSummaryValid(bool *aResult)
+{
+  NS_ENSURE_ARG_POINTER(aResult);
+  if (m_dbFolderInfo)
+  {
+    uint32_t version;
+    m_dbFolderInfo->GetVersion(&version);
+    *aResult = (GetCurVersion() == version);
+  }
+  else
+      *aResult = false;
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP	nsImapMailDatabase::SetSummaryValid(bool valid)
+{
+  if (m_dbFolderInfo)
+  {
+    m_dbFolderInfo->SetVersion(valid ? GetCurVersion() : 0);
+    Commit(nsMsgDBCommitType::kLargeCommit);
+  }
+  return NS_OK;
+}
+
+// IMAP does not set local file flags, override does nothing
+void nsImapMailDatabase::UpdateFolderFlag(nsIMsgDBHdr * /* msgHdr */, bool /* bSet */,
+                                          nsMsgMessageFlagType /* flag */, nsIOutputStream ** /* ppFileStream */)
+{
+}
+
+// We override this to avoid our parent class (nsMailDatabase)'s 
+// grabbing of the folder semaphore, and bailing on failure.
+NS_IMETHODIMP nsImapMailDatabase::DeleteMessages(uint32_t aNumKeys, nsMsgKey* nsMsgKeys, nsIDBChangeListener *instigator)
+{
+  return nsMsgDatabase::DeleteMessages(aNumKeys, nsMsgKeys, instigator);
+}
+
+// override so nsMailDatabase methods that deal with m_folderStream are *not* called
+NS_IMETHODIMP nsImapMailDatabase::StartBatch()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::EndBatch()
+{
+  return NS_OK;
+}
+
+nsresult nsImapMailDatabase::AdjustExpungedBytesOnDelete(nsIMsgDBHdr *msgHdr)
+{
+  uint32_t msgFlags;
+  msgHdr->GetFlags(&msgFlags);
+  if (msgFlags & nsMsgMessageFlags::Offline && m_dbFolderInfo)
+  {
+    uint32_t size = 0;
+    (void)msgHdr->GetOfflineMessageSize(&size);
+    return m_dbFolderInfo->ChangeExpungedBytes (size);
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::ForceClosed()
+{
+  m_mdbAllPendingHdrsTable = nullptr;
+  return nsMailDatabase::ForceClosed();
+}
+
+nsresult nsImapMailDatabase::GetAllPendingHdrsTable()
+{
+  nsresult rv = NS_OK;
+  if (!m_mdbAllPendingHdrsTable)
+    rv = GetTableCreateIfMissing(kPendingHdrsScope, kPendingHdrsTableKind, getter_AddRefs(m_mdbAllPendingHdrsTable),
+                                                m_pendingHdrsRowScopeToken, m_pendingHdrsTableKindToken) ;
+  return rv;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::AddNewHdrToDB(nsIMsgDBHdr *newHdr, bool notify)
+{
+  nsresult rv = nsMsgDatabase::AddNewHdrToDB(newHdr, notify);
+  if (NS_SUCCEEDED(rv))
+    rv = UpdatePendingAttributes(newHdr);
+  return rv;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::UpdatePendingAttributes(nsIMsgDBHdr* aNewHdr)
+{
+  nsresult rv = GetAllPendingHdrsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+  mdb_count numPendingHdrs = 0;
+  m_mdbAllPendingHdrsTable->GetCount(GetEnv(), &numPendingHdrs);
+  if (numPendingHdrs > 0)
+  {
+    mdbYarn messageIdYarn;
+    nsCOMPtr <nsIMdbRow> pendingRow;
+    mdbOid  outRowId;
+
+    nsCString messageId;
+    aNewHdr->GetMessageId(getter_Copies(messageId));
+    messageIdYarn.mYarn_Buf = (void*)messageId.get();
+    messageIdYarn.mYarn_Fill = messageId.Length();
+    messageIdYarn.mYarn_Form = 0;
+    messageIdYarn.mYarn_Size = messageIdYarn.mYarn_Fill;
+
+    m_mdbStore->FindRow(GetEnv(), m_pendingHdrsRowScopeToken,
+              m_messageIdColumnToken, &messageIdYarn, &outRowId, getter_AddRefs(pendingRow));
+    if (pendingRow)
+    {
+      mdb_count numCells;
+      mdbYarn cellYarn;
+      mdb_column cellColumn;
+      uint32_t existingFlags;
+
+      pendingRow->GetCount(GetEnv(), &numCells);
+      aNewHdr->GetFlags(&existingFlags);
+      // iterate over the cells in the pending hdr setting properties on the aNewHdr.
+      // we skip cell 0, which is the messageId;
+      nsMsgHdr* msgHdr = static_cast<nsMsgHdr*>(aNewHdr);      // closed system, cast ok
+      nsIMdbRow *row = msgHdr->GetMDBRow();
+      for (mdb_count cellIndex = 1; cellIndex < numCells; cellIndex++)
+      {
+        nsresult err = pendingRow->SeekCellYarn(GetEnv(), cellIndex, &cellColumn, nullptr);
+        if (NS_SUCCEEDED(err))
+        {
+          err = pendingRow->AliasCellYarn(GetEnv(), cellColumn, &cellYarn);
+          if (NS_SUCCEEDED(err))
+          {
+            if (row)
+              row->AddColumn(GetEnv(), cellColumn, &cellYarn);
+          }
+        }
+      }
+      // We might have changed some cached values, so force a refresh.
+      msgHdr->ClearCachedValues();
+      uint32_t resultFlags;
+      msgHdr->OrFlags(existingFlags, &resultFlags);
+      m_mdbAllPendingHdrsTable->CutRow(GetEnv(), pendingRow);
+      pendingRow->CutAllColumns(GetEnv());
+    }
+  }
+  return rv;
+}
+
+nsresult nsImapMailDatabase::GetRowForPendingHdr(nsIMsgDBHdr *pendingHdr,
+                                                 nsIMdbRow **row)
+{
+  nsresult rv = GetAllPendingHdrsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  mdbYarn messageIdYarn;
+  nsCOMPtr<nsIMdbRow> pendingRow;
+  mdbOid  outRowId;
+  nsCString messageId;
+  pendingHdr->GetMessageId(getter_Copies(messageId));
+  messageIdYarn.mYarn_Buf = (void*)messageId.get();
+  messageIdYarn.mYarn_Fill = messageId.Length();
+  messageIdYarn.mYarn_Form = 0;
+  messageIdYarn.mYarn_Size = messageIdYarn.mYarn_Fill;
+
+  rv = m_mdbStore->FindRow(GetEnv(), m_pendingHdrsRowScopeToken,
+            m_messageIdColumnToken, &messageIdYarn, &outRowId, getter_AddRefs(pendingRow));
+
+  if (!pendingRow)
+    rv  = m_mdbStore->NewRow(GetEnv(), m_pendingHdrsRowScopeToken, getter_AddRefs(pendingRow));
+
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (pendingRow)
+  {
+    // now we need to add cells to the row to remember the messageid, property and property value, and flags.
+    // Then, when hdrs are added to the db, we'll check if they have a matching message-id, and if so,
+    // set the property and flags
+    // XXX we already fetched messageId from the pending hdr, could it have changed by the time we get here? 
+    nsCString messageId;
+    pendingHdr->GetMessageId(getter_Copies(messageId));
+    // we're just going to ignore messages without a message-id. They should be rare. If SPAM messages often
+    // didn't have message-id's, they'd be filtered on the server, most likely, and spammers would then
+    // start putting in message-id's.
+    if (!messageId.IsEmpty())
+    {
+       extern const char *kMessageIdColumnName;
+       m_mdbAllPendingHdrsTable->AddRow(GetEnv(), pendingRow);
+       // make sure this is the first cell so that when we ignore the first
+       // cell in nsImapMailDatabase::AddNewHdrToDB, we're ignoring the right one
+      (void) SetProperty(pendingRow, kMessageIdColumnName, messageId.get());
+      pendingRow.forget(row);
+    }
+    else
+      return NS_ERROR_FAILURE;
+  }
+  return rv;
+}
+
+NS_IMETHODIMP nsImapMailDatabase::SetAttributeOnPendingHdr(nsIMsgDBHdr *pendingHdr, const char *property,
+                                  const char *propertyVal)
+{
+  NS_ENSURE_ARG_POINTER(pendingHdr);
+  nsCOMPtr<nsIMdbRow> pendingRow;
+  nsresult rv = GetRowForPendingHdr(pendingHdr, getter_AddRefs(pendingRow));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetProperty(pendingRow, property, propertyVal);
+}
+
+NS_IMETHODIMP
+nsImapMailDatabase::SetUint32AttributeOnPendingHdr(nsIMsgDBHdr *pendingHdr,
+                                                   const char *property,
+                                                   uint32_t propertyVal)
+{
+  NS_ENSURE_ARG_POINTER(pendingHdr);
+  nsCOMPtr<nsIMdbRow> pendingRow;
+  nsresult rv = GetRowForPendingHdr(pendingHdr, getter_AddRefs(pendingRow));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetUint32Property(pendingRow, property, propertyVal);
+}
+
+NS_IMETHODIMP
+nsImapMailDatabase::SetUint64AttributeOnPendingHdr(nsIMsgDBHdr *aPendingHdr,
+                                                   const char *aProperty,
+                                                   uint64_t aPropertyVal)
+{
+  NS_ENSURE_ARG_POINTER(aPendingHdr);
+  nsCOMPtr<nsIMdbRow> pendingRow;
+  nsresult rv = GetRowForPendingHdr(aPendingHdr, getter_AddRefs(pendingRow));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return SetUint64Property(pendingRow, aProperty, aPropertyVal);
+}

mailnews/db/msgdb/src/nsMailDatabase.cpp

@@ -0,0 +1,444 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsMailDatabase.h"
+#include "nsDBFolderInfo.h"
+#include "nsMsgLocalFolderHdrs.h"
+#include "nsNetUtil.h"
+#include "nsISeekableStream.h"
+#include "nsMsgOfflineImapOperation.h"
+#include "nsMsgFolderFlags.h"
+#include "mozilla/Logging.h"
+#include "prprf.h"
+#include "nsMsgUtils.h"
+#include "nsIMsgPluggableStore.h"
+
+extern PRLogModuleInfo *IMAPOffline;
+
+using namespace mozilla;
+
+// scope for all offine ops table
+const char *kOfflineOpsScope = "ns:msg:db:row:scope:ops:all";
+const char *kOfflineOpsTableKind = "ns:msg:db:table:kind:ops";
+struct mdbOid gAllOfflineOpsTableOID;
+
+nsMailDatabase::nsMailDatabase() : m_reparse(false)
+{
+  m_mdbAllOfflineOpsTable = nullptr;
+  m_offlineOpsRowScopeToken = 0;
+  m_offlineOpsTableKindToken = 0;
+}
+
+nsMailDatabase::~nsMailDatabase()
+{
+}
+
+// caller passes in upgrading==true if they want back a db even if the db is out of date.
+// If so, they'll extract out the interesting info from the db, close it, delete it, and
+// then try to open the db again, prior to reparsing.
+nsresult nsMailDatabase::Open(nsMsgDBService* aDBService, nsIFile *aSummaryFile,
+                              bool aCreate, bool aUpgrading)
+{
+#ifdef DEBUG
+  nsString leafName;
+  aSummaryFile->GetLeafName(leafName);
+  if (!StringEndsWith(leafName, NS_LITERAL_STRING(".msf"),
+                     nsCaseInsensitiveStringComparator()))
+    NS_ERROR("non summary file passed into open\n");
+#endif
+  return nsMsgDatabase::Open(aDBService, aSummaryFile, aCreate, aUpgrading);
+}
+
+NS_IMETHODIMP nsMailDatabase::ForceClosed()
+{
+  m_mdbAllOfflineOpsTable = nullptr;
+  return nsMsgDatabase::ForceClosed();
+}
+
+// get this on demand so that only db's that have offline ops will
+// create the table.
+nsresult nsMailDatabase::GetAllOfflineOpsTable()
+{
+  nsresult rv = NS_OK;
+  if (!m_mdbAllOfflineOpsTable)
+    rv = GetTableCreateIfMissing(kOfflineOpsScope, kOfflineOpsTableKind, getter_AddRefs(m_mdbAllOfflineOpsTable), 
+                                                m_offlineOpsRowScopeToken, m_offlineOpsTableKindToken) ;
+  return rv;
+}
+
+NS_IMETHODIMP nsMailDatabase::StartBatch()
+{
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMailDatabase::EndBatch()
+{
+  SetSummaryValid(true);
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMailDatabase::DeleteMessages(uint32_t aNumKeys, nsMsgKey* nsMsgKeys, nsIDBChangeListener *instigator)
+{
+  nsresult rv;
+  if (m_folder)
+  {
+    bool isLocked;
+    m_folder->GetLocked(&isLocked);
+    if (isLocked)
+    {
+      NS_ASSERTION(false, "Some other operation is in progress");
+      return NS_MSG_FOLDER_BUSY;
+    }
+  }
+
+  rv = nsMsgDatabase::DeleteMessages(aNumKeys, nsMsgKeys, instigator);
+  SetSummaryValid(true);
+  return rv;
+}
+
+NS_IMETHODIMP nsMailDatabase::GetSummaryValid(bool *aResult)
+{
+  uint32_t version;
+  m_dbFolderInfo->GetVersion(&version);
+  if (GetCurVersion() != version)
+  {
+    *aResult = false;
+    return NS_OK;
+  }
+  nsCOMPtr<nsIMsgPluggableStore> msgStore;
+  if (!m_folder) {
+    // If the folder is not set, we just return without checking the validity
+    // of the summary file. For now, this is an expected condition when the
+    // message database is being opened from a URL in 
+    // nsMailboxUrl::GetMsgHdrForKey() which calls
+    // nsMsgDBService::OpenMailDBFromFile() without a folder.
+    // Returning an error here would lead to the deletion of the MSF in the
+    // caller nsMsgDatabase::CheckForErrors().
+    *aResult = true;
+    return NS_OK;
+  }
+  nsresult rv = m_folder->GetMsgStore(getter_AddRefs(msgStore));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return msgStore->IsSummaryFileValid(m_folder, this, aResult);
+}
+
+NS_IMETHODIMP nsMailDatabase::SetSummaryValid(bool aValid)
+{
+  nsMsgDatabase::SetSummaryValid(aValid);
+
+  if (!m_folder)
+    return NS_ERROR_NULL_POINTER;
+
+  // If this is a virtual folder, there is no storage.
+  bool flag;
+  m_folder->GetFlag(nsMsgFolderFlags::Virtual, &flag);
+  if (flag)
+    return NS_OK;
+
+  nsCOMPtr<nsIMsgPluggableStore> msgStore;
+  nsresult rv = m_folder->GetMsgStore(getter_AddRefs(msgStore));
+  NS_ENSURE_SUCCESS(rv, rv);
+  return msgStore->SetSummaryFileValid(m_folder, this, aValid);
+}
+
+NS_IMETHODIMP  nsMailDatabase::RemoveOfflineOp(nsIMsgOfflineImapOperation *op)
+{
+  
+  nsresult rv = GetAllOfflineOpsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+  
+  if (!op || !m_mdbAllOfflineOpsTable)
+    return NS_ERROR_NULL_POINTER;
+  nsMsgOfflineImapOperation* offlineOp = static_cast<nsMsgOfflineImapOperation*>(op);  // closed system, so this is ok
+  nsIMdbRow* row = offlineOp->GetMDBRow();
+  rv = m_mdbAllOfflineOpsTable->CutRow(GetEnv(), row);
+  row->CutAllColumns(GetEnv());
+  return rv;
+}
+
+NS_IMETHODIMP nsMailDatabase::GetOfflineOpForKey(nsMsgKey msgKey, bool create, nsIMsgOfflineImapOperation **offlineOp)
+{
+  mdb_bool	hasOid;
+  mdbOid		rowObjectId;
+  nsresult err;
+
+  if (!IMAPOffline)
+    IMAPOffline = PR_NewLogModule("IMAPOFFLINE");
+  nsresult rv = GetAllOfflineOpsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+  
+  if (!offlineOp || !m_mdbAllOfflineOpsTable)
+    return NS_ERROR_NULL_POINTER;
+  
+  *offlineOp = NULL;
+  
+  rowObjectId.mOid_Id = msgKey;
+  rowObjectId.mOid_Scope = m_offlineOpsRowScopeToken;
+  err = m_mdbAllOfflineOpsTable->HasOid(GetEnv(), &rowObjectId, &hasOid);
+  if (NS_SUCCEEDED(err) && m_mdbStore && (hasOid  || create))
+  {
+    nsCOMPtr <nsIMdbRow> offlineOpRow;
+    err = m_mdbStore->GetRow(GetEnv(), &rowObjectId, getter_AddRefs(offlineOpRow));
+    
+    if (create)
+    {
+      if (!offlineOpRow)
+      {
+        err  = m_mdbStore->NewRowWithOid(GetEnv(), &rowObjectId, getter_AddRefs(offlineOpRow));
+        NS_ENSURE_SUCCESS(err, err);
+      }
+      if (offlineOpRow && !hasOid)
+        m_mdbAllOfflineOpsTable->AddRow(GetEnv(), offlineOpRow);
+    }
+    
+    if (NS_SUCCEEDED(err) && offlineOpRow)
+    {
+      *offlineOp = new nsMsgOfflineImapOperation(this, offlineOpRow);
+      if (*offlineOp)
+        (*offlineOp)->SetMessageKey(msgKey);
+      NS_IF_ADDREF(*offlineOp);
+    }
+    if (!hasOid && m_dbFolderInfo)
+    {
+      // set initial value for flags so we don't lose them.
+      nsCOMPtr <nsIMsgDBHdr> msgHdr;
+      GetMsgHdrForKey(msgKey, getter_AddRefs(msgHdr));
+      if (msgHdr)
+      {
+        uint32_t flags;
+        msgHdr->GetFlags(&flags);
+        (*offlineOp)->SetNewFlags(flags);
+      }
+      int32_t newFlags;
+      m_dbFolderInfo->OrFlags(nsMsgFolderFlags::OfflineEvents, &newFlags);
+    }
+  }
+  
+  return err;
+}
+
+NS_IMETHODIMP nsMailDatabase::EnumerateOfflineOps(nsISimpleEnumerator **enumerator)
+{
+  NS_ASSERTION(false, "not impl yet");
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+
+NS_IMETHODIMP nsMailDatabase::ListAllOfflineOpIds(nsTArray<nsMsgKey> *offlineOpIds)
+{
+  NS_ENSURE_ARG(offlineOpIds);
+  nsresult rv = GetAllOfflineOpsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsIMdbTableRowCursor *rowCursor;
+  if (!IMAPOffline)
+    IMAPOffline = PR_NewLogModule("IMAPOFFLINE");
+
+  if (m_mdbAllOfflineOpsTable)
+  {
+    nsresult err = m_mdbAllOfflineOpsTable->GetTableRowCursor(GetEnv(), -1, &rowCursor);
+    while (NS_SUCCEEDED(err) && rowCursor)
+    {
+      mdbOid outOid;
+      mdb_pos	outPos;
+      
+      err = rowCursor->NextRowOid(GetEnv(), &outOid, &outPos);
+      // is this right? Mork is returning a 0 id, but that should valid.
+      if (outPos < 0 || outOid.mOid_Id == (mdb_id) -1)
+        break;
+      if (NS_SUCCEEDED(err))
+      {
+        offlineOpIds->AppendElement(outOid.mOid_Id);
+        if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+        {
+          nsCOMPtr <nsIMsgOfflineImapOperation> offlineOp;
+          GetOfflineOpForKey(outOid.mOid_Id, false, getter_AddRefs(offlineOp));
+          if (offlineOp)
+          {
+            nsMsgOfflineImapOperation *logOp = static_cast<nsMsgOfflineImapOperation *>(static_cast<nsIMsgOfflineImapOperation *>(offlineOp.get()));
+            if (logOp)
+              logOp->Log(IMAPOffline);
+
+          }
+        }
+      }
+    }
+    // TODO: would it cause a problem to replace this with "rv = err;" ?
+    rv = (NS_SUCCEEDED(err)) ? NS_OK : NS_ERROR_FAILURE;
+    rowCursor->Release();
+  }
+
+  offlineOpIds->Sort();
+  return rv;
+}
+
+NS_IMETHODIMP nsMailDatabase::ListAllOfflineDeletes(nsTArray<nsMsgKey> *offlineDeletes)
+{
+  NS_ENSURE_ARG_POINTER(offlineDeletes);
+
+  nsresult rv = GetAllOfflineOpsTable();
+  NS_ENSURE_SUCCESS(rv, rv);
+  nsIMdbTableRowCursor *rowCursor;
+  if (m_mdbAllOfflineOpsTable)
+  {
+    nsresult err = m_mdbAllOfflineOpsTable->GetTableRowCursor(GetEnv(), -1, &rowCursor);
+    while (NS_SUCCEEDED(err) && rowCursor)
+    {
+      mdbOid outOid;
+      mdb_pos outPos;
+      nsIMdbRow* offlineOpRow;
+
+      err = rowCursor->NextRow(GetEnv(), &offlineOpRow, &outPos);
+      // is this right? Mork is returning a 0 id, but that should valid.
+      if (outPos < 0 || offlineOpRow == nullptr)
+        break;
+      if (NS_SUCCEEDED(err))
+      {
+        offlineOpRow->GetOid(GetEnv(), &outOid);
+        nsIMsgOfflineImapOperation *offlineOp = new nsMsgOfflineImapOperation(this, offlineOpRow);
+        if (offlineOp)
+        {
+          NS_ADDREF(offlineOp);
+          imapMessageFlagsType newFlags;
+          nsOfflineImapOperationType opType;
+          
+          offlineOp->GetOperation(&opType);
+          offlineOp->GetNewFlags(&newFlags);
+          if (opType & nsIMsgOfflineImapOperation::kMsgMoved || 
+            ((opType & nsIMsgOfflineImapOperation::kFlagsChanged) 
+            && (newFlags & nsIMsgOfflineImapOperation::kMsgMarkedDeleted)))
+            offlineDeletes->AppendElement(outOid.mOid_Id);
+          NS_RELEASE(offlineOp);
+        }
+        offlineOpRow->Release();
+      }
+    }
+    // TODO: would it cause a problem to replace this with "rv = err;" ?
+    rv = (NS_SUCCEEDED(err)) ? NS_OK : NS_ERROR_FAILURE;
+    rowCursor->Release();
+  }
+  return rv;
+}
+
+// This is used to remember that the db is out of sync with the mail folder
+// and needs to be regenerated.
+void nsMailDatabase::SetReparse(bool reparse)
+{
+  m_reparse = reparse;
+}
+
+class nsMsgOfflineOpEnumerator : public nsISimpleEnumerator {
+public:
+  NS_DECL_ISUPPORTS
+
+  // nsISimpleEnumerator methods:
+  NS_DECL_NSISIMPLEENUMERATOR
+
+  nsMsgOfflineOpEnumerator(nsMailDatabase* db);
+
+protected:
+  virtual ~nsMsgOfflineOpEnumerator();
+  nsresult					GetRowCursor();
+  nsresult					PrefetchNext();
+  nsMailDatabase*              mDB;
+  nsIMdbTableRowCursor*       mRowCursor;
+  nsCOMPtr <nsIMsgOfflineImapOperation> mResultOp;
+  bool              mDone;
+  bool						mNextPrefetched;
+};
+
+nsMsgOfflineOpEnumerator::nsMsgOfflineOpEnumerator(nsMailDatabase* db)
+    : mDB(db), mRowCursor(nullptr), mDone(false)
+{
+  NS_ADDREF(mDB);
+  mNextPrefetched = false;
+}
+
+nsMsgOfflineOpEnumerator::~nsMsgOfflineOpEnumerator()
+{
+  NS_IF_RELEASE(mRowCursor);
+  NS_RELEASE(mDB);
+}
+
+NS_IMPL_ISUPPORTS(nsMsgOfflineOpEnumerator, nsISimpleEnumerator)
+
+nsresult nsMsgOfflineOpEnumerator::GetRowCursor()
+{
+  nsresult rv = NS_OK;
+  mDone = false;
+
+  if (!mDB || !mDB->m_mdbAllOfflineOpsTable)
+    return NS_ERROR_NULL_POINTER;
+
+  rv = mDB->m_mdbAllOfflineOpsTable->GetTableRowCursor(mDB->GetEnv(), -1, &mRowCursor);
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgOfflineOpEnumerator::GetNext(nsISupports **aItem)
+{
+  NS_ENSURE_ARG_POINTER(aItem);
+
+  nsresult rv = NS_OK;
+  if (!mNextPrefetched)
+    rv = PrefetchNext();
+  if (NS_SUCCEEDED(rv))
+  {
+    if (mResultOp) 
+    {
+      *aItem = mResultOp;
+      NS_ADDREF(*aItem);
+      mNextPrefetched = false;
+    }
+  }
+  return rv;
+}
+
+nsresult nsMsgOfflineOpEnumerator::PrefetchNext()
+{
+  nsresult rv = NS_OK;
+  nsIMdbRow* offlineOpRow;
+  mdb_pos rowPos;
+
+  if (!mRowCursor)
+  {
+    rv = GetRowCursor();
+    if (NS_FAILED(rv))
+      return rv;
+  }
+
+  rv = mRowCursor->NextRow(mDB->GetEnv(), &offlineOpRow, &rowPos);
+  if (!offlineOpRow) 
+  {
+    mDone = true;
+    return NS_ERROR_FAILURE;
+  }
+  if (NS_FAILED(rv)) 
+  {
+    mDone = true;
+    return rv;
+  }
+
+  nsIMsgOfflineImapOperation *op = new nsMsgOfflineImapOperation(mDB, offlineOpRow);
+  mResultOp = op;
+  if (!op)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  if (mResultOp) 
+  {
+    mNextPrefetched = true;
+    return NS_OK;
+  }
+  return NS_ERROR_FAILURE;
+}
+
+NS_IMETHODIMP nsMsgOfflineOpEnumerator::HasMoreElements(bool *aResult)
+{
+  NS_ENSURE_ARG_POINTER(aResult);
+
+  if (!mNextPrefetched)
+    PrefetchNext();
+  *aResult = !mDone;
+  return NS_OK;
+}

mailnews/db/msgdb/src/nsMsgOfflineImapOperation.cpp

@@ -0,0 +1,378 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsMsgOfflineImapOperation.h"
+#include "nsMsgUtils.h"
+#include "mozilla/Logging.h"
+
+using namespace mozilla;
+
+PRLogModuleInfo *IMAPOffline;
+
+/* Implementation file */
+NS_IMPL_ISUPPORTS(nsMsgOfflineImapOperation, nsIMsgOfflineImapOperation)
+
+// property names for offine imap operation fields.
+#define PROP_OPERATION "op"
+#define PROP_OPERATION_FLAGS "opFlags"
+#define PROP_NEW_FLAGS "newFlags"
+#define PROP_MESSAGE_KEY "msgKey"
+#define PROP_SRC_MESSAGE_KEY "srcMsgKey"
+#define PROP_SRC_FOLDER_URI "srcFolderURI"
+#define PROP_MOVE_DEST_FOLDER_URI "moveDest"
+#define PROP_NUM_COPY_DESTS "numCopyDests"
+#define PROP_COPY_DESTS "copyDests" // how to delimit these? Or should we do the "dest1","dest2" etc trick? But then we'd need to shuffle
+                                    // them around since we delete off the front first.
+#define PROP_KEYWORD_ADD "addedKeywords"
+#define PROP_KEYWORD_REMOVE "removedKeywords"
+#define PROP_MSG_SIZE "msgSize"
+#define PROP_PLAYINGBACK "inPlayback"
+
+nsMsgOfflineImapOperation::nsMsgOfflineImapOperation(nsMsgDatabase *db, nsIMdbRow *row)
+{
+  NS_ASSERTION(db, "can't have null db");
+  NS_ASSERTION(row, "can't have null row");
+  m_operation = 0;
+  m_operationFlags = 0;
+  m_messageKey = nsMsgKey_None;
+  m_sourceMessageKey = nsMsgKey_None;
+  m_mdb = db;
+  NS_ADDREF(m_mdb);
+  m_mdbRow = row;
+  m_newFlags = 0;
+  m_mdb->GetUint32Property(m_mdbRow, PROP_OPERATION, (uint32_t *) &m_operation, 0);
+  m_mdb->GetUint32Property(m_mdbRow, PROP_MESSAGE_KEY, &m_messageKey, 0);
+  m_mdb->GetUint32Property(m_mdbRow, PROP_OPERATION_FLAGS, &m_operationFlags, 0);
+  m_mdb->GetUint32Property(m_mdbRow, PROP_NEW_FLAGS, (uint32_t *) &m_newFlags, 0);
+}
+
+nsMsgOfflineImapOperation::~nsMsgOfflineImapOperation()
+{
+  // clear the row first, in case we're holding the last reference
+  // to the db.
+  m_mdbRow = nullptr;
+  NS_IF_RELEASE(m_mdb);
+}
+
+/* attribute nsOfflineImapOperationType operation; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetOperation(nsOfflineImapOperationType *aOperation)
+{
+  NS_ENSURE_ARG(aOperation);
+  *aOperation = m_operation;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetOperation(nsOfflineImapOperationType aOperation)
+{
+  if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x setOperation was %x add %x", m_messageKey, m_operation, aOperation));
+
+  m_operation |= aOperation;
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_OPERATION, m_operation);
+}
+
+/* void clearOperation (in nsOfflineImapOperationType operation); */
+NS_IMETHODIMP nsMsgOfflineImapOperation::ClearOperation(nsOfflineImapOperationType aOperation)
+{
+  if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x clearOperation was %x clear %x", m_messageKey, m_operation, aOperation));
+  m_operation &= ~aOperation;
+  switch (aOperation)
+  {
+  case kMsgMoved:
+  case kAppendTemplate:
+  case kAppendDraft:
+    m_moveDestination.Truncate();
+    break;
+  case kMsgCopy:
+    m_copyDestinations.RemoveElementAt(0);
+    break;
+  }
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_OPERATION, m_operation);
+}
+
+/* attribute nsMsgKey messageKey; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetMessageKey(nsMsgKey *aMessageKey)
+{
+  NS_ENSURE_ARG(aMessageKey);
+  *aMessageKey = m_messageKey;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetMessageKey(nsMsgKey aMessageKey)
+{
+  m_messageKey = aMessageKey;
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_MESSAGE_KEY, m_messageKey);
+}
+
+/* attribute nsMsgKey srcMessageKey; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetSrcMessageKey(nsMsgKey *aMessageKey)
+{
+  NS_ENSURE_ARG(aMessageKey);
+  return m_mdb->GetUint32Property(m_mdbRow, PROP_SRC_MESSAGE_KEY, aMessageKey, nsMsgKey_None);
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetSrcMessageKey(nsMsgKey aMessageKey)
+{
+  m_messageKey = aMessageKey;
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_SRC_MESSAGE_KEY, m_messageKey);
+}
+
+/* attribute imapMessageFlagsType flagOperation; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetFlagOperation(imapMessageFlagsType *aFlagOperation)
+{
+  NS_ENSURE_ARG(aFlagOperation);
+  *aFlagOperation = m_operationFlags;
+  return NS_OK;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetFlagOperation(imapMessageFlagsType aFlagOperation)
+{
+  if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x setFlagOperation was %x add %x", m_messageKey, m_operationFlags, aFlagOperation));
+  SetOperation(kFlagsChanged);
+  nsresult rv = SetNewFlags(aFlagOperation);
+  NS_ENSURE_SUCCESS(rv, rv);
+  m_operationFlags |= aFlagOperation;
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_OPERATION_FLAGS, m_operationFlags);
+}
+
+/* attribute imapMessageFlagsType flagOperation; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetNewFlags(imapMessageFlagsType *aNewFlags)
+{
+  NS_ENSURE_ARG(aNewFlags);
+  uint32_t flags;
+  nsresult rv = m_mdb->GetUint32Property(m_mdbRow, PROP_NEW_FLAGS, &flags, 0);
+  *aNewFlags = m_newFlags = (imapMessageFlagsType) flags;
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetNewFlags(imapMessageFlagsType aNewFlags)
+{
+  if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info) && m_newFlags != aNewFlags)
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x SetNewFlags was %x to %x", m_messageKey, m_newFlags, aNewFlags));
+  m_newFlags = aNewFlags;
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_NEW_FLAGS, m_newFlags);
+}
+
+
+/* attribute string destinationFolderURI; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetDestinationFolderURI(char * *aDestinationFolderURI)
+{
+  NS_ENSURE_ARG(aDestinationFolderURI);
+  (void) m_mdb->GetProperty(m_mdbRow, PROP_MOVE_DEST_FOLDER_URI, getter_Copies(m_moveDestination));
+  *aDestinationFolderURI = ToNewCString(m_moveDestination);
+  return (*aDestinationFolderURI) ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetDestinationFolderURI(const char * aDestinationFolderURI)
+{
+  if (MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x SetDestinationFolderURI to %s", m_messageKey, aDestinationFolderURI));
+  m_moveDestination = aDestinationFolderURI ? aDestinationFolderURI : 0;
+  return m_mdb->SetProperty(m_mdbRow, PROP_MOVE_DEST_FOLDER_URI, aDestinationFolderURI);
+}
+
+/* attribute string sourceFolderURI; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetSourceFolderURI(char * *aSourceFolderURI)
+{
+  NS_ENSURE_ARG(aSourceFolderURI);
+  nsresult rv = m_mdb->GetProperty(m_mdbRow, PROP_SRC_FOLDER_URI, getter_Copies(m_sourceFolder));
+  *aSourceFolderURI = ToNewCString(m_sourceFolder);
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetSourceFolderURI(const char * aSourceFolderURI)
+{
+  m_sourceFolder = aSourceFolderURI ? aSourceFolderURI : 0;
+  SetOperation(kMoveResult);
+
+  return m_mdb->SetProperty(m_mdbRow, PROP_SRC_FOLDER_URI, aSourceFolderURI);
+}
+
+/* attribute string keyword; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetKeywordsToAdd(char * *aKeywords)
+{
+  NS_ENSURE_ARG(aKeywords);
+  nsresult rv = m_mdb->GetProperty(m_mdbRow, PROP_KEYWORD_ADD, getter_Copies(m_keywordsToAdd));
+  *aKeywords = ToNewCString(m_keywordsToAdd);
+  return rv;
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::AddKeywordToAdd(const char * aKeyword)
+{
+  SetOperation(kAddKeywords);
+  return AddKeyword(aKeyword, m_keywordsToAdd, PROP_KEYWORD_ADD, m_keywordsToRemove, PROP_KEYWORD_REMOVE);
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetKeywordsToRemove(char * *aKeywords)
+{
+  NS_ENSURE_ARG(aKeywords);
+  nsresult rv = m_mdb->GetProperty(m_mdbRow, PROP_KEYWORD_REMOVE, getter_Copies(m_keywordsToRemove));
+  *aKeywords = ToNewCString(m_keywordsToRemove);
+  return rv;
+}
+
+nsresult nsMsgOfflineImapOperation::AddKeyword(const char *aKeyword, nsCString &addList, const char *addProp,
+                                               nsCString &removeList, const char *removeProp)
+{
+  int32_t startOffset, keywordLength;
+  if (!MsgFindKeyword(nsDependentCString(aKeyword), addList, &startOffset, &keywordLength))
+  {
+    if (!addList.IsEmpty())
+      addList.Append(' ');
+    addList.Append(aKeyword);
+  }
+  // if the keyword we're removing was in the list of keywords to add,
+  // cut it from that list.
+  if (MsgFindKeyword(nsDependentCString(aKeyword), removeList, &startOffset, &keywordLength))
+  {
+    removeList.Cut(startOffset, keywordLength);
+    m_mdb->SetProperty(m_mdbRow, removeProp, removeList.get());
+  }
+  return m_mdb->SetProperty(m_mdbRow, addProp, addList.get());
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::AddKeywordToRemove(const char * aKeyword)
+{
+  SetOperation(kRemoveKeywords);
+  return AddKeyword(aKeyword, m_keywordsToRemove, PROP_KEYWORD_REMOVE, m_keywordsToAdd, PROP_KEYWORD_ADD);
+}
+
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::AddMessageCopyOperation(const char *destinationBox)
+{
+  SetOperation(kMsgCopy);
+  nsAutoCString newDest(destinationBox);
+  nsresult rv = GetCopiesFromDB();
+  NS_ENSURE_SUCCESS(rv, rv);
+  m_copyDestinations.AppendElement(newDest);
+  return SetCopiesToDB();
+}
+
+// we write out the folders as one string, separated by 0x1.
+#define FOLDER_SEP_CHAR '\001'
+
+nsresult nsMsgOfflineImapOperation::GetCopiesFromDB()
+{
+  nsCString copyDests;
+  m_copyDestinations.Clear();
+  nsresult rv = m_mdb->GetProperty(m_mdbRow, PROP_COPY_DESTS, getter_Copies(copyDests));
+  // use 0x1 as the delimiter between folder names since it's not a legal character
+  if (NS_SUCCEEDED(rv) && !copyDests.IsEmpty())
+  {
+    int32_t curCopyDestStart = 0;
+    int32_t nextCopyDestPos = 0;
+
+    while (nextCopyDestPos != -1)
+    {
+      nsCString curDest;
+      nextCopyDestPos = copyDests.FindChar(FOLDER_SEP_CHAR, curCopyDestStart);
+      if (nextCopyDestPos > 0)
+        curDest = Substring(copyDests, curCopyDestStart, nextCopyDestPos - curCopyDestStart);
+      else
+        curDest = Substring(copyDests, curCopyDestStart, copyDests.Length() - curCopyDestStart);
+      curCopyDestStart = nextCopyDestPos + 1;
+      m_copyDestinations.AppendElement(curDest);
+    }
+  }
+  return rv;
+}
+
+nsresult nsMsgOfflineImapOperation::SetCopiesToDB()
+{
+  nsAutoCString copyDests;
+
+  // use 0x1 as the delimiter between folders
+  for (uint32_t i = 0; i < m_copyDestinations.Length(); i++)
+  {
+    if (i > 0)
+      copyDests.Append(FOLDER_SEP_CHAR);
+    copyDests.Append(m_copyDestinations.ElementAt(i));
+  }
+  return m_mdb->SetProperty(m_mdbRow, PROP_COPY_DESTS, copyDests.get());
+}
+
+/* attribute long numberOfCopies; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetNumberOfCopies(int32_t *aNumberOfCopies)
+{
+  NS_ENSURE_ARG(aNumberOfCopies);
+  nsresult rv = GetCopiesFromDB();
+  NS_ENSURE_SUCCESS(rv, rv);
+  *aNumberOfCopies = m_copyDestinations.Length();
+  return NS_OK;
+}
+
+/* string getCopyDestination (in long copyIndex); */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetCopyDestination(int32_t copyIndex, char **retval)
+{
+  NS_ENSURE_ARG(retval);
+  nsresult rv = GetCopiesFromDB();
+  NS_ENSURE_SUCCESS(rv, rv);
+  if (copyIndex >= (int32_t)m_copyDestinations.Length())
+    return NS_ERROR_ILLEGAL_VALUE;
+  *retval = ToNewCString(m_copyDestinations.ElementAt(copyIndex));
+  return (*retval) ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
+}
+
+/* attribute unsigned log msgSize; */
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetMsgSize(uint32_t *aMsgSize)
+{
+  NS_ENSURE_ARG(aMsgSize);
+  return m_mdb->GetUint32Property(m_mdbRow, PROP_MSG_SIZE, aMsgSize, 0);
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetMsgSize(uint32_t aMsgSize)
+{
+  return m_mdb->SetUint32Property(m_mdbRow, PROP_MSG_SIZE, aMsgSize);
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::SetPlayingBack(bool aPlayingBack)
+{
+  return m_mdb->SetBooleanProperty(m_mdbRow, PROP_PLAYINGBACK, aPlayingBack);
+}
+
+NS_IMETHODIMP nsMsgOfflineImapOperation::GetPlayingBack(bool *aPlayingBack)
+{
+  NS_ENSURE_ARG(aPlayingBack);
+  return m_mdb->GetBooleanProperty(m_mdbRow, PROP_PLAYINGBACK, aPlayingBack);
+}
+
+
+void nsMsgOfflineImapOperation::Log(PRLogModuleInfo *logFile)
+{
+  if (!IMAPOffline)
+    IMAPOffline = PR_NewLogModule("IMAPOFFLINE");
+  if (!MOZ_LOG_TEST(IMAPOffline, LogLevel::Info))
+    return;
+  //  const long kMoveResult              = 0x8;
+  //  const long kAppendDraft           = 0x10;
+  //  const long kAddedHeader           = 0x20;
+  //  const long kDeletedMsg              = 0x40;
+  //  const long kMsgMarkedDeleted = 0x80;
+  //  const long kAppendTemplate      = 0x100;
+  //  const long kDeleteAllMsgs          = 0x200;
+  if (m_operation & nsIMsgOfflineImapOperation::kFlagsChanged)
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x changeFlag:%x", m_messageKey, m_newFlags));
+  if (m_operation & nsIMsgOfflineImapOperation::kMsgMoved)
+  {
+    nsCString moveDestFolder;
+    GetDestinationFolderURI(getter_Copies(moveDestFolder));
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x moveTo:%s", m_messageKey, moveDestFolder.get()));
+  }
+  if (m_operation & nsIMsgOfflineImapOperation::kMsgCopy)
+  {
+    nsCString copyDests;
+    m_mdb->GetProperty(m_mdbRow, PROP_COPY_DESTS, getter_Copies(copyDests));
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x moveTo:%s", m_messageKey, copyDests.get()));
+  }
+  if (m_operation & nsIMsgOfflineImapOperation::kAppendDraft)
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x append draft", m_messageKey));
+  if (m_operation & nsIMsgOfflineImapOperation::kAddKeywords)
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x add keyword:%s", m_messageKey, m_keywordsToAdd.get()));
+  if (m_operation & nsIMsgOfflineImapOperation::kRemoveKeywords)
+    MOZ_LOG(IMAPOffline, LogLevel::Info, ("msg id %x remove keyword:%s", m_messageKey, m_keywordsToRemove.get()));
+}

mailnews/db/msgdb/src/nsMsgOfflineImapOperation.h

@@ -0,0 +1,55 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef _nsMsgOfflineImapOperation_H_
+
+#include "nsIMsgOfflineImapOperation.h"
+#include "mdb.h"
+#include "nsMsgDatabase.h"
+#include "prlog.h"
+
+class nsMsgOfflineImapOperation : public nsIMsgOfflineImapOperation
+{
+public:
+  /** Instance Methods **/
+  nsMsgOfflineImapOperation(nsMsgDatabase *db, nsIMdbRow *row);
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIMSGOFFLINEIMAPOPERATION
+
+
+  nsIMdbRow   *GetMDBRow() {return m_mdbRow;}
+  nsresult    GetCopiesFromDB();
+  nsresult    SetCopiesToDB();
+  void        Log(PRLogModuleInfo *logFile);
+protected:
+  virtual ~nsMsgOfflineImapOperation();
+  nsresult AddKeyword(const char *aKeyword, nsCString &addList, const char *addProp,
+                      nsCString &removeList, const char *removeProp);
+
+  nsOfflineImapOperationType m_operation;
+  nsMsgKey          m_messageKey;
+  nsMsgKey          m_sourceMessageKey;
+  uint32_t          m_operationFlags; // what to do on sync
+  imapMessageFlagsType m_newFlags; // used for kFlagsChanged
+
+  // these are URI's, and are escaped. Thus, we can use a delimter like ' '
+  // because the real spaces should be escaped.
+  nsCString  m_sourceFolder;
+  nsCString  m_moveDestination;
+  nsTArray<nsCString>  m_copyDestinations;
+
+  nsCString      m_keywordsToAdd;
+  nsCString      m_keywordsToRemove;
+
+  // nsMsgOfflineImapOperation will have to know what db and row they belong to, since they are really
+  // just a wrapper around the offline operation row in the mdb.
+  // though I hope not.
+  nsMsgDatabase    *m_mdb;
+  nsCOMPtr <nsIMdbRow> m_mdbRow;
+};
+
+
+
+#endif /* _nsMsgOfflineImapOperation_H_ */
+

mailnews/db/msgdb/src/nsNewsDatabase.cpp

@@ -0,0 +1,360 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "msgCore.h"
+#include "nsIMsgDBView.h"
+#include "nsIMsgThread.h"
+#include "nsNewsDatabase.h"
+#include "nsMsgKeySet.h"
+#include "nsCOMPtr.h"
+#include "prlog.h"
+
+#if defined(DEBUG_sspitzer_) || defined(DEBUG_seth_)
+#define DEBUG_NEWS_DATABASE 1
+#endif
+
+nsNewsDatabase::nsNewsDatabase()
+{
+  m_readSet = nullptr;
+}
+
+nsNewsDatabase::~nsNewsDatabase()
+{
+}
+
+NS_IMPL_ADDREF_INHERITED(nsNewsDatabase, nsMsgDatabase)
+NS_IMPL_RELEASE_INHERITED(nsNewsDatabase, nsMsgDatabase)
+
+NS_IMETHODIMP nsNewsDatabase::QueryInterface(REFNSIID aIID, void** aInstancePtr)
+{
+  if (!aInstancePtr) return NS_ERROR_NULL_POINTER;
+  *aInstancePtr = nullptr;
+
+  if (aIID.Equals(NS_GET_IID(nsINewsDatabase)))
+  {
+    *aInstancePtr = static_cast<nsINewsDatabase *>(this);
+  }
+
+  if(*aInstancePtr)
+  {
+    AddRef();
+    return NS_OK;
+  }
+
+  return nsMsgDatabase::QueryInterface(aIID, aInstancePtr);
+}
+
+nsresult nsNewsDatabase::Close(bool forceCommit)
+{
+  return nsMsgDatabase::Close(forceCommit);
+}
+
+nsresult nsNewsDatabase::ForceClosed()
+{
+  return nsMsgDatabase::ForceClosed();
+}
+
+nsresult nsNewsDatabase::Commit(nsMsgDBCommit commitType)
+{
+  if (m_dbFolderInfo && m_readSet)
+  {
+    // let's write out our idea of the read set so we can compare it with that of
+    // the .rc file next time we start up.
+    nsCString readSet;
+    m_readSet->Output(getter_Copies(readSet));
+    m_dbFolderInfo->SetCharProperty("readSet", readSet);
+  }
+  return nsMsgDatabase::Commit(commitType);
+}
+
+
+uint32_t nsNewsDatabase::GetCurVersion()
+{
+  return kMsgDBVersion;
+}
+
+NS_IMETHODIMP nsNewsDatabase::IsRead(nsMsgKey key, bool *pRead)
+{
+  NS_ASSERTION(pRead, "null out param in IsRead");
+  if (!pRead) return NS_ERROR_NULL_POINTER;
+
+  if (!m_readSet) return NS_ERROR_FAILURE;
+
+  *pRead = m_readSet->IsMember(key);
+  return NS_OK;
+}
+
+nsresult nsNewsDatabase::IsHeaderRead(nsIMsgDBHdr *msgHdr, bool *pRead)
+{
+    nsresult rv;
+    nsMsgKey messageKey;
+
+    if (!msgHdr || !pRead) return NS_ERROR_NULL_POINTER;
+
+    rv = msgHdr->GetMessageKey(&messageKey);
+    if (NS_FAILED(rv)) return rv;
+
+    rv = IsRead(messageKey,pRead);
+    return rv;
+}
+
+// return highest article number we've seen.
+NS_IMETHODIMP nsNewsDatabase::GetHighWaterArticleNum(nsMsgKey *key)
+{
+  NS_ASSERTION(m_dbFolderInfo, "null db folder info");
+  if (!m_dbFolderInfo)
+    return NS_ERROR_FAILURE;
+  return m_dbFolderInfo->GetHighWater(key);
+}
+
+// return the key of the first article number we know about.
+// Since the iterator iterates in id order, we can just grab the
+// messagekey of the first header it returns.
+// ### dmb
+// This will not deal with the situation where we get holes in
+// the headers we know about. Need to figure out how and when
+// to solve that. This could happen if a transfer is interrupted.
+// Do we need to keep track of known arts permanently?
+NS_IMETHODIMP nsNewsDatabase::GetLowWaterArticleNum(nsMsgKey *key)
+{
+  nsresult  rv;
+  nsMsgHdr  *pHeader;
+
+  nsCOMPtr<nsISimpleEnumerator> hdrs;
+  rv = EnumerateMessages(getter_AddRefs(hdrs));
+  if (NS_FAILED(rv))
+    return rv;
+
+  rv = hdrs->GetNext((nsISupports**)&pHeader);
+  NS_ASSERTION(NS_SUCCEEDED(rv), "nsMsgDBEnumerator broken");
+  if (NS_FAILED(rv))
+    return rv;
+
+  return pHeader->GetMessageKey(key);
+}
+
+nsresult  nsNewsDatabase::ExpireUpTo(nsMsgKey expireKey)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+nsresult		nsNewsDatabase::ExpireRange(nsMsgKey startRange, nsMsgKey endRange)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+
+NS_IMETHODIMP nsNewsDatabase::GetReadSet(nsMsgKeySet **pSet)
+{
+    if (!pSet) return NS_ERROR_NULL_POINTER;
+    *pSet = m_readSet;
+    return NS_OK;
+}
+
+NS_IMETHODIMP nsNewsDatabase::SetReadSet(nsMsgKeySet *pSet)
+{
+  m_readSet = pSet;
+
+  if (m_readSet)
+  {
+    // compare this read set with the one in the db folder info.
+    // If not equivalent, sync with this one.
+    nsCString dbReadSet;
+    if (m_dbFolderInfo)
+      m_dbFolderInfo->GetCharProperty("readSet", dbReadSet);
+    nsCString newsrcReadSet;
+    m_readSet->Output(getter_Copies(newsrcReadSet));
+    if (!dbReadSet.Equals(newsrcReadSet))
+      SyncWithReadSet();
+  }
+  return NS_OK;
+}
+
+
+bool nsNewsDatabase::SetHdrReadFlag(nsIMsgDBHdr *msgHdr, bool bRead)
+{
+    nsresult rv;
+    bool isRead;
+    rv = IsHeaderRead(msgHdr, &isRead);
+
+    if (isRead == bRead)
+    {
+      // give the base class a chance to update m_flags.
+      nsMsgDatabase::SetHdrReadFlag(msgHdr, bRead);
+      return false;
+    }
+    else {
+      nsMsgKey messageKey;
+
+      // give the base class a chance to update m_flags.
+      nsMsgDatabase::SetHdrReadFlag(msgHdr, bRead);
+      rv = msgHdr->GetMessageKey(&messageKey);
+      if (NS_FAILED(rv)) return false;
+
+      NS_ASSERTION(m_readSet, "m_readSet is null");
+      if (!m_readSet) return false;
+
+      if (!bRead) {
+#ifdef DEBUG_NEWS_DATABASE
+        printf("remove %d from the set\n",messageKey);
+#endif
+
+        m_readSet->Remove(messageKey);
+
+        rv = NotifyReadChanged(nullptr);
+        if (NS_FAILED(rv)) return false;
+      }
+      else {
+#ifdef DEBUG_NEWS_DATABASE
+        printf("add %d to the set\n",messageKey);
+#endif
+
+        if (m_readSet->Add(messageKey) < 0) return false;
+
+        rv = NotifyReadChanged(nullptr);
+        if (NS_FAILED(rv)) return false;
+      }
+    }
+    return true;
+}
+
+NS_IMETHODIMP nsNewsDatabase::MarkAllRead(uint32_t *aNumMarked,
+                                          nsMsgKey **aThoseMarked)
+{
+  nsMsgKey lowWater = nsMsgKey_None, highWater;
+  nsCString knownArts;
+  if (m_dbFolderInfo)
+  {
+    m_dbFolderInfo->GetKnownArtsSet(getter_Copies(knownArts));
+    nsMsgKeySet *knownKeys = nsMsgKeySet::Create(knownArts.get());
+    if (knownKeys)
+      lowWater = knownKeys->GetFirstMember();
+
+    delete knownKeys;
+  }
+  if (lowWater == nsMsgKey_None)
+    GetLowWaterArticleNum(&lowWater);
+  GetHighWaterArticleNum(&highWater);
+  if (lowWater > 2)
+    m_readSet->AddRange(1, lowWater - 1);
+  nsresult err = nsMsgDatabase::MarkAllRead(aNumMarked, aThoseMarked);
+  if (NS_SUCCEEDED(err) && 1 <= highWater)
+    m_readSet->AddRange(1, highWater); // mark everything read in newsrc.
+
+  return err;
+}
+
+nsresult nsNewsDatabase::SyncWithReadSet()
+{
+
+  // The code below attempts to update the underlying nsMsgDatabase's idea
+  // of read/unread flags to match the read set in the .newsrc file. It should
+  // only be called when they don't match, e.g., we crashed after committing the
+  // db but before writing out the .newsrc
+  nsCOMPtr <nsISimpleEnumerator> hdrs;
+  nsresult rv = EnumerateMessages(getter_AddRefs(hdrs));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  bool hasMore = false, readInNewsrc, isReadInDB, changed = false;
+  int32_t numMessages = 0, numUnreadMessages = 0;
+  nsMsgKey messageKey;
+  nsCOMPtr <nsIMsgThread> threadHdr;
+
+  // Scan all messages in DB
+  while (NS_SUCCEEDED(rv = hdrs->HasMoreElements(&hasMore)) && hasMore)
+  {
+      nsCOMPtr<nsISupports> supports;
+      rv = hdrs->GetNext(getter_AddRefs(supports));
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      nsCOMPtr <nsIMsgDBHdr> header = do_QueryInterface(supports, &rv);
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      rv = nsMsgDatabase::IsHeaderRead(header, &isReadInDB);
+      NS_ENSURE_SUCCESS(rv, rv);
+
+      header->GetMessageKey(&messageKey);
+      IsRead(messageKey,&readInNewsrc);
+
+      numMessages++;
+      if (!readInNewsrc)
+        numUnreadMessages++;
+
+      // If DB and readSet disagree on Read/Unread, fix DB
+      if (readInNewsrc!=isReadInDB)
+      {
+        MarkHdrRead(header, readInNewsrc, nullptr);
+        changed = true;
+      }
+  }
+
+  // Update FolderInfo Counters
+  int32_t oldMessages, oldUnreadMessages;
+  rv = m_dbFolderInfo->GetNumMessages(&oldMessages);
+  if (NS_SUCCEEDED(rv) && oldMessages!=numMessages)
+  {
+      changed = true;
+      m_dbFolderInfo->ChangeNumMessages(numMessages-oldMessages);
+  }
+  rv = m_dbFolderInfo->GetNumUnreadMessages(&oldUnreadMessages);
+  if (NS_SUCCEEDED(rv) && oldUnreadMessages!=numUnreadMessages)
+  {
+      changed = true;
+      m_dbFolderInfo->ChangeNumUnreadMessages(numUnreadMessages-oldUnreadMessages);
+  }
+
+  if (changed)
+      Commit(nsMsgDBCommitType::kLargeCommit);
+
+  return rv;
+}
+
+nsresult nsNewsDatabase::AdjustExpungedBytesOnDelete(nsIMsgDBHdr *msgHdr)
+{
+  uint32_t msgFlags;
+  msgHdr->GetFlags(&msgFlags);
+  if (msgFlags & nsMsgMessageFlags::Offline && m_dbFolderInfo)
+  {
+    uint32_t size = 0;
+    (void)msgHdr->GetOfflineMessageSize(&size);
+    return m_dbFolderInfo->ChangeExpungedBytes (size);
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNewsDatabase::GetDefaultViewFlags(nsMsgViewFlagsTypeValue *aDefaultViewFlags)
+{
+  NS_ENSURE_ARG_POINTER(aDefaultViewFlags);
+  GetIntPref("mailnews.default_news_view_flags", aDefaultViewFlags);
+  if (*aDefaultViewFlags < nsMsgViewFlagsType::kNone ||
+      *aDefaultViewFlags > (nsMsgViewFlagsType::kThreadedDisplay |
+                            nsMsgViewFlagsType::kShowIgnored |
+                            nsMsgViewFlagsType::kUnreadOnly |
+                            nsMsgViewFlagsType::kExpandAll |
+                            nsMsgViewFlagsType::kGroupBySort))
+    *aDefaultViewFlags = nsMsgViewFlagsType::kThreadedDisplay;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNewsDatabase::GetDefaultSortType(nsMsgViewSortTypeValue *aDefaultSortType)
+{
+  NS_ENSURE_ARG_POINTER(aDefaultSortType);
+  GetIntPref("mailnews.default_news_sort_type", aDefaultSortType);
+  if (*aDefaultSortType < nsMsgViewSortType::byDate ||
+      *aDefaultSortType > nsMsgViewSortType::byAccount)
+    *aDefaultSortType = nsMsgViewSortType::byThread;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsNewsDatabase::GetDefaultSortOrder(nsMsgViewSortOrderValue *aDefaultSortOrder)
+{
+  NS_ENSURE_ARG_POINTER(aDefaultSortOrder);
+  GetIntPref("mailnews.default_news_sort_order", aDefaultSortOrder);
+  if (*aDefaultSortOrder != nsMsgViewSortOrder::descending)
+    *aDefaultSortOrder = nsMsgViewSortOrder::ascending;
+  return NS_OK;
+}