Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

mailnews/db/gloda/modules/collection.js

@@ -0,0 +1,772 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaCollection', 'GlodaCollectionManager'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.collection");
+
+/**
+ * @namespace Central registry and logic for all collections.
+ *
+ * The collection manager is a singleton that has the following tasks:
+ * - Let views of objects (nouns) know when their objects have changed.  For
+ *   example, an attribute has changed due to user action.
+ * - Let views of objects based on queries know when new objects match their
+ *   query, or when their existing objects no longer match due to changes.
+ * - Caching/object-identity maintenance.  It is ideal if we only ever have
+ *   one instance of an object at a time.  (More specifically, only one instance
+ *   per database row 'id'.)  The collection mechanism lets us find existing
+ *   instances to this end.  Caching can be directly integrated by being treated
+ *   as a special collection.
+ */
+var GlodaCollectionManager = {
+  _collectionsByNoun: {},
+  _cachesByNoun: {},
+
+  /**
+   * Registers the existence of a collection with the collection manager.  This
+   *  is done using a weak reference so that the collection can go away if it
+   *  wants to.
+   */
+  registerCollection: function gloda_colm_registerCollection(aCollection) {
+    let collections;
+    let nounID = aCollection.query._nounDef.id;
+    if (!(nounID in this._collectionsByNoun))
+      collections = this._collectionsByNoun[nounID] = [];
+    else {
+      // purge dead weak references while we're at it
+      collections = this._collectionsByNoun[nounID].filter(function (aRef) {
+        return aRef.get(); });
+      this._collectionsByNoun[nounID] = collections;
+    }
+    collections.push(Cu.getWeakReference(aCollection));
+  },
+
+  getCollectionsForNounID: function gloda_colm_getCollectionsForNounID(aNounID){
+    if (!(aNounID in this._collectionsByNoun))
+      return [];
+
+    // generator would be nice, but I suspect get() is too expensive to use
+    //  twice (guard/predicate and value)
+    let weakCollections = this._collectionsByNoun[aNounID];
+    let collections = [];
+    for (let iColl = 0; iColl < weakCollections.length; iColl++) {
+      let collection = weakCollections[iColl].get();
+      if (collection)
+        collections.push(collection);
+    }
+    return collections;
+  },
+
+  defineCache: function gloda_colm_defineCache(aNounDef, aCacheSize) {
+    this._cachesByNoun[aNounDef.id] = new GlodaLRUCacheCollection(aNounDef,
+                                                                   aCacheSize);
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it was't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOne: function gloda_colm_cacheLookupOne(aNounID, aID, aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aID in cache._idMap) {
+        let item = cache._idMap[aID];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aID in collection._idMap) {
+        let item = collection._idMap[aID];
+        if (cache)
+          cache.add([item]);
+        return item;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Lookup multiple nouns by ID from the cache/existing collections.
+   *
+   * @param aNounID The kind of noun identified by its ID.
+   * @param aIDMap A dictionary/map whose keys must be gloda noun ids for the
+   *     given noun type and whose values are ignored.
+   * @param aTargetMap An object to hold the noun id's (key) and noun instances
+   *     (value) for the noun instances that were found available in memory
+   *     because they were cached or in existing query collections.
+   * @param [aDoCache=true] Should we add any items to the cache that we found
+   *     in collections that were in memory but not in the cache?  You would
+   *     likely want to pass false if you are only updating in-memory
+   *     representations rather than performing a new query.
+   *
+   * @return [The number that were found, the number that were not found,
+   *          a dictionary whose keys are the ids of noun instances that
+   *          were not found.]
+   */
+  cacheLookupMany: function gloda_colm_cacheLookupMany(aNounID, aIDMap,
+      aTargetMap, aDoCache) {
+    let foundCount = 0, notFoundCount = 0, notFound = {};
+
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      for (let key in aIDMap) {
+        let cacheValue = cache._idMap[key];
+        if (cacheValue === undefined) {
+          notFoundCount++;
+          notFound[key] = null;
+        }
+        else {
+          foundCount++;
+          aTargetMap[key] = cacheValue;
+          cache.hit(cacheValue);
+        }
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let key in notFound) {
+        let collValue = collection._idMap[key];
+        if (collValue !== undefined) {
+          aTargetMap[key] = collValue;
+          delete notFound[key];
+          foundCount++;
+          notFoundCount--;
+          if (cache)
+            cache.add([collValue]);
+        }
+      }
+    }
+
+    return [foundCount, notFoundCount, notFound];
+  },
+
+  /**
+   * Friendlier version of |cacheLookupMany|; takes a list of ids and returns
+   *  an object whose keys and values are the gloda id's and instances of the
+   *  instances that were found.  We don't tell you who we didn't find.  The
+   *  assumption is this is being used for in-memory updates where we only need
+   *  to tweak what is in memory.
+   */
+  cacheLookupManyList: function gloda_colm_cacheLookupManyList(aNounID, aIds) {
+    let checkMap = {}, targetMap = {};
+    for (let id of aIds) {
+      checkMap[id] = null;
+    }
+    // do not promote found items into the cache
+    this.cacheLookupMany(aNounID, checkMap, targetMap, false);
+    return targetMap;
+  },
+
+  /**
+   * Attempt to locate an instance of the object of the given noun type with the
+   *  given id.  Counts as a cache hit if found.  (And if it was't in a cache,
+   *  but rather a collection, it is added to the cache.)
+   */
+  cacheLookupOneByUniqueValue:
+      function gloda_colm_cacheLookupOneByUniqueValue(aNounID, aUniqueValue,
+                                                      aDoCache) {
+    let cache = this._cachesByNoun[aNounID];
+
+    if (cache) {
+      if (aUniqueValue in cache._uniqueValueMap) {
+        let item = cache._uniqueValueMap[aUniqueValue];
+        return cache.hit(item);
+      }
+    }
+
+    if (aDoCache === false)
+      cache = null;
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      if (aUniqueValue in collection._uniqueValueMap) {
+        let item = collection._uniqueValueMap[aUniqueValue];
+        if (cache)
+          cache.add([item]);
+        return item;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Checks whether the provided item with the given id is actually a duplicate
+   *  of an instance that already exists in the cache/a collection.  If it is,
+   *  the pre-existing instance is returned and counts as a cache hit.  If it
+   *  is not, the passed-in instance is added to the cache and returned.
+   */
+  cacheLoadUnifyOne: function gloda_colm_cacheLoadUnifyOne(aItem) {
+    let items = [aItem];
+    this.cacheLoadUnify(aItem.NOUN_ID, items);
+    return items[0];
+  },
+
+  /**
+   * Given a list of items, check if any of them already have duplicate,
+   *  canonical, instances in the cache or collections.  Items with pre-existing
+   *  instances are replaced by those instances in the provided list, and each
+   *  counts as a cache hit.  Items without pre-existing instances are added
+   *  to the cache and left intact.
+   */
+  cacheLoadUnify: function gloda_colm_cacheLoadUnify(aNounID, aItems,
+      aCacheIfMissing) {
+    let cache = this._cachesByNoun[aNounID];
+    if (aCacheIfMissing === undefined)
+      aCacheIfMissing = true;
+
+    // track the items we haven't yet found in a cache/collection (value) and
+    //  their index in aItems (key).  We're somewhat abusing the dictionary
+    //  metaphor with the intent of storing tuples here.  We also do it because
+    //  it allows random-access deletion theoretically without cost.  (Since
+    //  we delete during iteration, that may be wrong, but it sounds like the
+    //  semantics still work?)
+    let unresolvedIndexToItem = {};
+    let numUnresolved = 0;
+
+    if (cache) {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        let item = aItems[iItem];
+
+        if (item.id in cache._idMap) {
+          let realItem = cache._idMap[item.id];
+          // update the caller's array with the reference to the 'real' item
+          aItems[iItem] = realItem;
+          cache.hit(realItem);
+        }
+        else {
+          unresolvedIndexToItem[iItem] = item;
+          numUnresolved++;
+        }
+      }
+
+      // we're done if everyone was a hit.
+      if (numUnresolved == 0)
+        return;
+    }
+    else {
+      for (let iItem = 0; iItem < aItems.length; iItem++) {
+        unresolvedIndexToItem[iItem] = aItems[iItem];
+      }
+      numUnresolved = aItems.length;
+    }
+
+    let needToCache = [];
+    // next, let's fall back to our collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      for (let [iItem, item] in Iterator(unresolvedIndexToItem)) {
+        if (item.id in collection._idMap) {
+          let realItem = collection._idMap[item.id];
+          // update the caller's array to now have the 'real' object
+          aItems[iItem] = realItem;
+          // flag that we need to cache this guy (we use an inclusive cache)
+          needToCache.push(realItem);
+          // we no longer need to resolve this item...
+          delete unresolvedIndexToItem[iItem];
+          // stop checking collections if we got everybody
+          if (--numUnresolved == 0)
+            break;
+        }
+      }
+    }
+
+    // anything left in unresolvedIndexToItem should be added to the cache
+    //  unless !aCacheIfMissing.  plus, we already have 'needToCache'
+    if (cache && aCacheIfMissing) {
+      cache.add(needToCache.concat(Object.keys(unresolvedIndexToItem).
+                                   map(key => unresolvedIndexToItem[key])));
+    }
+
+    return aItems;
+  },
+
+  cacheCommitDirty: function glod_colm_cacheCommitDirty() {
+    for (let id in this._cachesByNoun) {
+      let cache = this._cachesByNoun[id];
+      cache.commitDirty();
+    }
+  },
+
+  /**
+   * Notifies the collection manager that an item has been loaded and should
+   *  be cached, assuming caching is active.
+   */
+  itemLoaded: function gloda_colm_itemsLoaded(aItem) {
+    let cache = this._cachesByNoun[aItem.NOUN_ID];
+    if (cache) {
+      cache.add([aItem]);
+    }
+  },
+
+  /**
+   * Notifies the collection manager that multiple items has been loaded and
+   *  should be cached, assuming caching is active.
+   */
+  itemsLoaded: function gloda_colm_itemsLoaded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+  },
+
+  /**
+   * This should be called when items are added to the global database.  This
+   *  should generally mean during indexing by indexers or an attribute
+   *  provider.
+   * We walk all existing collections for the given noun type and add the items
+   *  to the collection if the item meets the query that defines the collection.
+   */
+  itemsAdded: function gloda_colm_itemsAdded(aNounID, aItems) {
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      cache.add(aItems);
+    }
+
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let addItems = aItems.filter(item => collection.query.test(item));
+      if (addItems.length)
+        collection._onItemsAdded(addItems);
+    }
+  },
+  /**
+   * This should be called when items in the global database are modified.  For
+   *  example, as a result of indexing.  This should generally only be called
+   *  by indexers or by attribute providers.
+   * We walk all existing collections for the given noun type.  For items
+   *  currently included in each collection but should no longer be (per the
+   *  collection's defining query) we generate onItemsRemoved events.  For items
+   *  not currently included in the collection but should now be, we generate
+   *  onItemsAdded events.  For items included that still match the query, we
+   *  generate onItemsModified events.
+   */
+  itemsModified: function gloda_colm_itemsModified(aNounID, aItems) {
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let added = [], modified = [], removed = [];
+      for (let item of aItems) {
+        if (item.id in collection._idMap) {
+          // currently in... but should it still be there?
+          if (collection.query.test(item))
+            modified.push(item); // yes, keep it
+          // oy, so null queries really don't want any notifications, and they
+          //  sorta fit into our existing model, except for the removal bit.
+          //  so we need a specialized check for them, and we're using the
+          //  frozen attribute to this end.
+          else if (!collection.query.frozen)
+            removed.push(item); // no, bin it
+        }
+        else if (collection.query.test(item)) // not in, should it be?
+          added.push(item); // yep, add it
+      }
+      if (added.length)
+        collection._onItemsAdded(added);
+      if (modified.length)
+        collection._onItemsModified(modified);
+      if (removed.length)
+        collection._onItemsRemoved(removed);
+    }
+  },
+  /**
+   * This should be called when items in the global database are permanently-ish
+   *  deleted.  (This is distinct from concepts like message deletion which may
+   *  involved trash folders or other modified forms of existence.  Deleted
+   *  means the data is gone and if it were to come back, it would come back
+   *  via an itemsAdded event.)
+   * We walk all existing collections for the given noun type.  For items
+   *  currently in the collection, we generate onItemsRemoved events.
+   * 
+   * @param aItemIds A list of item ids that are being deleted.
+   */
+  itemsDeleted: function gloda_colm_itemsDeleted(aNounID, aItemIds) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let itemId of aItemIds) {
+        if (itemId in cache._idMap)
+          cache.deleted(cache._idMap[itemId]);
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = aItemIds.filter(itemId => itemId in collection._idMap).
+        map(itemId => collection._idMap[itemId]);
+      if (removeItems.length)
+        collection._onItemsRemoved(removeItems);
+    }
+  },
+  /**
+   * Like |itemsDeleted| but for the case where the deletion is based on an
+   *  attribute that SQLite can more efficiently check than we can and where the
+   *  cost of scanning the in-memory items is presumably much cheaper than
+   *  trying to figure out what actually got deleted.
+   * 
+   * Since we are doing an in-memory walk, this is obviously O(n) where n is the
+   *  number of noun instances of a given type in-memory.  We are assuming this
+   *  is a reasonable number of things and that this type of deletion call is
+   *  not going to happen all that frequently.  If these assumptions are wrong,
+   *  callers are advised to re-think the whole situation.
+   * 
+   * @param aNounID Type of noun we are talking about here.
+   * @param aFilter A filter function that returns true when the item should be
+   *     thought of as deleted, or false if the item is still good.  Screw this
+   *     up and you will get some seriously wacky bugs, yo.
+   */
+  itemsDeletedByAttribute: function gloda_colm_itemsDeletedByAttribute(
+      aNounID, aFilter) {
+    // cache
+    let cache = this._cachesByNoun[aNounID];
+    if (cache) {
+      for (let id in cache._idMap) {
+        let item = cache._idMap[id];
+        if (aFilter(item))
+          cache.deleted(item);
+      }
+    }
+
+    // collections
+    for (let collection of this.getCollectionsForNounID(aNounID)) {
+      let removeItems = collection.items.filter(aFilter);
+      if (removeItems.length)
+        collection._onItemsRemoved(removeItems);
+    }
+  },
+};
+
+/**
+ * @class A current view of the set of first-class nouns meeting a given query.
+ *  Assuming a listener is present, events are
+ *  generated when new objects meet the query, existing objects no longer meet
+ *  the query, or existing objects have experienced a change in attributes that
+ *  does not affect their ability to be present (but the listener may care about
+ *  because it is exposing those attributes).
+ * @constructor
+ */
+function GlodaCollection(aNounDef, aItems, aQuery, aListener,
+      aMasterCollection) {
+  // if aNounDef is null, we are just being invoked for subclassing
+  if (aNounDef === undefined)
+    return;
+
+  this._nounDef = aNounDef;
+  // should we also maintain a unique value mapping...
+  if (this._nounDef.usesUniqueValue)
+    this._uniqueValueMap = {};
+
+  this.pendingItems = [];
+  this._pendingIdMap = {};
+  this.items = [];
+  this._idMap = {};
+
+  // force the listener to null for our call to _onItemsAdded; no events for
+  //  the initial load-out.
+  this._listener = null;
+  if (aItems && aItems.length)
+    this._onItemsAdded(aItems);
+
+  this.query = aQuery || null;
+  if (this.query) {
+    this.query.collection = this;
+    if (this.query.options.stashColumns)
+      this.stashedColumns = {};
+  }
+  this._listener = aListener || null;
+
+  this.deferredCount = 0;
+  this.resolvedCount = 0;
+
+  if (aMasterCollection) {
+    this.masterCollection = aMasterCollection.masterCollection;
+  }
+  else {
+    this.masterCollection = this;
+    /** a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of these sub-dictionaries maps the IDs of desired noun instances to
+     * the actual instance, or null if it has not yet been loaded.
+     */
+    this.referencesByNounID = {};
+    /**
+     * a dictionary of dictionaries. at the top level, the keys are noun IDs.
+     * each of the sub-dictionaries maps the IDs of the _recognized parent
+     * noun_ to the list of children, or null if the list has not yet been
+     * populated.
+     *
+     * So if we have a noun definition A with ID 1 who is the recognized parent
+     *  noun of noun definition B with ID 2, AND we have an instance A(1) with
+     *  two children B(10), B(11), then an example might be: {2: {1: [10, 11]}}.
+     */
+    this.inverseReferencesByNounID = {};
+    this.subCollections = {};
+  }
+}
+
+GlodaCollection.prototype = {
+  get listener() { return this._listener; },
+  set listener(aListener) { this._listener = aListener; },
+
+  /**
+   * If this collection still has a query associated with it, drop the query
+   *  and replace it with an 'explicit query'.  This means that the Collection
+   *  Manager will not attempt to match new items indexed to the system against
+   *  our query criteria.
+   * Once you call this method, your collection's listener will no longer
+   *  receive onItemsAdded notifications that are not the result of your
+   *  initial database query.  It will, however, receive onItemsModified
+   *  notifications if items in the collection are re-indexed.
+   */
+  becomeExplicit: function gloda_coll_becomeExplicit() {
+    if (!(this.query instanceof this._nounDef.explicitQueryClass)) {
+      this.query = new this._nounDef.explicitQueryClass(this);
+    }
+  },
+
+  /**
+   * Clear the contents of this collection.  This only makes sense for explicit
+   *  collections or wildcard collections.  (Actual query-based collections
+   *  should represent the state of the query, so unless we're going to delete
+   *  all the items, clearing the collection would violate that constraint.)
+   */
+  clear: function gloda_coll_clear() {
+    this._idMap = {};
+    if (this._uniqueValueMap)
+      this._uniqueValueMap = {};
+    this.items = [];
+  },
+
+  _onItemsAdded: function gloda_coll_onItemsAdded(aItems) {
+    this.items.push.apply(this.items, aItems);
+    if (this._uniqueValueMap) {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+        this._uniqueValueMap[item.uniqueValue] = item;
+      }
+    }
+    else {
+      for (let item of this.items) {
+        this._idMap[item.id] = item;
+      }
+    }
+    if (this._listener) {
+      try {
+        this._listener.onItemsAdded(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsAdded: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  _onItemsModified: function gloda_coll_onItemsModified(aItems) {
+    if (this._listener) {
+      try {
+        this._listener.onItemsModified(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsModified: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  /**
+   * Given a list of items that definitely no longer belong in this collection,
+   *  remove them from the collection and notify the listener.  The 'tricky'
+   *  part is that we need to remove the deleted items from our list of items.
+   */
+  _onItemsRemoved: function gloda_coll_onItemsRemoved(aItems) {
+    // we want to avoid the O(n^2) deletion performance case, and deletion
+    //  should be rare enough that the extra cost of building the deletion map
+    //  should never be a real problem.
+    let deleteMap = {};
+    // build the delete map while also nuking from our id map/unique value map
+    for (let item of aItems) {
+      deleteMap[item.id] = true;
+      delete this._idMap[item.id];
+      if (this._uniqueValueMap)
+        delete this._uniqueValueMap[item.uniqueValue];
+    }
+    let items = this.items;
+    // in-place filter.  probably needless optimization.
+    let iWrite=0;
+    for (let iRead = 0; iRead < items.length; iRead++) {
+      let item = items[iRead];
+      if (!(item.id in deleteMap))
+        items[iWrite++] = item;
+    }
+    items.splice(iWrite);
+
+    if (this._listener) {
+      try {
+        this._listener.onItemsRemoved(aItems, this);
+      }
+      catch (ex) {
+        LOG.error("caught exception from listener in onItemsRemoved: " +
+            ex.fileName + ":" + ex.lineNumber + ": " + ex);
+      }
+    }
+  },
+
+  _onQueryCompleted: function gloda_coll_onQueryCompleted() {
+    this.query.completed = true;
+    if (this._listener && this._listener.onQueryCompleted)
+      this._listener.onQueryCompleted(this);
+  }
+};
+
+/**
+ * Create an LRU cache collection for the given noun with the given size.
+ * @constructor
+ */
+function GlodaLRUCacheCollection(aNounDef, aCacheSize) {
+  GlodaCollection.call(this, aNounDef, null, null, null);
+
+  this._head = null; // aka oldest!
+  this._tail = null; // aka newest!
+  this._size = 0;
+  // let's keep things sane, and simplify our logic a little...
+  if (aCacheSize < 32)
+    aCacheSize = 32;
+  this._maxCacheSize = aCacheSize;
+}
+/**
+ * @class A LRU-discard cache.  We use a doubly linked-list for the eviction
+ *  tracking.  Since we require that there is at most one LRU-discard cache per
+ *  noun class, we simplify our lives by adding our own attributes to the
+ *  cached objects.
+ * @augments GlodaCollection
+ */
+GlodaLRUCacheCollection.prototype = new GlodaCollection;
+GlodaLRUCacheCollection.prototype.add = function cache_add(aItems) {
+  for (let item of aItems) {
+    if (item.id in this._idMap) {
+      // DEBUGME so, we're dealing with this, but it shouldn't happen.  need
+      //  trace-debuggage.
+      continue;
+    }
+    this._idMap[item.id] = item;
+    if (this._uniqueValueMap)
+      this._uniqueValueMap[item.uniqueValue] = item;
+
+    item._lruPrev = this._tail;
+    // we do have to make sure that we will set _head the first time we insert
+    //  something
+    if (this._tail !== null)
+      this._tail._lruNext = item;
+    else
+      this._head = item;
+    item._lruNext = null;
+    this._tail = item;
+
+    this._size++;
+  }
+
+  while (this._size > this._maxCacheSize) {
+    let item = this._head;
+
+    // we never have to deal with the possibility of needing to make _head/_tail
+    //  null.
+    this._head = item._lruNext;
+    this._head._lruPrev = null;
+    // (because we are nice, we will delete the properties...)
+    delete item._lruNext;
+    delete item._lruPrev;
+
+    // nuke from our id map
+    delete this._idMap[item.id];
+    if (this._uniqueValueMap)
+      delete this._uniqueValueMap[item.uniqueValue];
+
+    // flush dirty items to disk (they may not have this attribute, in which
+    //  case, this returns false, which is fine.)
+    if (item.dirty) {
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+
+    this._size--;
+  }
+};
+
+GlodaLRUCacheCollection.prototype.hit = function cache_hit(aItem) {
+  // don't do anything in the 0 or 1 items case, or if we're already
+  //  the last item
+  if ((this._head === this._tail) || (this._tail === aItem))
+    return aItem;
+
+  // - unlink the item
+  if (aItem._lruPrev !== null)
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  else
+    this._head = aItem._lruNext;
+  // (_lruNext cannot be null)
+  aItem._lruNext._lruPrev = aItem._lruPrev;
+  // - link it in to the end
+  this._tail._lruNext = aItem;
+  aItem._lruPrev = this._tail;
+  aItem._lruNext = null;
+  // update tail tracking
+  this._tail = aItem;
+
+  return aItem;
+};
+
+GlodaLRUCacheCollection.prototype.deleted = function cache_deleted(aItem) {
+  // unlink the item
+  if (aItem._lruPrev !== null)
+    aItem._lruPrev._lruNext = aItem._lruNext;
+  else
+    this._head = aItem._lruNext;
+  if (aItem._lruNext !== null)
+    aItem._lruNext._lruPrev = aItem._lruPrev;
+  else
+    this._tail = aItem._lruPrev;
+
+  // (because we are nice, we will delete the properties...)
+  delete aItem._lruNext;
+  delete aItem._lruPrev;
+
+  // nuke from our id map
+  delete this._idMap[aItem.id];
+  if (this._uniqueValueMap)
+    delete this._uniqueValueMap[aItem.uniqueValue];
+
+  this._size--;
+};
+
+/**
+ * If any of the cached items are dirty, commit them, and make them no longer
+ *  dirty.
+ */
+GlodaLRUCacheCollection.prototype.commitDirty = function cache_commitDirty() {
+  // we can only do this if there is an update method available...
+  if (!this._nounDef.objUpdate)
+    return;
+
+  for (let iItem in this._idMap) {
+    let item = this._idMap[iItem];
+    if (item.dirty) {
+      LOG.debug("flushing dirty: " + item);
+      this._nounDef.objUpdate.call(this._nounDef.datastore, item);
+      delete item.dirty;
+    }
+  }
+};

mailnews/db/gloda/modules/connotent.js

@@ -0,0 +1,273 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaContent', 'whittlerRegistry',
+                          'mimeMsgToContentAndMeta', 'mimeMsgToContentSnippetAndMeta'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.connotent");
+
+
+
+/**
+ * Given a MimeMsg and the corresponding folder, return the GlodaContent object.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @return an array containing the GlodaContent instance, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentAndMeta(aMimeMsg, folder) {
+  let content = new GlodaContent();
+  let meta = {subject: aMimeMsg.get("subject")};
+  let bodyLines = aMimeMsg.coerceBodyToPlaintext(folder).split(/\r?\n/);
+
+  for (let whittler of whittlerRegistry.getWhittlers())
+    whittler.contentWhittle(meta, bodyLines, content);
+
+  return [content, meta];
+}
+
+
+/**
+ * Given a MimeMsg, return the whittled content string, suitable for summarizing
+ * a message.
+ *
+ * @param aMimeMsg: the MimeMessage instance
+ * @param folder: the nsIMsgDBFolder
+ * @param length: optional number of characters to trim the whittled content.
+ * If the actual length of the message is greater than |length|, then the return
+ * value is the first (length-1) characters with an ellipsis appended.
+ * @return an array containing the text of the snippet, and the meta dictionary
+ * that the Gloda content providers may have filled with useful data.
+ */
+
+function mimeMsgToContentSnippetAndMeta(aMimeMsg, folder, length) {
+  let [content, meta] = mimeMsgToContentAndMeta(aMimeMsg, folder);
+
+  let text = content.getContentSnippet(length + 1);
+  if (length && text.length > length)
+    text = text.substring(0, length-1) + "\u2026"; // ellipsis
+
+  return [text, meta];
+}
+
+
+/**
+ * A registry of gloda providers that have contentWhittle() functions.
+ * used by mimeMsgToContentSnippet, but populated by the Gloda object as it's
+ * processing providers.
+ */
+function WhittlerRegistry() {
+  this._whittlers = [];
+}
+
+WhittlerRegistry.prototype = {
+  /**
+   * Add a provider as a content whittler.
+   */
+  registerWhittler: function whittler_registry_registerWhittler(provider) {
+    this._whittlers.push(provider);
+  },
+  /**
+   * get the list of content whittlers, sorted from the most specific to
+   * the most generic
+   */
+  getWhittlers: function whittler_registry_getWhittlers() {
+    // Use the concat() trick to avoid mutating the internal object and
+    // leaking an internal representation.
+    return this._whittlers.concat().reverse();
+  }
+}
+
+this.whittlerRegistry = new WhittlerRegistry();
+
+function GlodaContent() {
+  this._contentPriority = null;
+  this._producing = false;
+  this._hunks = [];
+}
+
+GlodaContent.prototype = {
+  kPriorityBase: 0,
+  kPriorityPerfect: 100,
+
+  kHunkMeta: 1,
+  kHunkQuoted: 2,
+  kHunkContent: 3,
+
+  _resetContent: function gloda_content__resetContent() {
+    this._keysAndValues = [];
+    this._keysAndDeltaValues = [];
+    this._hunks = [];
+    this._curHunk = null;
+  },
+
+  /* ===== Consumer API ===== */
+  hasContent: function gloda_content_hasContent() {
+    return (this._contentPriority != null);
+  },
+
+  /**
+   * Return content suitable for snippet display.  This means that no quoting
+   *  or meta-data should be returned.
+   *
+   * @param aMaxLength The maximum snippet length desired.
+   */
+  getContentSnippet: function gloda_content_getContentSnippet(aMaxLength) {
+    let content = this.getContentString();
+    if (aMaxLength)
+      content = content.substring(0, aMaxLength);
+    return content;
+  },
+
+  getContentString: function gloda_content_getContent(aIndexingPurposes) {
+    let data = "";
+    for (let hunk of this._hunks) {
+      if (hunk.hunkType == this.kHunkContent) {
+        if (data)
+          data += "\n" + hunk.data;
+        else
+          data = hunk.data;
+      }
+    }
+
+    if (aIndexingPurposes) {
+      // append the values for indexing.  we assume the keywords are cruft.
+      // this may be crazy, but things that aren't a science aren't an exact
+      // science.
+      for (let kv of this._keysAndValues) {
+        data += "\n" + kv[1];
+      }
+      for (let kon of this._keysAndValues) {
+        data += "\n" + kon[1] + "\n" + kon[2];
+      }
+    }
+
+    return data;
+  },
+
+  /* ===== Producer API ===== */
+  /**
+   * Called by a producer with the priority they believe their interpretation
+   *  of the content comes in at.
+   *
+   * @returns true if we believe the producer's interpretation will be
+   *     interesting and they should go ahead and generate events.  We return
+   *     false if we don't think they are interesting, in which case they should
+   *     probably not issue calls to us, although we don't care.  (We will
+   *     ignore their calls if we return false, this allows the simplification
+   *     of code that needs to run anyways.)
+   */
+  volunteerContent: function gloda_content_volunteerContent(aPriority) {
+    if (this._contentPriority === null || this._contentPriority < aPriority) {
+      this._contentPriority = aPriority;
+      this._resetContent();
+      this._producing = true;
+      return true;
+    }
+    this._producing = false;
+    return false;
+  },
+
+  keyValue: function gloda_content_keyValue(aKey, aValue) {
+    if (!this._producing)
+      return;
+
+    this._keysAndValues.push([aKey, aValue]);
+  },
+  keyValueDelta: function gloda_content_keyValueDelta (aKey, aOldValue,
+      aNewValue) {
+    if (!this._producing)
+      return;
+
+    this._keysAndDeltaValues.push([aKey, aOldValue, aNewValue]);
+  },
+
+  /**
+   * Meta lines are lines that have to do with the content but are not the
+   *  content and can generally be related to an attribute that has been derived
+   *  and stored on the item.
+   * For example, a bugzilla bug may note that an attachment was created; this
+   *  is not content and wouldn't be desired in a snippet, but is still
+   *  potentially interesting meta-data.
+   *
+   * @param aLineOrLines The line or list of lines that are meta-data.
+   * @param aAttr The attribute this meta-data is associated with.
+   * @param aIndex If the attribute is non-singular, indicate the specific
+   *     index of the item in the attribute's bound list that the meta-data
+   *     is associated with.
+   */
+  meta: function gloda_content_meta(aLineOrLines, aAttr, aIndex) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    this._curHunk = {hunkType: this.kHunkMeta, attr: aAttr, index: aIndex,
+                     data: data};
+    this._hunks.push(this._curHunk);
+  },
+  /**
+   * Quoted lines reference previous messages or what not.
+   *
+   * @param aLineOrLiens The line or list of lines that are quoted.
+   * @param aDepth The depth of the quoting.
+   * @param aOrigin The item that originated the original content, if known.
+   *     For example, perhaps a GlodaMessage?
+   * @param aTarget A reference to the location in the original content, if
+   *     known.  For example, the index of a line in a message or something?
+   */
+  quoted: function gloda_content_quoted(aLineOrLines, aDepth, aOrigin,
+      aTarget) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    if (!this._curHunk ||
+        this._curHunk.hunkType != this.kHunkQuoted ||
+        this._curHunk.depth != aDepth ||
+        this._curHunk.origin != aOrigin || this._curHunk.target != aTarget) {
+      this._curHunk = {hunkType: this.kHunkQuoted, data: data,
+                       depth: aDepth, origin: aOrigin, target: aTarget};
+      this._hunks.push(this._curHunk);
+    }
+    else
+      this._curHunk.data += "\n" + data;
+  },
+
+  content: function gloda_content_content(aLineOrLines) {
+    if (!this._producing)
+      return;
+
+    let data;
+    if (typeof(aLineOrLines) == "string")
+      data = aLineOrLines;
+    else
+      data = aLineOrLines.join("\n");
+
+    if (!this._curHunk || this._curHunk.hunkType != this.kHunkContent) {
+      this._curHunk = {hunkType: this.kHunkContent, data: data};
+      this._hunks.push(this._curHunk);
+    }
+    else
+      this._curHunk.data += "\n" + data;
+  },
+};

mailnews/db/gloda/modules/databind.js

@@ -0,0 +1,194 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaDatabind"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+var DBC_LOG = Log4Moz.repository.getLogger("gloda.ds.dbc");
+
+function GlodaDatabind(aNounDef, aDatastore) {
+  this._nounDef = aNounDef;
+  this._tableName = aNounDef.tableName;
+  this._tableDef = aNounDef.schema;
+  this._datastore = aDatastore;
+  this._log = Log4Moz.repository.getLogger("gloda.databind." + this._tableName);
+
+  // process the column definitions and make sure they have an attribute mapping
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    // default to the other dude's thing.
+    if (coldef.length < 3)
+      coldef[2] = coldef[0];
+    if (coldef[0] == "id")
+      this._idAttr = coldef[2];
+    // colDef[3] is the index of us in our SQL bindings, storage-numbering
+    coldef[3] = iColDef;
+  }
+
+  // XXX This is obviously synchronous and not perfectly async.  Since we are
+  //  doing this, we don't actually need to move to ordinal binding below
+  //  since we could just as well compel creation of the name map and thereby
+  //  avoid ever acquiring the mutex after bootstrap.
+  // However, this specific check can be cleverly avoided with future work.
+  // Namely, at startup we can scan for extension-defined tables and get their
+  //  maximum id so that we don't need to do it here.  The table will either
+  //  be brand new and thus have a maximum id of 1 or we will already know it
+  //  because of that scan.
+  this._nextId = 1;
+  let stmt = this._datastore._createSyncStatement(
+    "SELECT MAX(id) FROM " + this._tableName, true);
+  if (stmt.executeStep()) {  // no chance of this SQLITE_BUSY on this call
+    this._nextId = stmt.getInt64(0) + 1;
+  }
+  stmt.finalize();
+
+  let insertColumns = [];
+  let insertValues = [];
+  let updateItems = [];
+  for (let [iColDef, coldef] of this._tableDef.columns.entries()) {
+    let column = coldef[0];
+    let placeholder = "?" + (iColDef + 1);
+    insertColumns.push(column);
+    insertValues.push(placeholder);
+    if (column != "id") {
+      updateItems.push(column + " = " + placeholder);
+    }
+  }
+
+  let insertSql = "INSERT INTO " + this._tableName + " (" +
+    insertColumns.join(", ") + ") VALUES (" + insertValues.join(", ") + ")";
+
+  // For the update, we want the 'id' to be a constraint and not a value
+  //  that gets set...
+  let updateSql = "UPDATE " + this._tableName + " SET " +
+    updateItems.join(", ") + " WHERE id = ?1";
+  this._insertStmt = aDatastore._createAsyncStatement(insertSql);
+  this._updateStmt = aDatastore._createAsyncStatement(updateSql);
+
+  if (this._tableDef.fulltextColumns) {
+    for (let [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      if (coldef.length < 3)
+        coldef[2] = coldef[0];
+      // colDef[3] is the index of us in our SQL bindings, storage-numbering
+      coldef[3] = iColDef + 1;
+    }
+
+    let insertColumns = [];
+    let insertValues = [];
+    let updateItems = [];
+    for (var [iColDef, coldef] of this._tableDef.fulltextColumns.entries()) {
+      let column = coldef[0];
+      // +2 instead of +1 because docid is implied
+      let placeholder = "?" + (iColDef + 2);
+      insertColumns.push(column);
+      insertValues.push(placeholder);
+      if (column != "id") {
+        updateItems.push(column + " = " + placeholder);
+      }
+    }
+
+    let insertFulltextSql = "INSERT INTO " + this._tableName + "Text (docid," +
+      insertColumns.join(", ") + ") VALUES (?1," + insertValues.join(", ") +
+      ")";
+
+    // For the update, we want the 'id' to be a constraint and not a value
+    //  that gets set...
+    let updateFulltextSql = "UPDATE " + this._tableName + "Text SET " +
+      updateItems.join(", ") + " WHERE docid = ?1";
+
+    this._insertFulltextStmt =
+      aDatastore._createAsyncStatement(insertFulltextSql);
+    this._updateFulltextStmt =
+      aDatastore._createAsyncStatement(updateFulltextSql);
+  }
+}
+
+GlodaDatabind.prototype = {
+  /**
+   * Perform appropriate binding coercion based on the schema provided to us.
+   * Although we end up effectively coercing JS Date objects to numeric values,
+   *  we should not be provided with JS Date objects!  There is no way for us
+   *  to know to turn them back into JS Date objects on the way out.
+   *  Additionally, there is the small matter of storage's bias towards
+   *  PRTime representations which may not always be desirable.
+   */
+  bindByType: function(aStmt, aColDef, aValue) {
+    if (aValue == null)
+      aStmt.bindNullParameter(aColDef[3]);
+    else if (aColDef[1] == "STRING" || aColDef[1] == "TEXT")
+      aStmt.bindStringParameter(aColDef[3], aValue);
+    else
+      aStmt.bindInt64Parameter(aColDef[3], aValue);
+  },
+
+  objFromRow: function(aRow) {
+    let getVariant = this._datastore._getVariant;
+    let obj = new this._nounDef.class();
+    for (let [iCol, colDef] of this._tableDef.columns.entries()) {
+      obj[colDef[2]] = getVariant(aRow, iCol);
+    }
+    return obj;
+  },
+
+  objInsert: function(aThing) {
+    let bindByType = this.bindByType;
+    if (!aThing[this._idAttr])
+      aThing[this._idAttr] = this._nextId++;
+
+    let stmt = this._insertStmt;
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._insertFulltextStmt) {
+      stmt = this._insertFulltextStmt;
+      stmt.bindInt64Parameter(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  objUpdate: function(aThing) {
+    let bindByType = this.bindByType;
+    let stmt = this._updateStmt;
+    // note, we specially bound the location of 'id' for the insert, but since
+    //  we're using named bindings, there is nothing special about setting it
+    for (let colDef of this._tableDef.columns) {
+      bindByType(stmt, colDef, aThing[colDef[2]]);
+    }
+    stmt.executeAsync(this._datastore.trackAsync());
+
+    if (this._updateFulltextStmt) {
+      stmt = this._updateFulltextStmt;
+      // fulltextColumns doesn't include id/docid, need to explicitly set it
+      stmt.bindInt64Parameter(0, aThing[this._idAttr]);
+      for (let colDef of this._tableDef.fulltextColumns) {
+        bindByType(stmt, colDef, aThing[colDef[2]]);
+      }
+      stmt.executeAsync(this._datastore.trackAsync());
+    }
+  },
+
+  adjustAttributes: function() {
+    // just proxy the call over to the datastore... we have to do this for
+    //  'this' reasons.  we don't refactor things to avoid this because it does
+    //  make some sense to have all the methods exposed from a single object,
+    //  even if the implementation does live elsewhere.
+    return this._datastore.adjustAttributes.apply(this._datastore, arguments);
+  },
+
+  // also proxied...
+  queryFromQuery: function() {
+    return this._datastore.queryFromQuery.apply(this._datastore, arguments);
+  }
+};

mailnews/db/gloda/modules/datamodel.js

@@ -0,0 +1,907 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaAttributeDBDef", "GlodaAccount",
+                    "GlodaConversation", "GlodaFolder", "GlodaMessage",
+                    "GlodaContact", "GlodaIdentity", "GlodaAttachment"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/mailServices.js");
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+var LOG = Log4Moz.repository.getLogger("gloda.datamodel");
+
+Cu.import("resource:///modules/gloda/utils.js");
+
+// Make it lazy.
+var gMessenger;
+function getMessenger () {
+  if (!gMessenger)
+    gMessenger = Cc["@mozilla.org/messenger;1"].createInstance(Ci.nsIMessenger);
+  return gMessenger;
+}
+
+/**
+ * @class Represents a gloda attribute definition's DB form.  This class
+ *  stores the information in the database relating to this attribute
+ *  definition.  Access its attrDef attribute to get at the realy juicy data.
+ *  This main interesting thing this class does is serve as the keeper of the
+ *  mapping from parameters to attribute ids in the database if this is a
+ *  parameterized attribute.
+ */
+function GlodaAttributeDBDef(aDatastore, aID, aCompoundName, aAttrType,
+                           aPluginName, aAttrName) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._compoundName = aCompoundName;
+  this._attrType = aAttrType;
+  this._pluginName = aPluginName;
+  this._attrName = aAttrName;
+
+  this.attrDef = null;
+
+  /** Map parameter values to the underlying database id. */
+  this._parameterBindings = {};
+}
+
+GlodaAttributeDBDef.prototype = {
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get attributeName() { return this._attrName; },
+
+  get parameterBindings() { return this._parameterBindings; },
+
+  /**
+   * Bind a parameter value to the attribute definition, allowing use of the
+   *  attribute-parameter as an attribute.
+   *
+   * @return
+   */
+  bindParameter: function gloda_attr_bindParameter(aValue) {
+    // people probably shouldn't call us with null, but handle it
+    if (aValue == null) {
+      return this._id;
+    }
+    if (aValue in this._parameterBindings) {
+      return this._parameterBindings[aValue];
+    }
+    // no database entry exists if we are here, so we must create it...
+    let id = this._datastore._createAttributeDef(this._attrType,
+                 this._pluginName, this._attrName, aValue);
+    this._parameterBindings[aValue] = id;
+    this._datastore.reportBinding(id, this, aValue);
+    return id;
+  },
+
+  /**
+   * Given a list of values, return a list (regardless of plurality) of
+   *  database-ready [attribute id,  value] tuples.  This is intended to be used
+   *  to directly convert the value of a property on an object that corresponds
+   *  to a bound attribute.
+   *
+   * @param {Array} aInstanceValues An array of instance values regardless of
+   *     whether or not the attribute is singular.
+   */
+  convertValuesToDBAttributes:
+      function gloda_attr_convertValuesToDBAttributes(aInstanceValues) {
+    let nounDef = this.attrDef.objectNounDef;
+    let dbAttributes = [];
+    if (nounDef.usesParameter) {
+      for (let instanceValue of aInstanceValues) {
+        let [param, dbValue] = nounDef.toParamAndValue(instanceValue);
+        dbAttributes.push([this.bindParameter(param), dbValue]);
+      }
+    }
+    else {
+      // Not generating any attributes is ok. This basically means the noun is
+      // just an informative property on the Gloda Message and has no real
+      // indexing purposes.
+      if ("toParamAndValue" in nounDef) {
+        for (let instanceValue of aInstanceValues) {
+          dbAttributes.push([this._id,
+                             nounDef.toParamAndValue(instanceValue)[1]]);
+        }
+      }
+    }
+    return dbAttributes;
+  },
+
+  toString: function() {
+    return this._compoundName;
+  }
+};
+
+var GlodaHasAttributesMixIn = {
+  enumerateAttributes: function* gloda_attrix_enumerateAttributes() {
+    let nounDef = this.NOUN_DEF;
+    for (let key in this) {
+      let value = this[key];
+      let attrDef = nounDef.attribsByBoundName[key];
+      // we expect to not have attributes for underscore prefixed values (those
+      //  are managed by the instance's logic.  we also want to not explode
+      //  should someone crap other values in there, we get both birds with this
+      //  one stone.
+      if (attrDef === undefined)
+        continue;
+      if (attrDef.singular) {
+        // ignore attributes with null values
+        if (value != null)
+          yield [attrDef, [value]];
+      }
+      else {
+        // ignore attributes with no values
+        if (value.length)
+          yield [attrDef, value];
+      }
+    }
+  },
+
+  domContribute: function gloda_attrix_domContribute(aDomNode) {
+    let nounDef = this.NOUN_DEF;
+    for (let attrName in nounDef.domExposeAttribsByBoundName) {
+      let attr = nounDef.domExposeAttribsByBoundName[attrName];
+      if (this[attrName])
+        aDomNode.setAttribute(attr.domExpose, this[attrName]);
+    }
+  },
+};
+
+function MixIn(aConstructor, aMixIn) {
+  let proto = aConstructor.prototype;
+  for (let [name, func] in Iterator(aMixIn)) {
+    if (name.startsWith("get_"))
+      proto.__defineGetter__(name.substring(4), func);
+    else
+      proto[name] = func;
+  }
+}
+
+/**
+ * @class A gloda wrapper around nsIMsgIncomingServer.
+ */
+function GlodaAccount(aIncomingServer) {
+  this._incomingServer = aIncomingServer;
+}
+
+GlodaAccount.prototype = {
+  NOUN_ID: 106,
+  get id() { return this._incomingServer.key; },
+  get name() { return this._incomingServer.prettyName; },
+  get incomingServer() { return this._incomingServer; },
+  toString: function gloda_account_toString() {
+    return "Account: " + this.id;
+  },
+
+  toLocaleString: function gloda_account_toLocaleString() {
+    return this.name;
+  }
+};
+
+/**
+ * @class A gloda conversation (thread) exists so that messages can belong.
+ */
+function GlodaConversation(aDatastore, aID, aSubject, aOldestMessageDate,
+                           aNewestMessageDate) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._subject = aSubject;
+  this._oldestMessageDate = aOldestMessageDate;
+  this._newestMessageDate = aNewestMessageDate;
+}
+
+GlodaConversation.prototype = {
+  NOUN_ID: 101,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get subject() { return this._subject; },
+  get oldestMessageDate() { return this._oldestMessageDate; },
+  get newestMessageDate() { return this._newestMessageDate; },
+
+  getMessagesCollection: function gloda_conversation_getMessagesCollection(
+    aListener, aData) {
+    let query = new GlodaMessage.prototype.NOUN_DEF.queryClass();
+    query.conversation(this._id).orderBy("date");
+    return query.getCollection(aListener, aData);
+  },
+
+  toString: function gloda_conversation_toString() {
+    return "Conversation:" + this._id;
+  },
+
+  toLocaleString: function gloda_conversation_toLocaleString() {
+    return this._subject;
+  }
+};
+
+function GlodaFolder(aDatastore, aID, aURI, aDirtyStatus, aPrettyName,
+                     aIndexingPriority) {
+  // _datastore is now set by GlodaDatastore
+  this._id = aID;
+  this._uri = aURI;
+  this._dirtyStatus = aDirtyStatus;
+  this._prettyName = aPrettyName;
+  this._xpcomFolder = null;
+  this._account = null;
+  this._activeIndexing = false;
+  this._activeHeaderRetrievalLastStamp = 0;
+  this._indexingPriority = aIndexingPriority;
+  this._deleted = false;
+  this._compacting = false;
+}
+
+GlodaFolder.prototype = {
+  NOUN_ID: 100,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  /** The folder is believed to be up-to-date */
+  kFolderClean: 0,
+  /** The folder has some un-indexed or dirty messages */
+  kFolderDirty: 1,
+  /** The folder needs to be entirely re-indexed, regardless of the flags on
+   * the messages in the folder. This state will be downgraded to dirty */
+  kFolderFilthy: 2,
+
+  _kFolderDirtyStatusMask: 0x7,
+  /**
+   * The (local) folder has been compacted and all of its message keys are
+   *  potentially incorrect.  This is not a possible state for IMAP folders
+   *  because their message keys are based on UIDs rather than offsets into
+   *  the mbox file.
+   */
+  _kFolderCompactedFlag: 0x8,
+
+  /** The folder should never be indexed. */
+  kIndexingNeverPriority: -1,
+  /** The lowest priority assigned to a folder. */
+  kIndexingLowestPriority: 0,
+  /** The highest priority assigned to a folder. */
+  kIndexingHighestPriority: 100,
+
+  /** The indexing priority for a folder if no other priority is assigned. */
+  kIndexingDefaultPriority: 20,
+  /** Folders marked check new are slightly more important I guess. */
+  kIndexingCheckNewPriority: 30,
+  /** Favorite folders are more interesting to the user, presumably. */
+  kIndexingFavoritePriority: 40,
+  /** The indexing priority for inboxes. */
+  kIndexingInboxPriority: 50,
+  /** The indexing priority for sent mail folders. */
+  kIndexingSentMailPriority: 60,
+
+  get id() { return this._id; },
+  get uri() { return this._uri; },
+  get dirtyStatus() {
+    return this._dirtyStatus & this._kFolderDirtyStatusMask;
+  },
+  /**
+   * Mark a folder as dirty if it was clean.  Do nothing if it was already dirty
+   *  or filthy.  For use by GlodaMsgIndexer only.  And maybe rkent and his
+   *  marvelous extensions.
+   */
+  _ensureFolderDirty: function gloda_folder__markFolderDirty() {
+    if (this.dirtyStatus == this.kFolderClean) {
+      this._dirtyStatus = (this.kFolderDirty & this._kFolderDirtyStatusMask) |
+                          (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Definitely for use only by GlodaMsgIndexer to downgrade the dirty status of
+   *  a folder.
+   */
+  _downgradeDirtyStatus: function gloda_folder__downgradeDirtyStatus(
+                           aNewStatus) {
+    if (this.dirtyStatus != aNewStatus) {
+      this._dirtyStatus = (aNewStatus & this._kFolderDirtyStatusMask) |
+                          (this._dirtyStatus & ~this._kFolderDirtyStatusMask);
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  /**
+   * Indicate whether this folder is currently being compacted.  The
+   *  |GlodaMsgIndexer| keeps this in-memory-only value up-to-date.
+   */
+  get compacting() {
+    return this._compacting;
+  },
+  /**
+   * Set whether this folder is currently being compacted.  This is really only
+   *  for the |GlodaMsgIndexer| to set.
+   */
+  set compacting(aCompacting) {
+    this._compacting = aCompacting;
+  },
+  /**
+   * Indicate whether this folder was compacted and has not yet been
+   *  compaction processed.
+   */
+  get compacted() {
+    return Boolean(this._dirtyStatus & this._kFolderCompactedFlag);
+  },
+  /**
+   * For use only by GlodaMsgIndexer to set/clear the compaction state of this
+   *  folder.
+   */
+  _setCompactedState: function gloda_folder__clearCompactedState(aCompacted) {
+    if (this.compacted != aCompacted) {
+      if (aCompacted)
+        this._dirtyStatus |= this._kFolderCompactedFlag;
+      else
+        this._dirtyStatus &= ~this._kFolderCompactedFlag;
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+
+  get name() { return this._prettyName; },
+  toString: function gloda_folder_toString() {
+    return "Folder:" + this._id;
+  },
+
+  toLocaleString: function gloda_folder_toLocaleString() {
+    let xpcomFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+    if (!xpcomFolder)
+      return this._prettyName;
+    return xpcomFolder.prettiestName +
+      " (" + xpcomFolder.rootFolder.prettiestName + ")";
+  },
+
+  get indexingPriority() {
+    return this._indexingPriority;
+  },
+
+  /** We are going to index this folder. */
+  kActivityIndexing: 0,
+  /** Asking for the folder to perform header retrievals. */
+  kActivityHeaderRetrieval: 1,
+  /** We only want the folder for its metadata but are not going to open it. */
+  kActivityFolderOnlyNoData: 2,
+
+
+  /** Is this folder known to be actively used for indexing? */
+  _activeIndexing: false,
+  /** Get our indexing status. */
+  get indexing() {
+    return this._activeIndexing;
+  },
+  /**
+   * Set our indexing status.  Normally, this will be enabled through passing
+   *  an activity type of kActivityIndexing (which will set us), but we will
+   *  still need to be explicitly disabled by the indexing code.
+   * When disabling indexing, we will call forgetFolderIfUnused to take care of
+   *  shutting things down.
+   * We are not responsible for committing changes to the message database!
+   *  That is on you!
+   */
+  set indexing(aIndexing) {
+    this._activeIndexing = aIndexing;
+    if (!aIndexing)
+      this.forgetFolderIfUnused();
+  },
+  /** When was this folder last used for header retrieval purposes? */
+  _activeHeaderRetrievalLastStamp: 0,
+
+  /**
+   * Retrieve the nsIMsgFolder instance corresponding to this folder, providing
+   *  an explanation of why you are requesting it for tracking/cleanup purposes.
+   *
+   * @param aActivity One of the kActivity* constants.  If you pass
+   *     kActivityIndexing, we will set indexing for you, but you will need to
+   *     clear it when you are done.
+   * @return The nsIMsgFolder if available, null on failure.
+   */
+  getXPCOMFolder: function gloda_folder_getXPCOMFolder(aActivity) {
+    if (!this._xpcomFolder) {
+      let rdfService = Cc['@mozilla.org/rdf/rdf-service;1']
+                         .getService(Ci.nsIRDFService);
+      this._xpcomFolder = rdfService.GetResource(this.uri)
+                                    .QueryInterface(Ci.nsIMsgFolder);
+    }
+    switch (aActivity) {
+      case this.kActivityIndexing:
+        // mark us as indexing, but don't bother with live tracking.  we do
+        //  that independently and only for header retrieval.
+        this.indexing = true;
+        break;
+      case this.kActivityHeaderRetrieval:
+        if (this._activeHeaderRetrievalLastStamp === 0)
+          this._datastore.markFolderLive(this);
+        this._activeHeaderRetrievalLastStamp = Date.now();
+        break;
+      case this.kActivityFolderOnlyNoData:
+        // we don't have to do anything here.
+        break;
+    }
+
+    return this._xpcomFolder;
+  },
+
+  /**
+   * Retrieve a GlodaAccount instance corresponding to this folder.
+   *
+   * @return The GlodaAccount instance.
+   */
+  getAccount: function gloda_folder_getAccount() {
+    if (!this._account) {
+      let msgFolder = this.getXPCOMFolder(this.kActivityFolderOnlyNoData);
+      this._account = new GlodaAccount(msgFolder.server);
+    }
+    return this._account;
+  },
+
+  /**
+   * How many milliseconds must a folder have not had any header retrieval
+   *  activity before it's okay to lose the database reference?
+   */
+  ACCEPTABLY_OLD_THRESHOLD: 10000,
+
+  /**
+   * Cleans up our nsIMsgFolder reference if we have one and it's not "in use".
+   * In use, from our perspective, means that it is not being used for indexing
+   *  and some arbitrary interval of time has elapsed since it was last
+   *  retrieved for header retrieval reasons.  The time interval is because if
+   *  we have one GlodaMessage requesting a header, there's a high probability
+   *  that another message will request a header in the near future.
+   * Because setting indexing to false disables us, we are written in an
+   *  idempotent fashion.  (It is possible for disabling indexing's call to us
+   *  to cause us to return true but for the datastore's timer call to have not
+   *  yet triggered.)
+   *
+   * @returns true if we are cleaned up and can be considered 'dead', false if
+   *     we should still be considered alive and this method should be called
+   *     again in the future.
+   */
+  forgetFolderIfUnused: function gloda_folder_forgetFolderIfUnused() {
+    // we are not cleaning/cleaned up if we are indexing
+    if (this._activeIndexing)
+      return false;
+
+    // set a point in the past as the threshold.  the timestamp must be older
+    //  than this to be eligible for cleanup.
+    let acceptablyOld = Date.now() - this.ACCEPTABLY_OLD_THRESHOLD;
+    // we are not cleaning/cleaned up if we have retrieved a header more
+    //  recently than the acceptably old threshold.
+    if (this._activeHeaderRetrievalLastStamp > acceptablyOld)
+      return false;
+
+    if (this._xpcomFolder) {
+      // This is the key action we take; the nsIMsgFolder will continue to
+      //  exist, but we want it to forget about its database so that it can
+      //  be closed and its memory can be reclaimed.
+      this._xpcomFolder.msgDatabase = null;
+      this._xpcomFolder = null;
+      // since the last retrieval time tracks whether we have marked live or
+      //  not, this needs to be reset to 0 too.
+      this._activeHeaderRetrievalLastStamp = 0;
+    }
+
+    return true;
+  }
+};
+
+/**
+ * @class A message representation.
+ */
+function GlodaMessage(aDatastore, aID, aFolderID, aMessageKey,
+                      aConversationID, aConversation, aDate,
+                      aHeaderMessageID, aDeleted, aJsonText,
+                      aNotability,
+                      aSubject, aIndexedBodyText, aAttachmentNames) {
+  // _datastore is now set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._folderID = aFolderID;
+  this._messageKey = aMessageKey;
+  this._conversationID = aConversationID;
+  this._conversation = aConversation;
+  this._date = aDate;
+  this._headerMessageID = aHeaderMessageID;
+  this._jsonText = aJsonText;
+  this._notability = aNotability;
+  this._subject = aSubject;
+  this._indexedBodyText = aIndexedBodyText;
+  this._attachmentNames = aAttachmentNames;
+
+  // only set _deleted if we're deleted, otherwise the undefined does our
+  //  speaking for us.
+  if (aDeleted)
+    this._deleted = aDeleted;
+}
+
+GlodaMessage.prototype = {
+  NOUN_ID: 102,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get folderID() { return this._folderID; },
+  get messageKey() { return this._messageKey; },
+  get conversationID() { return this._conversationID; },
+  // conversation is special
+  get headerMessageID() { return this._headerMessageID; },
+  get notability() { return this._notability; },
+  set notability(aNotability) { this._notability = aNotability; },
+
+  get subject() { return this._subject; },
+  get indexedBodyText() { return this._indexedBodyText; },
+  get attachmentNames() { return this._attachmentNames; },
+
+  get date() { return this._date; },
+  set date(aNewDate) { this._date = aNewDate; },
+
+  get folder() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID != null)
+        return this._datastore._mapFolderID(this._folderID);
+    }
+    catch (ex) {
+    }
+    return null;
+  },
+  get folderURI() {
+    // XXX just like for folder, handle mapping failures and return null
+    try {
+      if (this._folderID != null)
+        return this._datastore._mapFolderID(this._folderID).uri;
+    }
+    catch (ex) {
+    }
+    return null;
+  },
+  get account() {
+    // XXX due to a deletion bug it is currently possible to get in a state
+    //  where we have an illegal folderID value.  This will result in an
+    //  exception.  As a workaround, let's just return null in that case.
+    try {
+      if (this._folderID == null)
+        return null;
+      let folder = this._datastore._mapFolderID(this._folderID);
+      return folder.getAccount();
+    }
+    catch (ex) { }
+    return null;
+  },
+  get conversation() {
+    return this._conversation;
+  },
+
+  toString: function gloda_message_toString() {
+    // uh, this is a tough one...
+    return "Message:" + this._id;
+  },
+
+  _clone: function gloda_message_clone() {
+    return new GlodaMessage(/* datastore */ null, this._id, this._folderID,
+      this._messageKey, this._conversationID, this._conversation, this._date,
+      this._headerMessageID, "_deleted" in this ? this._deleted : undefined,
+      "_jsonText" in this ? this._jsonText : undefined, this._notability,
+      this._subject, this._indexedBodyText, this._attachmentNames);
+  },
+
+  /**
+   * Provide a means of propagating changed values on our clone back to
+   *  ourselves.  This is required because of an object identity trick gloda
+   *  does; when indexing an already existing object, all mutations happen on
+   *  a clone of the existing object so that
+   */
+  _declone: function gloda_message_declone(aOther) {
+    if ("_content" in aOther)
+      this._content = aOther._content;
+
+    // The _indexedAuthor/_indexedRecipients fields don't get updated on
+    //  fulltext update so we don't need to propagate.
+    this._indexedBodyText = aOther._indexedBodyText;
+    this._attachmentNames = aOther._attachmentNames;
+  },
+
+  /**
+   * Mark this message as a ghost.  Ghosts are characterized by having no folder
+   *  id and no message key.  They also are not deleted or they would be of
+   *  absolutely no use to us.
+   *
+   * These changes are suitable for persistence.
+   */
+  _ghost: function gloda_message_ghost() {
+    this._folderID = null;
+    this._messageKey = null;
+    if ("_deleted" in this)
+      delete this._deleted;
+  },
+
+  /**
+   * Are we a ghost (which implies not deleted)?  We are not a ghost if we have
+   *  a definite folder location (we may not know our message key in the case
+   *  of IMAP moves not fully completed) and are not deleted.
+   */
+  get _isGhost() {
+    return this._folderID == null && !this._isDeleted;
+  },
+
+  /**
+   * If we were dead, un-dead us.
+   */
+  _ensureNotDeleted: function gloda_message__ensureNotDeleted() {
+    if ("_deleted" in this)
+      delete this._deleted;
+  },
+
+  /**
+   * Are we deleted?  This is private because deleted gloda messages are not
+   *  visible to non-core-gloda code.
+   */
+  get _isDeleted() {
+    return ("_deleted" in this) && this._deleted;
+  },
+
+  /**
+   * Trash this message's in-memory representation because it should no longer
+   *  be reachable by any code.  The database record is gone, it's not coming
+   *  back.
+   */
+  _objectPurgedMakeYourselfUnpleasant: function gloda_message_nuke() {
+    this._id = null;
+    this._folderID = null;
+    this._messageKey = null;
+    this._conversationID = null;
+    this._conversation = null;
+    this.date = null;
+    this._headerMessageID = null;
+  },
+
+  /**
+   * Return the underlying nsIMsgDBHdr from the folder storage for this, or
+   *  null if the message does not exist for one reason or another.  We may log
+   *  to our logger in the failure cases.
+   *
+   * This method no longer caches the result, so if you need to hold onto it,
+   *  hold onto it.
+   *
+   * In the process of retrieving the underlying message header, we may have to
+   *  open the message header database associated with the folder.  This may
+   *  result in blocking while the load happens, so you may want to try and find
+   *  an alternate way to initiate the load before calling us.
+   * We provide hinting to the GlodaDatastore via the GlodaFolder so that it
+   *  knows when it's a good time for it to go and detach from the database.
+   *
+   * @returns The nsIMsgDBHdr associated with this message if available, null on
+   *     failure.
+   */
+  get folderMessage() {
+    if (this._folderID === null || this._messageKey === null)
+      return null;
+
+    // XXX like for folder and folderURI, return null if we can't map the folder
+    let glodaFolder;
+    try {
+      glodaFolder = this._datastore._mapFolderID(this._folderID);
+    }
+    catch (ex) {
+      return null;
+    }
+    let folder = glodaFolder.getXPCOMFolder(
+                   glodaFolder.kActivityHeaderRetrieval);
+    if (folder) {
+      let folderMessage;
+      try {
+        folderMessage = folder.GetMessageHeader(this._messageKey);
+      }
+      catch (ex) {
+        folderMessage = null;
+      }
+      if (folderMessage !== null) {
+        // verify the message-id header matches what we expect...
+        if (folderMessage.messageId != this._headerMessageID) {
+          LOG.info("Message with message key " + this._messageKey +
+                   " in folder '" + folder.URI + "' does not match expected " +
+                   "header! (" + this._headerMessageID + " expected, got " +
+                   folderMessage.messageId + ")");
+          folderMessage = null;
+        }
+      }
+      return folderMessage;
+    }
+
+    // this only gets logged if things have gone very wrong.  we used to throw
+    //  here, but it's unlikely our caller can do anything more meaningful than
+    //  treating this as a disappeared message.
+    LOG.info("Unable to locate folder message for: " + this._folderID + ":" +
+             this._messageKey);
+    return null;
+  },
+  get folderMessageURI() {
+    let folderMessage = this.folderMessage;
+    if (folderMessage)
+      return folderMessage.folder.getUriForMsg(folderMessage);
+    else
+      return null;
+  }
+};
+MixIn(GlodaMessage, GlodaHasAttributesMixIn);
+
+/**
+ * @class Contacts correspond to people (one per person), and may own multiple
+ *  identities (e-mail address, IM account, etc.)
+ */
+function GlodaContact(aDatastore, aID, aDirectoryUUID, aContactUUID, aName,
+                      aPopularity, aFrecency, aJsonText) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._directoryUUID = aDirectoryUUID;
+  this._contactUUID = aContactUUID;
+  this._name = aName;
+  this._popularity = aPopularity;
+  this._frecency = aFrecency;
+  if (aJsonText)
+    this._jsonText = aJsonText;
+
+  this._identities = null;
+}
+
+GlodaContact.prototype = {
+  NOUN_ID: 103,
+  // set by GlodaDatastore
+  _datastore: null,
+
+  get id() { return this._id; },
+  get directoryUUID() { return this._directoryUUID; },
+  get contactUUID() { return this._contactUUID; },
+  get name() { return this._name; },
+  set name(aName) { this._name = aName; },
+
+  get popularity() { return this._popularity; },
+  set popularity(aPopularity) {
+    this._popularity = aPopularity;
+    this.dirty = true;
+  },
+
+  get frecency() { return this._frecency; },
+  set frecency(aFrecency) {
+    this._frecency = aFrecency;
+    this.dirty = true;
+  },
+
+  get identities() {
+    return this._identities;
+  },
+
+  toString: function gloda_contact_toString() {
+    return "Contact:" + this._id;
+  },
+
+  get accessibleLabel() {
+    return "Contact: " + this._name;
+  },
+
+  _clone: function gloda_contact_clone() {
+    return new GlodaContact(/* datastore */ null, this._id, this._directoryUUID,
+      this._contactUUID, this._name, this._popularity, this._frecency);
+  },
+};
+MixIn(GlodaContact, GlodaHasAttributesMixIn);
+
+
+/**
+ * @class A specific means of communication for a contact.
+ */
+function GlodaIdentity(aDatastore, aID, aContactID, aContact, aKind, aValue,
+                       aDescription, aIsRelay) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._id = aID;
+  this._contactID = aContactID;
+  this._contact = aContact;
+  this._kind = aKind;
+  this._value = aValue;
+  this._description = aDescription;
+  this._isRelay = aIsRelay;
+  /// Cached indication of whether there is an address book card for this
+  ///  identity.  We keep this up-to-date via address book listener
+  ///  notifications in |GlodaABIndexer|.
+  this._hasAddressBookCard = undefined;
+}
+
+GlodaIdentity.prototype = {
+  NOUN_ID: 104,
+  // set by GlodaDatastore
+  _datastore: null,
+  get id() { return this._id; },
+  get contactID() { return this._contactID; },
+  get contact() { return this._contact; },
+  get kind() { return this._kind; },
+  get value() { return this._value; },
+  get description() { return this._description; },
+  get isRelay() { return this._isRelay; },
+
+  get uniqueValue() {
+    return this._kind + "@" + this._value;
+  },
+
+  toString: function gloda_identity_toString() {
+    return "Identity:" + this._kind + ":" + this._value;
+  },
+
+  toLocaleString: function gloda_identity_toLocaleString() {
+    if (this.contact.name == this.value)
+      return this.value;
+    return this.contact.name + " : " + this.value;
+  },
+
+  get abCard() {
+    // for our purposes, the address book only speaks email
+    if (this._kind != "email")
+      return false;
+    let card = GlodaUtils.getCardForEmail(this._value);
+    this._hasAddressBookCard = (card != null);
+    return card;
+  },
+
+  /**
+   * Indicates whether we have an address book card for this identity.  This
+   *  value is cached once looked-up and kept up-to-date by |GlodaABIndexer|
+   *  and its notifications.
+   */
+  get inAddressBook() {
+    if (this._hasAddressBookCard !== undefined)
+      return this._hasAddressBookCard;
+    return (this.abCard && true) || false;
+  },
+
+  pictureURL: function(aSize) {
+    if (this.inAddressBook) {
+      // XXX should get the photo if we have it.
+    }
+    return "";
+  }
+};
+
+
+/**
+ * An attachment, with as much information as we can gather on it
+ */
+function GlodaAttachment(aGlodaMessage, aName, aContentType, aSize, aPart, aExternalUrl, aIsExternal) {
+  // _datastore set on the prototype by GlodaDatastore
+  this._glodaMessage = aGlodaMessage;
+  this._name = aName;
+  this._contentType = aContentType;
+  this._size = aSize;
+  this._part = aPart;
+  this._externalUrl = aExternalUrl;
+  this._isExternal = aIsExternal;
+}
+
+GlodaAttachment.prototype = {
+  NOUN_ID: 105,
+  // set by GlodaDatastore
+  get name() { return this._name; },
+  get contentType() { return this._contentType; },
+  get size() { return this._size; },
+  get url() {
+    if (this.isExternal)
+      return this._externalUrl;
+    else {
+      let uri = this._glodaMessage.folderMessageURI;
+      if (!uri)
+        throw new Error("The message doesn't exist anymore, unable to rebuild attachment URL");
+      let neckoURL = {};
+      let msgService = getMessenger().messageServiceFromURI(uri);
+      msgService.GetUrlForUri(uri, neckoURL, null);
+      let url = neckoURL.value.spec;
+      let hasParamAlready = url.match(/\?[a-z]+=[^\/]+$/);
+      let sep = hasParamAlready ? "&" : "?";
+      return url+sep+"part="+this._part+"&filename="+encodeURIComponent(this._name);
+    }
+  },
+  get isExternal() { return this._isExternal; },
+
+  toString: function gloda_attachment_toString() {
+    return "attachment: " + this._name + ":" + this._contentType;
+  },
+
+};

mailnews/db/gloda/modules/dbview.js

@@ -0,0 +1,178 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file is charged with providing you a way to have a pretty gloda-backed
+ *  nsIMsgDBView.
+ */
+
+this.EXPORTED_SYMBOLS = ["GlodaSyntheticView"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+Cu.import("resource:///modules/gloda/public.js");
+Cu.import("resource:///modules/gloda/msg_search.js");
+
+/**
+ * Create a synthetic view suitable for passing to |FolderDisplayWidget.show|.
+ * You must pass a query, collection, or conversation in.
+ *
+ * @param {GlodaQuery} [aArgs.query] A gloda query to run.
+ * @param {GlodaCollection} [aArgs.collection] An already-populated collection
+ *     to display.  Do not call getCollection on a query and hand us that.  We
+ *     will not register ourselves as a listener and things will not work.
+ * @param {GlodaConversation} [aArgs.conversation] A conversation whose messages
+ *     you want to display.
+ */
+function GlodaSyntheticView(aArgs) {
+  if ("query" in aArgs) {
+    this.query = aArgs.query;
+    this.collection = this.query.getCollection(this);
+    this.completed = false;
+    this.viewType = "global";
+  }
+  else if ("collection" in aArgs) {
+    this.query = null;
+    this.collection = aArgs.collection;
+    this.completed = true;
+    this.viewType = "global";
+  }
+  else if ("conversation" in aArgs) {
+    this.collection = aArgs.conversation.getMessagesCollection(this);
+    this.query = this.collection.query;
+    this.completed = false;
+    this.viewType = "conversation";
+  }
+  else {
+    throw new Error("You need to pass a query or collection");
+  }
+
+  this.customColumns = [];
+}
+GlodaSyntheticView.prototype = {
+  defaultSort: [[Ci.nsMsgViewSortType.byDate, Ci.nsMsgViewSortOrder.descending]],
+
+  /**
+   * Request the search be performed and notification provided to
+   *  aSearchListener.  If results are already available, they should
+   *  be provided to aSearchListener without re-performing the search.
+   */
+  search: function(aSearchListener, aCompletionCallback) {
+    this.searchListener = aSearchListener;
+    this.completionCallback = aCompletionCallback;
+
+    this.searchListener.onNewSearch();
+    if (this.completed) {
+      this.reportResults(this.collection.items);
+      // we're not really aborting, but it closes things out nicely
+      this.abortSearch();
+      return;
+    }
+  },
+
+  abortSearch: function() {
+    if (this.searchListener)
+      this.searchListener.onSearchDone(Cr.NS_OK);
+    if (this.completionCallback)
+      this.completionCallback();
+    this.searchListener = null;
+    this.completionCallback = null;
+  },
+
+  reportResults: function(aItems) {
+    for (let item of aItems) {
+      let hdr = item.folderMessage;
+      if (hdr)
+        this.searchListener.onSearchHit(hdr, hdr.folder);
+    }
+  },
+
+  /**
+   * Helper function used by |DBViewWrapper.getMsgHdrForMessageID| since there
+   *  are no actual backing folders for it to check.
+   */
+  getMsgHdrForMessageID: function(aMessageId) {
+    for (let item of this.collection.items) {
+      if (item.headerMessageID == aMessageId) {
+        let hdr = item.folderMessage;
+        if (hdr)
+          return hdr;
+      }
+    }
+    return null;
+  },
+
+  /**
+   * The default set of columns to show.
+   */
+  DEFAULT_COLUMN_STATES: {
+    threadCol: {
+      visible: true,
+    },
+    flaggedCol: {
+      visible: true,
+    },
+    subjectCol: {
+      visible: true,
+    },
+    correspondentCol: {
+      visible: Services.prefs.getBoolPref("mail.threadpane.use_correspondents"),
+    },
+    senderCol: {
+      visible: !Services.prefs.getBoolPref("mail.threadpane.use_correspondents"),
+    },
+    dateCol: {
+      visible: true,
+    },
+    locationCol: {
+      visible: true,
+    },
+  },
+
+  // --- settings persistence
+  getPersistedSetting: function(aSetting) {
+    try {
+      return JSON.parse(Services.prefs.getCharPref(
+        "mailnews.database.global.views." + this.viewType + "." + aSetting
+      ));
+    }
+    catch (e) {
+      return this.getDefaultSetting(aSetting);
+    }
+  },
+  setPersistedSetting: function(aSetting, aValue) {
+    Services.prefs.setCharPref(
+      "mailnews.database.global.views." + this.viewType + "." + aSetting,
+      JSON.stringify(aValue)
+    );
+  },
+  getDefaultSetting: function(aSetting) {
+    if (aSetting == "columns")
+      return this.DEFAULT_COLUMN_STATES;
+    else
+      return undefined;
+  },
+
+  // --- collection listener
+  onItemsAdded: function(aItems, aCollection) {
+    if (this.searchListener)
+      this.reportResults(aItems);
+  },
+  onItemsModified: function(aItems, aCollection) {
+  },
+  onItemsRemoved: function(aItems, aCollection) {
+  },
+  onQueryCompleted: function(aCollection) {
+    this.completed = true;
+    this.searchListener.onSearchDone(Cr.NS_OK);
+    if (this.completionCallback)
+      this.completionCallback();
+  },
+};

mailnews/db/gloda/modules/everybody.js

@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+var LOG = Log4Moz.repository.getLogger("gloda.everybody");
+
+var importNS = {};
+
+function loadModule(aModuleURI, aNSContrib) {
+  try {
+    LOG.info("... loading " + aModuleURI);
+    Cu.import(aModuleURI, importNS);
+  }
+  catch (ex) {
+    LOG.error("!!! error loading " + aModuleURI);
+    LOG.error("(" + ex.fileName + ":" + ex.lineNumber + ") " + ex);
+    return false;
+  }
+  LOG.info("+++ loaded " + aModuleURI);
+
+  if (aNSContrib) {
+    try {
+      importNS[aNSContrib].init();
+    }
+    catch (ex) {
+      LOG.error("!!! error initializing " + aModuleURI);
+      LOG.error("(" + ex.fileName + ":" + ex.lineNumber + ") " + ex);
+      return false;
+    }
+    LOG.info("+++ inited " + aModuleURI);
+  }
+  return true;
+}
+
+loadModule("resource:///modules/gloda/fundattr.js", "GlodaFundAttr");
+loadModule("resource:///modules/gloda/explattr.js", "GlodaExplicitAttr");
+
+loadModule("resource:///modules/gloda/noun_tag.js");
+loadModule("resource:///modules/gloda/noun_freetag.js");
+loadModule("resource:///modules/gloda/noun_mimetype.js");
+loadModule("resource:///modules/gloda/index_msg.js");
+loadModule("resource:///modules/gloda/index_ab.js", "GlodaABAttrs");

mailnews/db/gloda/modules/explattr.js

@@ -0,0 +1,191 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides the "explicit attribute" provider for messages.  It is
+ *  concerned with attributes that are the result of user actions.  For example,
+ *  whether a message is starred (flagged), message tags, whether it is
+ *  read/unread, etc.
+ */
+
+this.EXPORTED_SYMBOLS = ['GlodaExplicitAttr'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/noun_tag.js");
+Cu.import("resource:///modules/mailServices.js");
+
+
+var nsMsgMessageFlags_Replied = Ci.nsMsgMessageFlags.Replied;
+var nsMsgMessageFlags_Forwarded = Ci.nsMsgMessageFlags.Forwarded;
+
+var EXT_BUILTIN = "built-in";
+
+/**
+ * @namespace Explicit attribute provider.  Indexes/defines attributes that are
+ *  explicitly a result of user action.  This dubiously includes marking a
+ *  message as read.
+ */
+var GlodaExplicitAttr = {
+  providerName: "gloda.explattr",
+  strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+  _log: null,
+  _msgTagService: null,
+
+  init: function gloda_explattr_init() {
+    this._log =  Log4Moz.repository.getLogger("gloda.explattr");
+
+    this._msgTagService = MailServices.tags;
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  /** Boost for starred messages. */
+  NOTABILITY_STARRED: 16,
+  /** Boost for tagged messages, first tag. */
+  NOTABILITY_TAGGED_FIRST: 8,
+  /** Boost for tagged messages, each additional tag. */
+  NOTABILITY_TAGGED_ADDL: 1,
+
+  defineAttributes: function() {
+    // Tag
+    this._attrTag = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "tag",
+                        bindName: "tags",
+                        singular: false,
+                        emptySetIsSignificant: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_TAG,
+                        parameterNoun: null,
+                        // Property change notifications that we care about:
+                        propertyChanges: ["keywords"],
+                        }); // not-tested
+
+    // Star
+    this._attrStar = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "star",
+                        bindName: "starred",
+                        singular: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_BOOLEAN,
+                        parameterNoun: null,
+                        }); // tested-by: test_attributes_explicit
+    // Read/Unread
+    this._attrRead = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrExplicit,
+                        attributeName: "read",
+                        singular: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_BOOLEAN,
+                        parameterNoun: null,
+                        }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this message been replied to by the user.
+     */
+    this._attrRepliedTo = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "repliedTo",
+      singular: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+
+    /**
+     * Has this user forwarded this message to someone.
+     */
+    this._attrForwarded = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "forwarded",
+      singular: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_BOOLEAN,
+      parameterNoun: null,
+    }); // tested-by: test_attributes_explicit
+  },
+
+  process: function* Gloda_explattr_process(aGlodaMessage, aRawReps, aIsNew,
+                                            aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+
+    aGlodaMessage.starred = aMsgHdr.isFlagged;
+    if (aGlodaMessage.starred)
+      aGlodaMessage.notability += this.NOTABILITY_STARRED;
+
+    aGlodaMessage.read = aMsgHdr.isRead;
+
+    let flags = aMsgHdr.flags;
+    aGlodaMessage.repliedTo = Boolean(flags & nsMsgMessageFlags_Replied);
+    aGlodaMessage.forwarded = Boolean(flags & nsMsgMessageFlags_Forwarded);
+
+    let tags = aGlodaMessage.tags = [];
+
+    // -- Tag
+    // build a map of the keywords
+    let keywords = aMsgHdr.getStringProperty("keywords");
+    let keywordList = keywords.split(' ');
+    let keywordMap = {};
+    for (let iKeyword = 0; iKeyword < keywordList.length; iKeyword++) {
+      let keyword = keywordList[iKeyword];
+      keywordMap[keyword] = true;
+    }
+
+    let tagArray = TagNoun.getAllTags();
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      if (tag.key in keywordMap)
+        tags.push(tag);
+    }
+
+    if (tags.length)
+      aGlodaMessage.notability += this.NOTABILITY_TAGGED_FIRST +
+        (tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+
+    yield Gloda.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from process().  Arguably process should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score: function Gloda_explattr_score(aMessage, aContext) {
+    let score = 0;
+    if (aMessage.starred)
+      score += this.NOTABILITY_STARRED;
+    if (aMessage.tags.length)
+      score += this.NOTABILITY_TAGGED_FIRST +
+        (aMessage.tags.length - 1) * this.NOTABILITY_TAGGED_ADDL;
+    return score;
+  },
+};

mailnews/db/gloda/modules/facet.js

@@ -0,0 +1,582 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file provides faceting logic.
+ */
+
+var EXPORTED_SYMBOLS = ["FacetDriver", "FacetUtils"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/public.js");
+
+/**
+ * Decides the appropriate faceters for the noun type and drives the faceting
+ *  process.  This class and the faceters are intended to be reusable so that
+ *  you only need one instance per faceting session.  (Although each faceting
+ *  pass is accordingly destructive to previous results.)
+ *
+ * Our strategy for faceting is to process one attribute at a time across all
+ *  the items in the provided set.  The alternative would be to iterate over
+ *  the items and then iterate over the attributes on each item.  While both
+ *  approaches have caching downsides
+ */
+function FacetDriver(aNounDef, aWindow) {
+  this.nounDef = aNounDef;
+  this._window = aWindow;
+
+  this._makeFaceters();
+}
+FacetDriver.prototype = {
+  /**
+   * Populate |this.faceters| with a set of faceters appropriate to the noun
+   *  definition associated with this instance.
+   */
+  _makeFaceters: function() {
+    let faceters = this.faceters = [];
+
+    function makeFaceter(aAttrDef, aFacetDef) {
+      let facetType = aFacetDef.type;
+
+      if (aAttrDef.singular) {
+        if (facetType == "date")
+          faceters.push(new DateFaceter(aAttrDef, aFacetDef));
+        else
+          faceters.push(new DiscreteFaceter(aAttrDef, aFacetDef));
+      }
+      else {
+        if (facetType == "nonempty?")
+          faceters.push(new NonEmptySetFaceter(aAttrDef, aFacetDef));
+        else
+          faceters.push(new DiscreteSetFaceter(aAttrDef, aFacetDef));
+      }
+    }
+
+    for (let key in this.nounDef.attribsByBoundName) {
+      let attrDef = this.nounDef.attribsByBoundName[key];
+      // ignore attributes that do not want to be faceted
+      if (!attrDef.facet)
+        continue;
+
+      makeFaceter(attrDef, attrDef.facet);
+
+      if ("extraFacets" in attrDef) {
+        for (let facetDef of attrDef.extraFacets) {
+          makeFaceter(attrDef, facetDef);
+        }
+      }
+    }
+  },
+  /**
+   * Asynchronously facet the provided items, calling the provided callback when
+   *  completed.
+   */
+  go: function FacetDriver_go(aItems, aCallback, aCallbackThis) {
+    this.items = aItems;
+    this.callback = aCallback;
+    this.callbackThis = aCallbackThis;
+
+    this._nextFaceter = 0;
+    this._drive();
+  },
+
+  _MAX_FACETING_TIMESLICE_MS: 100,
+  _FACETING_YIELD_DURATION_MS: 0,
+  _driveWrapper: function(aThis) {
+    aThis._drive();
+  },
+  _drive: function() {
+    let start = Date.now();
+
+    while (this._nextFaceter < this.faceters.length) {
+      let faceter = this.faceters[this._nextFaceter++];
+      // for now we facet in one go, but the long-term plan allows for them to
+      //  be generators.
+      faceter.facetItems(this.items);
+
+      let delta = Date.now() - start;
+      if (delta > this._MAX_FACETING_TIMESLICE_MS) {
+        this._window.setTimeout(this._driveWrapper,
+                                this._FACETING_YIELD_DURATION_MS,
+                                this);
+        return;
+      }
+    }
+
+    // we only get here once we are done with the faceters
+    this.callback.call(this.callbackThis);
+  }
+};
+
+var FacetUtils = {
+  _groupSizeComparator: function(a, b) {
+    return b[1].length - a[1].length;
+  },
+
+  /**
+   * Given a list where each entry is a tuple of [group object, list of items
+   *  belonging to that group], produce a new list of the top grouped items.  We
+   *  used to also produce an "other" aggregation, but that turned out to be
+   *  conceptually difficult to deal with, so that's gone, leaving this method
+   *  with much less to do.
+   *
+   * @param aAttrDef The attribute for the facet we are working with.
+   * @param aGroups The list of groups built for the facet.
+   * @param aMaxCount The number of result rows you want back.
+   */
+  makeTopGroups: function FacetUtils_makeTopGroups(aAttrDef, aGroups,
+                                                   aMaxCount) {
+    let nounDef = aAttrDef.objectNounDef;
+    let realGroupsToUse = aMaxCount;
+
+    let orderedBySize = aGroups.concat();
+    orderedBySize.sort(this._groupSizeComparator);
+
+    // - get the real groups to use and order them by the attribute comparator
+    let outGroups = orderedBySize.slice(0, realGroupsToUse);
+    let comparator = nounDef.comparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    outGroups.sort(comparatorHelper);
+
+    return outGroups;
+  }
+};
+
+/**
+ * Facet discrete things like message authors, boolean values, etc.  Only
+ *  appropriate for use on singular values.  Use |DiscreteSetFaceter| for
+ *  non-singular values.
+ */
+function DiscreteFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive)
+      return this.facetPrimitiveItems(aItems);
+    else
+      return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+
+    let valStrToVal = {};
+    let groups = this.groups = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      if (val === Gloda.IGNORE_FACET)
+        continue;
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val))
+        continue;
+
+      // We need to use hasOwnProperty because we cannot guarantee that the
+      //  contents of val won't collide with the attributes in Object.prototype.
+      if (groups.hasOwnProperty(val))
+        groups[val].push(item);
+      else {
+        groups[val] = [item];
+        valStrToVal[val] = val;
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [valStrToVal[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = this.groups = {};
+    let groupMap = this.groupMap = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      if (val === Gloda.IGNORE_FACET)
+        continue;
+
+      // skip items the filter tells us to ignore
+      if (filter && !filter(val))
+        continue;
+
+      let valId = (val == null) ? null : val[idAttr];
+      // We need to use hasOwnProperty because tag nouns are complex objects
+      //  with id's that are non-numeric and so can collide with the contents
+      //  of Object.prototype.  (Note: the "tags" attribute is actually handled
+      //  by the DiscreteSetFaceter.)
+      if (groupMap.hasOwnProperty(valId)) {
+        groups[valId].push(item);
+      }
+      else {
+        groupMap[valId] = val;
+        groups[valId] = [item];
+        this.groupCount++;
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [groupMap[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Facet sets of discrete items.  For example, tags applied to messages.
+ *
+ * The main differences between us and |DiscreteFaceter| are:
+ * - The empty set is notable.
+ * - Specific set configurations could be interesting, but are not low-hanging
+ *    fruit.
+ */
+function DiscreteSetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DiscreteSetFaceter.prototype = {
+  type: "discrete",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    if (this.attrDef.objectNounDef.isPrimitive)
+      return this.facetPrimitiveItems(aItems);
+    else
+      return this.facetComplexItems(aItems);
+  },
+  /**
+   * Facet an attribute whose value is primitive, meaning that it is a raw
+   *  numeric value or string, rather than a complex object.
+   */
+  facetPrimitiveItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+
+    let groups = this.groups = {};
+    let valStrToVal = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals === Gloda.IGNORE_FACET)
+        continue;
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val))
+          continue;
+
+        // We need to use hasOwnProperty because we cannot guarantee that the
+        //  contents of val won't collide with the attributes in
+        //  Object.prototype.
+        if (groups.hasOwnProperty(val))
+          groups[val].push(item);
+        else {
+          groups[val] = [item];
+          valStrToVal[val] = val;
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [valStrToVal[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+  /**
+   * Facet an attribute whose value is a complex object that can be identified
+   *  by its 'id' attribute.  This is the case where the value is itself a noun
+   *  instance.
+   */
+  facetComplexItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+    let filter = this.facetDef.filter;
+    let idAttr = this.facetDef.groupIdAttr;
+
+    let groups = this.groups = {};
+    let groupMap = this.groupMap = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals === Gloda.IGNORE_FACET)
+        continue;
+
+      if (vals == null || vals.length == 0) {
+        vals = [null];
+      }
+      for (let val of vals) {
+        // skip items the filter tells us to ignore
+        if (filter && !filter(val))
+          continue;
+
+        let valId = (val == null) ? null : val[idAttr];
+        // We need to use hasOwnProperty because tag nouns are complex objects
+        //  with id's that are non-numeric and so can collide with the contents
+        //  of Object.prototype.
+        if (groupMap.hasOwnProperty(valId)) {
+          groups[valId].push(item);
+        }
+        else {
+          groupMap[valId] = val;
+          groups[valId] = [item];
+          this.groupCount++;
+        }
+      }
+    }
+
+    let orderedGroups = Object.keys(groups).
+      map(key => [groupMap[key], groups[key]]);
+    let comparator = this.facetDef.groupComparator;
+    function comparatorHelper(a, b) {
+      return comparator(a[0], b[0]);
+    }
+    orderedGroups.sort(comparatorHelper);
+    this.orderedGroups = orderedGroups;
+  },
+};
+
+/**
+ * Given a non-singular attribute, facet it as if it were a boolean based on
+ *  whether there is anything in the list (set).
+ */
+function NonEmptySetFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+NonEmptySetFaceter.prototype = {
+  type: "boolean",
+  /**
+   * Facet the given set of items, deferring to the appropriate helper method
+   */
+  facetItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+
+    let trueValues = [];
+    let falseValues = [];
+
+    let groups = this.groups = {};
+    this.groupCount = 0;
+
+    for (let item of aItems) {
+      let vals = (attrKey in item) ? item[attrKey] : null;
+      if (vals == null || vals.length == 0)
+        falseValues.push(item);
+      else
+        trueValues.push(item);
+    }
+
+    this.orderedGroups = [];
+    if (trueValues.length)
+      this.orderedGroups.push([true, trueValues]);
+    if (falseValues.length)
+      this.orderedGroups.push([false, falseValues]);
+    this.groupCount = this.orderedGroups.length;
+  },
+  makeQuery: function(aGroupValues, aInclusive) {
+    let query = this.query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+
+    let constraintFunc = query[this.attrDef.boundName];
+    constraintFunc.call(query);
+
+    // Our query is always for non-empty lists (at this time), so we want to
+    //  invert if they're excluding 'true' or including 'false', which means !=.
+    let invert = aGroupValues[0] != aInclusive;
+
+    return [query, invert];
+  }
+};
+
+
+/**
+ * Facet dates.  We build a hierarchical nested structure of year, month, and
+ *  day nesting levels.  This decision was made speculatively in the hopes that
+ *  it would allow us to do clustered analysis and that there might be a benefit
+ *  for that.  For example, if you search for "Christmas", we might notice
+ *  clusters of messages around December of each year.  We could then present
+ *  these in a list as likely candidates, rather than a graphical timeline.
+ *  Alternately, it could be used to inform a non-linear visualization.  As it
+ *  stands (as of this writing), it's just a complicating factor.
+ */
+function DateFaceter(aAttrDef, aFacetDef) {
+  this.attrDef = aAttrDef;
+  this.facetDef = aFacetDef;
+}
+DateFaceter.prototype = {
+  type: "date",
+  /**
+   *
+   */
+  facetItems: function(aItems) {
+    let attrKey = this.attrDef.boundName;
+    let nounDef = this.attrDef.objectNounDef;
+
+    let years = this.years = {_subCount: 0};
+    // generally track the time range
+    let oldest = null, newest = null;
+
+    let validItems = this.validItems = [];
+
+    // just cheat and put us at the front...
+    this.groupCount = aItems.length ? 1000 : 0;
+    this.orderedGroups = null;
+
+    /** The number of items with a null/missing attribute. */
+    this.missing = 0;
+
+    /**
+     * The number of items with a date that is unreasonably far in the past or
+     *  in the future.  Old-wise, we are concerned about incorrectly formatted
+     *  messages (spam) that end up placed around the UNIX epoch.  New-wise,
+     *  we are concerned about messages that can't be explained by users who
+     *  don't know how to set their clocks (both the current user and people
+     *  sending them mail), mainly meaning spam.
+     * We want to avoid having our clever time-scale logic being made useless by
+     *  these unreasonable messages.
+     */
+    this.unreasonable = 0;
+    // feb 1, 1970
+    let tooOld = new Date(1970, 1, 1);
+    // 3 days from now
+    let tooNew = new Date(Date.now() + 3 * 24 * 60 * 60 * 1000);
+
+    for (let item of aItems) {
+      let val = (attrKey in item) ? item[attrKey] : null;
+      // -- missing
+      if (val == null) {
+        this.missing++;
+        continue;
+      }
+
+      // -- unreasonable
+      if (val < tooOld || val > tooNew) {
+        this.unreasonable++;
+        continue;
+      }
+
+      this.validItems.push(item);
+
+      // -- time range
+      if (oldest == null)
+        oldest = newest = val;
+      else if (val < oldest)
+        oldest = val;
+      else if (val > newest)
+        newest = val;
+
+      // -- bucket
+      // - year
+      let year, valYear = val.getYear();
+      if (valYear in years) {
+        year = years[valYear];
+        year._dateCount++;
+      }
+      else {
+        year = years[valYear] = {
+          _dateCount: 1,
+          _subCount: 0
+        };
+        years._subCount++;
+      }
+
+      // - month
+      let month, valMonth = val.getMonth();
+      if (valMonth in year) {
+        month = year[valMonth];
+        month._dateCount++;
+      }
+      else {
+        month = year[valMonth] = {
+          _dateCount: 1,
+          _subCount: 0
+        };
+        year._subCount++;
+      }
+
+      // - day
+      let valDate = val.getDate();
+      if (valDate in month) {
+        month[valDate].push(item);
+      }
+      else {
+        month[valDate] = [item];
+      }
+    }
+
+    this.oldest = oldest;
+    this.newest = newest;
+  },
+
+  _unionMonth: function(aMonthObj) {
+    let dayItemLists = [];
+    for (let key in aMonthObj) {
+      let dayItemList = aMonthObj[key];
+      if (typeof(key) == "string" && key.startsWith('_'))
+        continue;
+      dayItemLists.push(dayItemList);
+    }
+    return Array.concat.apply([], dayItemLists);
+  },
+
+  _unionYear: function(aYearObj) {
+    let monthItemLists = [];
+    for (let key in aYearObj) {
+      let monthObj = aYearObj[key];
+      if (typeof(key) == "string" && key.startsWith('_'))
+        continue;
+      monthItemLists.push(this._unionMonth(monthObj));
+    }
+    return Array.concat.apply([], monthItemLists);
+  }
+};

mailnews/db/gloda/modules/fundattr.js

@@ -0,0 +1,907 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaFundAttr'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/datastore.js");
+Cu.import("resource:///modules/gloda/datamodel.js"); // for GlodaAttachment
+
+Cu.import("resource:///modules/gloda/noun_mimetype.js");
+Cu.import("resource:///modules/gloda/connotent.js");
+
+/**
+ * @namespace The Gloda Fundamental Attribute provider is a special attribute
+ *  provider; it provides attributes that the rest of the providers should be
+ *  able to assume exist.  Also, it may end up accessing things at a lower level
+ *  than most extension providers should do.  In summary, don't mimic this code
+ *  unless you won't complain when your code breaks.
+ */
+var GlodaFundAttr = {
+  providerName: "gloda.fundattr",
+  strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+  _log: null,
+
+  init: function gloda_explattr_init() {
+    this._log =  Log4Moz.repository.getLogger("gloda.fundattr");
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  POPULARITY_FROM_ME_TO: 10,
+  POPULARITY_FROM_ME_CC: 4,
+  POPULARITY_FROM_ME_BCC: 3,
+  POPULARITY_TO_ME: 5,
+  POPULARITY_CC_ME: 1,
+  POPULARITY_BCC_ME: 1,
+
+  /** Boost for messages 'I' sent */
+  NOTABILITY_FROM_ME: 10,
+  /** Boost for messages involving 'me'. */
+  NOTABILITY_INVOLVING_ME: 1,
+  /** Boost for message from someone in 'my' address book. */
+  NOTABILITY_FROM_IN_ADDR_BOOK: 10,
+  /** Boost for the first person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_FIRST: 8,
+  /** Boost for each additional person involved in my address book. */
+  NOTABILITY_INVOLVING_ADDR_BOOK_ADDL: 2,
+
+  defineAttributes: function gloda_fundattr_defineAttributes() {
+    /* ***** Conversations ***** */
+    // conversation: subjectMatches
+    this._attrConvSubject = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [Gloda.NOUN_CONVERSATION],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    /* ***** Messages ***** */
+    // folder
+    this._attrFolder = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "folder",
+      singular: true,
+      facet: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "folderID",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FOLDER,
+      }); // tested-by: test_attributes_fundamental
+    this._attrAccount = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "account",
+      canQuery: "memory",
+      singular: true,
+      facet: true,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_ACCOUNT
+      });
+    this._attrMessageKey = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "messageKey",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "messageKey",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+
+    // We need to surface the deleted attribute for querying, but there is no
+    //  reason for user code, so let's call it "_deleted" rather than deleted.
+    // (In fact, our validity constraints require a special query formulation
+    //  that user code should have no clue exists.  That's right user code,
+    //  that's a dare.)
+    Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "_deleted",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "deleted",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      });
+
+
+    // -- fulltext search helpers
+    // fulltextMatches.  Match over message subject, body, and attachments
+    // @testpoint gloda.noun.message.attr.fulltextMatches
+    this._attrFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "fulltextMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "messagesText",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // subjectMatches.  Fulltext match on subject
+    // @testpoint gloda.noun.message.attr.subjectMatches
+    this._attrSubjectText = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "subjectMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "subject",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // bodyMatches. super-synthetic full-text matching...
+    // @testpoint gloda.noun.message.attr.bodyMatches
+    this._attrBody = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "bodyMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "body",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // attachmentNamesMatch
+    // @testpoint gloda.noun.message.attr.attachmentNamesMatch
+    this._attrAttachmentNames = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "attachmentNamesMatch",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "attachmentNames",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // @testpoint gloda.noun.message.attr.authorMatches
+    this._attrAuthorFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "authorMatches",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "author",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // @testpoint gloda.noun.message.attr.recipientsMatch
+    this._attrRecipientsFulltext = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "recipientsMatch",
+      singular: true,
+      special: Gloda.kSpecialFulltext,
+      specialColumnName: "recipients",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_FULLTEXT,
+      });
+
+    // --- synthetic stuff for some reason
+    // conversation
+    // @testpoint gloda.noun.message.attr.conversation
+    this._attrConversation = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "conversation",
+      singular: true,
+      special: Gloda.kSpecialColumnParent,
+      specialColumnName: "conversationID",
+      idStorageAttributeName: "_conversationID",
+      valueStorageAttributeName: "_conversation",
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_CONVERSATION,
+      canQuery: true,
+      });
+
+    // --- Fundamental
+    // From
+    this._attrFrom = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "from",
+                        singular: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // tested-by: test_attributes_fundamental
+    // To
+    this._attrTo = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "to",
+                        singular: false,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // tested-by: test_attributes_fundamental
+    // Cc
+    this._attrCc = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "cc",
+                        singular: false,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // not-tested
+    /**
+     * Bcc'ed recipients; only makes sense for sent messages.
+     */
+    this._attrBcc = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "bcc",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+    }); // not-tested
+
+    // Date.  now lives on the row.
+    this._attrDate = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "date",
+                        singular: true,
+                        facet: {
+                          type: "date",
+                        },
+                        special: Gloda.kSpecialColumn,
+                        specialColumnName: "date",
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_DATE,
+                        }); // tested-by: test_attributes_fundamental
+
+    // Header message ID.
+    this._attrHeaderMessageID = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "headerMessageID",
+                        singular: true,
+                        special: Gloda.kSpecialString,
+                        specialColumnName: "headerMessageID",
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_STRING,
+                        canQuery: true,
+                        }); // tested-by: test_attributes_fundamental
+
+    // Attachment MIME Types
+    this._attrAttachmentTypes = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "attachmentTypes",
+      singular: false,
+      emptySetIsSignificant: true,
+      facet: {
+        type: "default",
+        // This will group the MIME types by their category.
+        groupIdAttr: "category",
+        queryHelper: "Category",
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_MIME_TYPE,
+      });
+
+    // Attachment infos
+    this._attrIsEncrypted = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "isEncrypted",
+      singular: true,
+      emptySetIsSignificant: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_NUMBER,
+      });
+
+    // Attachment infos
+    this._attrAttachmentInfos = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "attachmentInfos",
+      singular: false,
+      emptySetIsSignificant: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_ATTACHMENT,
+      });
+
+    // --- Optimization
+    /**
+     * Involves means any of from/to/cc/bcc.  The queries get ugly enough
+     *  without this that it seems to justify the cost, especially given the
+     *  frequent use case.  (In fact, post-filtering for the specific from/to/cc
+     *  is probably justifiable rather than losing this attribute...)
+     */
+    this._attrInvolves = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "involves",
+      singular: false,
+      facet: {
+        type: "default",
+        /**
+         * Filter out 'me', as we have other facets that deal with that, and the
+         *  'me' identities are so likely that they distort things.
+         *
+         * @return true if the identity is not one of my identities, false if it
+         *   is.
+         */
+        filter: function gloda_explattr_involves_filter(aItem) {
+          return (!(aItem.id in Gloda.myIdentities));
+        }
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // not-tested
+
+    /**
+     * Any of to/cc/bcc.
+     */
+    this._attrRecipients = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "recipients",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // not-tested
+
+    // From Me (To/Cc/Bcc)
+    this._attrFromMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrOptimization,
+      attributeName: "fromMe",
+      singular: false,
+      // The interesting thing to a facet is whether the message is from me.
+      facet: {
+        type: "nonempty?"
+      },
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_PARAM_IDENTITY,
+      }); // not-tested
+    // To/Cc/Bcc Me
+    this._attrToMe = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "toMe",
+      // The interesting thing to a facet is whether the message is to me.
+      facet: {
+        type: "nonempty?"
+      },
+      singular: false,
+      subjectNouns: [Gloda.NOUN_MESSAGE],
+      objectNoun: Gloda.NOUN_PARAM_IDENTITY,
+      }); // not-tested
+
+
+    // -- Mailing List
+    // Non-singular, but a hard call.  Namely, it is obvious that a message can
+    //  be addressed to multiple mailing lists.  However, I don't see how you
+    //  could receive a message with more than one set of List-* headers,
+    //  since each list-serve would each send you a copy.  Based on our current
+    //  decision to treat each physical message as separate, it almost seems
+    //  right to limit the list attribute to the copy that originated at the
+    //  list.  That may sound entirely wrong, but keep in mind that until we
+    //  have seen a message from the list with the List headers, we can't
+    //  definitely know it's a mailing list (although heuristics could take us
+    //  pretty far).  As such, the quasi-singular thing is appealing.
+    // Of course, the reality is that we really want to know if a message was
+    //  sent to multiple mailing lists and be able to query on that.
+    //  Additionally, our implicit-to logic needs to work on messages that
+    //  weren't relayed by the list-serve, especially messages sent to the list
+    //  by the user.
+    this._attrList = Gloda.defineAttribute({
+                        provider: this,
+                        extensionName: Gloda.BUILT_IN,
+                        attributeType: Gloda.kAttrFundamental,
+                        attributeName: "mailing-list",
+                        bindName: "mailingLists",
+                        singular: false,
+                        emptySetIsSignificant: true,
+                        facet: true,
+                        subjectNouns: [Gloda.NOUN_MESSAGE],
+                        objectNoun: Gloda.NOUN_IDENTITY,
+                        }); // not-tested, not-implemented
+  },
+
+  RE_LIST_POST: /<mailto:([^>]+)>/,
+
+  /**
+   *
+   * Specializations:
+   * - Mailing Lists.  Replies to a message on a mailing list frequently only
+   *   have the list-serve as the 'to', so we try to generate a synthetic 'to'
+   *   based on the author of the parent message when possible.  (The 'possible'
+   *   part is that we may not have a copy of the parent message at the time of
+   *   processing.)
+   * - Newsgroups.  Same deal as mailing lists.
+   */
+  process: function* gloda_fundattr_process(aGlodaMessage, aRawReps,
+                                            aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+    let aMimeMsg = aRawReps.mime;
+
+    // -- From
+    // Let's use replyTo if available.
+    // er, since we are just dealing with mailing lists for now, forget the
+    //  reply-to...
+    // TODO: deal with default charset issues
+    let author = null;
+    /*
+    try {
+      author = aMsgHdr.getStringProperty("replyTo");
+    }
+    catch (ex) {
+    }
+    */
+    if (author == null || author == "")
+      author = aMsgHdr.author;
+
+    let normalizedListPost = "";
+    if (aMimeMsg && aMimeMsg.has("list-post")) {
+      let match = this.RE_LIST_POST.exec(aMimeMsg.get("list-post"));
+      if (match)
+        normalizedListPost = "<" + match[1] + ">";
+    }
+
+    // Do not use the MIME decoded variants of any of the email addresses
+    //  because if name is encoded and has a comma in it, it will break the
+    //  address parser (which already knows how to do the decoding anyways).
+    let [authorIdentities, toIdentities, ccIdentities, bccIdentities,
+         listIdentities] =
+      yield aCallbackHandle.pushAndGo(
+        Gloda.getOrCreateMailIdentities(aCallbackHandle,
+                                        author, aMsgHdr.recipients,
+                                        aMsgHdr.ccList, aMsgHdr.bccList,
+                                        normalizedListPost));
+
+    if (authorIdentities.length != 1) {
+      throw new Gloda.BadItemContentsError(
+        "Message with subject '" + aMsgHdr.mime2DecodedSubject +
+          "' somehow lacks a valid author.  Bailing.");
+    }
+    let authorIdentity = authorIdentities[0];
+    aGlodaMessage.from = authorIdentity;
+
+    // -- To, Cc, Bcc
+    aGlodaMessage.to = toIdentities;
+    aGlodaMessage.cc = ccIdentities;
+    aGlodaMessage.bcc = bccIdentities;
+
+    // -- Mailing List
+    if (listIdentities.length)
+      aGlodaMessage.mailingLists = listIdentities;
+
+    let findIsEncrypted = x =>
+      x.isEncrypted || (x.parts ? x.parts.some(findIsEncrypted) : false);
+
+    // -- Encryption
+    aGlodaMessage.isEncrypted = false;
+    if (aMimeMsg) {
+      aGlodaMessage.isEncrypted = findIsEncrypted(aMimeMsg);
+    }
+
+    // -- Attachments
+    if (aMimeMsg) {
+      // nsParseMailbox.cpp puts the attachment flag on msgHdrs as soon as it
+      // finds a multipart/mixed part. This is a good heuristic, but if it turns
+      // out the part has no filename, then we don't treat it as an attachment.
+      // We just streamed the message, and we have all the information to figure
+      // that out, so now is a good place to clear the flag if needed.
+      let foundRealAttachment = false;
+      let attachmentTypes = [];
+      for (let attachment of aMimeMsg.allAttachments) {
+        // We don't care about would-be attachments that are not user-intended
+        //  attachments but rather artifacts of the message content.
+        // We also want to avoid dealing with obviously bogus mime types.
+        //  (If you don't have a "/", you are probably bogus.)
+        if (attachment.isRealAttachment &&
+            attachment.contentType.includes("/")) {
+          attachmentTypes.push(MimeTypeNoun.getMimeType(attachment.contentType));
+        }
+        if (attachment.isRealAttachment)
+          foundRealAttachment = true;
+      }
+      if (attachmentTypes.length) {
+        aGlodaMessage.attachmentTypes = attachmentTypes;
+      }
+
+      let aMsgHdr = aRawReps.header;
+      let wasStreamed = aMsgHdr &&
+        !aGlodaMessage.isEncrypted &&
+        ((aMsgHdr.flags & Ci.nsMsgMessageFlags.Offline) ||
+        (aMsgHdr.folder instanceof Ci.nsIMsgLocalMailFolder));
+
+      // Clear the flag if it turns out there's no attachment after all and we
+      // streamed completely the message (if we didn't, then we have no
+      // knowledge of attachments, unless bug 673370 is fixed).
+      if (!foundRealAttachment && wasStreamed)
+        aMsgHdr.markHasAttachments(false);
+
+      // This is not the same kind of attachments as above. Now, we want to
+      // provide convenience attributes to Gloda consumers, so that they can run
+      // through the list of attachments of a given message, to possibly build a
+      // visualization on top of it. We still reject bogus mime types, which
+      // means yencode won't be supported. Oh, I feel really bad.
+      let attachmentInfos = [];
+      for (let att of aMimeMsg.allUserAttachments) {
+        attachmentInfos.push(this.glodaAttFromMimeAtt(aRawReps.trueGlodaRep,
+                                                      att));
+      }
+      aGlodaMessage.attachmentInfos = attachmentInfos;
+    }
+
+    // TODO: deal with mailing lists, including implicit-to.  this will require
+    //  convincing the indexer to pass us in the previous message if it is
+    //  available.  (which we'll simply pass to everyone... it can help body
+    //  logic for quoting purposes, etc. too.)
+
+    yield Gloda.kWorkDone;
+  },
+
+  glodaAttFromMimeAtt:
+      function gloda_fundattr_glodaAttFromMimeAtt(aGlodaMessage, aAtt) {
+    // So we don't want to store the URL because it can change over time if
+    // the message is moved. What we do is store the full URL if it's a
+    // detached attachment, otherwise just keep the part information, and
+    // rebuild the URL according to where the message is sitting.
+    let part, externalUrl;
+    if (aAtt.isExternal) {
+      externalUrl = aAtt.url;
+    } else {
+      let matches = aAtt.url.match(GlodaUtils.PART_RE);
+      if (matches && matches.length)
+        part = matches[1];
+      else
+        this._log.error("Error processing attachment: " + aAtt.url);
+    }
+    return new GlodaAttachment(aGlodaMessage,
+                               aAtt.name,
+                               aAtt.contentType,
+                               aAtt.size,
+                               part,
+                               externalUrl,
+                               aAtt.isExternal);
+  },
+
+  optimize: function* gloda_fundattr_optimize(aGlodaMessage, aRawReps,
+                                              aIsNew, aCallbackHandle) {
+
+    let aMsgHdr = aRawReps.header;
+
+    // for simplicity this is used for both involves and recipients
+    let involvesIdentities = {};
+    let involves = aGlodaMessage.involves || [];
+    let recipients = aGlodaMessage.recipients || [];
+
+    // 'me' specialization optimizations
+    let toMe = aGlodaMessage.toMe || [];
+    let fromMe = aGlodaMessage.fromMe || [];
+
+    let myIdentities = Gloda.myIdentities; // needless optimization?
+    let authorIdentity = aGlodaMessage.from;
+    let isFromMe = authorIdentity.id in myIdentities;
+
+    // The fulltext search column for the author.  We want to have in here:
+    // - The e-mail address and display name as enclosed on the message.
+    // - The name per the address book card for this e-mail address, if we have
+    //   one.
+    aGlodaMessage._indexAuthor = aMsgHdr.mime2DecodedAuthor;
+    // The fulltext search column for the recipients. (same deal)
+    aGlodaMessage._indexRecipients = aMsgHdr.mime2DecodedRecipients;
+
+    if (isFromMe)
+      aGlodaMessage.notability += this.NOTABILITY_FROM_ME;
+    else {
+      let authorCard = authorIdentity.abCard;
+      if (authorCard) {
+        aGlodaMessage.notability += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+        // @testpoint gloda.noun.message.attr.authorMatches
+        aGlodaMessage._indexAuthor += ' ' + authorCard.displayName;
+      }
+    }
+
+    involves.push(authorIdentity);
+    involvesIdentities[authorIdentity.id] = true;
+
+    let involvedAddrBookCount = 0;
+
+    for (let toIdentity of aGlodaMessage.to) {
+      if (!(toIdentity.id in involvesIdentities)) {
+        involves.push(toIdentity);
+        recipients.push(toIdentity);
+        involvesIdentities[toIdentity.id] = true;
+        let toCard = toIdentity.abCard;
+        if (toCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + toCard.displayName;
+        }
+      }
+
+      // optimization attribute to-me ('I' am the parameter)
+      if (toIdentity.id in myIdentities) {
+        toMe.push([toIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_TO_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, toIdentity]);
+        // also, popularity
+        if (aIsNew)
+          toIdentity.contact.popularity += this.POPULARITY_FROM_ME_TO;
+      }
+    }
+    for (let ccIdentity of aGlodaMessage.cc) {
+      if (!(ccIdentity.id in involvesIdentities)) {
+        involves.push(ccIdentity);
+        recipients.push(ccIdentity);
+        involvesIdentities[ccIdentity.id] = true;
+        let ccCard = ccIdentity.abCard;
+        if (ccCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + ccCard.displayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (ccIdentity.id in myIdentities) {
+        toMe.push([ccIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_CC_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, ccIdentity]);
+        // also, popularity
+        if (aIsNew)
+          ccIdentity.contact.popularity += this.POPULARITY_FROM_ME_CC;
+      }
+    }
+    // just treat bcc like cc; the intent is the same although the exact
+    //  semantics differ.
+    for (let bccIdentity of aGlodaMessage.bcc) {
+      if (!(bccIdentity.id in involvesIdentities)) {
+        involves.push(bccIdentity);
+        recipients.push(bccIdentity);
+        involvesIdentities[bccIdentity.id] = true;
+        let bccCard = bccIdentity.abCard;
+        if (bccCard) {
+          involvedAddrBookCount++;
+          // @testpoint gloda.noun.message.attr.recipientsMatch
+          aGlodaMessage._indexRecipients += ' ' + bccCard.displayName;
+        }
+      }
+      // optimization attribute cc-me ('I' am the parameter)
+      if (bccIdentity.id in myIdentities) {
+        toMe.push([bccIdentity, authorIdentity]);
+        if (aIsNew)
+          authorIdentity.contact.popularity += this.POPULARITY_BCC_ME;
+      }
+      // optimization attribute from-me-to ('I' am the parameter)
+      if (isFromMe) {
+        fromMe.push([authorIdentity, bccIdentity]);
+        // also, popularity
+        if (aIsNew)
+          bccIdentity.contact.popularity += this.POPULARITY_FROM_ME_BCC;
+      }
+    }
+
+    if (involvedAddrBookCount)
+      aGlodaMessage.notability += this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+
+    aGlodaMessage.involves = involves;
+    aGlodaMessage.recipients = recipients;
+    if (toMe.length) {
+      aGlodaMessage.toMe = toMe;
+      aGlodaMessage.notability += this.NOTABILITY_INVOLVING_ME;
+    }
+    if (fromMe.length)
+      aGlodaMessage.fromMe = fromMe;
+
+    // Content
+    if (aRawReps.bodyLines) {
+      aGlodaMessage._content = aRawReps.content = new GlodaContent();
+      if (this.contentWhittle({}, aRawReps.bodyLines, aGlodaMessage._content)) {
+        // we were going to do something here?
+      }
+    }
+    else {
+      aRawReps.content = null;
+    }
+
+    yield Gloda.kWorkDone;
+  },
+
+  /**
+   * Duplicates the notability logic from optimize().  Arguably optimize should
+   *  be factored to call us, grokNounItem should be factored to call us, or we
+   *  should get sufficiently fancy that our code wildly diverges.
+   */
+  score: function gloda_fundattr_score(aMessage, aContext) {
+    let score = 0;
+
+    let authorIdentity = aMessage.from;
+    if (authorIdentity.id in Gloda.myIdentities)
+      score += this.NOTABILITY_FROM_ME;
+    else if (authorIdentity.inAddressBook)
+      score += this.NOTABILITY_FROM_IN_ADDR_BOOK;
+    if (aMessage.toMe)
+      score += this.NOTABILITY_INVOLVING_ME;
+
+    let involvedAddrBookCount = 0;
+    for (let [, identity] in Iterator(aMessage.to))
+      if (identity.inAddressBook)
+        involvedAddrBookCount++;
+    for (let [, identity] in Iterator(aMessage.cc))
+      if (identity.inAddressBook)
+        involvedAddrBookCount++;
+    if (involvedAddrBookCount)
+      score += this.NOTABILITY_INVOLVING_ADDR_BOOK_FIRST +
+        (involvedAddrBookCount - 1) * this.NOTABILITY_INVOLVING_ADDR_BOOK_ADDL;
+    return score;
+  },
+
+  _countQuoteDepthAndNormalize:
+    function gloda_fundattr__countQuoteDepthAndNormalize(aLine) {
+    let count = 0;
+    let lastStartOffset = 0;
+
+    for (let i = 0; i < aLine.length; i++) {
+      let c = aLine[i];
+      if (c == ">") {
+        count++;
+        lastStartOffset = i+1;
+      }
+      else if (c == " ") {
+      }
+      else {
+        return [count,
+                lastStartOffset ? aLine.substring(lastStartOffset) : aLine];
+      }
+    }
+
+    return [count, lastStartOffset ? aLine.substring(lastStartOffset) : aLine];
+  },
+
+  /**
+   * Attempt to understand simple quoting constructs that use ">" with
+   * obvious phrases to enter the quoting block.  No support for other types
+   * of quoting at this time.  Also no support for piercing the wrapper of
+   * forwarded messages to actually be the content of the forwarded message.
+   */
+  contentWhittle: function gloda_fundattr_contentWhittle(aMeta,
+      aBodyLines, aContent) {
+    if (!aContent.volunteerContent(aContent.kPriorityBase))
+      return false;
+
+    // duplicate the list; we mutate somewhat...
+    let bodyLines = aBodyLines.concat();
+
+    // lastNonBlankLine originally was just for detecting quoting idioms where
+    //  the "wrote" line was separated from the quoted block by a blank line.
+    // Now we also use it for whitespace suppression at the boundaries of
+    //  quoted and un-quoted text.  (We keep blank lines within the same
+    //  'block' of quoted or non-quoted text.)
+    // Because we now have two goals for it, and we still want to suppress blank
+    //  lines when there is a 'wrote' line involved, we introduce...
+    //  prevLastNonBlankLine!  This arguably suggests refactoring should be the
+    //  next step, but things work for now.
+    let rangeStart = 0, lastNonBlankLine = null, prevLastNonBlankLine = null;
+    let inQuoteDepth = 0;
+    for (let [iLine, line] of bodyLines.entries()) {
+      if (!line || (line == "\xa0")) /* unicode non breaking space */
+        continue;
+
+      if (line.startsWith(">")) {
+        if (!inQuoteDepth) {
+          let rangeEnd = iLine - 1;
+          let quoteRangeStart = iLine;
+          // see if the last non-blank-line was a lead-in...
+          if (lastNonBlankLine != null) {
+            // TODO: localize quote range start detection
+            if (aBodyLines[lastNonBlankLine].includes("wrote")) {
+              quoteRangeStart = lastNonBlankLine;
+              rangeEnd = lastNonBlankLine - 1;
+              // we 'used up' lastNonBlankLine, let's promote the prev guy to
+              //  be the new lastNonBlankLine for the next logic block
+              lastNonBlankLine = prevLastNonBlankLine;
+            }
+            // eat the trailing whitespace...
+            if (lastNonBlankLine != null)
+              rangeEnd = Math.min(rangeEnd, lastNonBlankLine);
+          }
+          if (rangeEnd >= rangeStart)
+            aContent.content(aBodyLines.slice(rangeStart, rangeEnd+1));
+
+          [inQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+          rangeStart = quoteRangeStart;
+        }
+        else {
+          let curQuoteDepth;
+          [curQuoteDepth, line] = this._countQuoteDepthAndNormalize(line);
+          bodyLines[iLine] = line;
+
+          if (curQuoteDepth != inQuoteDepth) {
+            // we could do some "wrote" compensation here, but it's not really
+            //  as important.  let's wait for a more clever algorithm.
+            aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+            inQuoteDepth = curQuoteDepth;
+            rangeStart = iLine;
+          }
+        }
+      }
+      else {
+        if (inQuoteDepth) {
+          aContent.quoted(aBodyLines.slice(rangeStart, iLine), inQuoteDepth);
+          inQuoteDepth = 0;
+          rangeStart = iLine;
+        }
+      }
+
+      prevLastNonBlankLine = lastNonBlankLine;
+      lastNonBlankLine = iLine;
+    }
+
+    if (inQuoteDepth) {
+      aContent.quoted(aBodyLines.slice(rangeStart), inQuoteDepth);
+    }
+    else {
+      aContent.content(aBodyLines.slice(rangeStart, lastNonBlankLine+1));
+    }
+
+    return true;
+  },
+};

mailnews/db/gloda/modules/index_ab.js

@@ -0,0 +1,287 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaABIndexer', 'GlodaABAttrs'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/collection.js");
+Cu.import("resource:///modules/gloda/datastore.js");
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/indexer.js");
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/gloda/noun_freetag.js");
+Cu.import("resource:///modules/gloda/utils.js");
+Cu.import("resource:///modules/mailServices.js");
+
+
+var GlodaABIndexer = {
+  _log: null,
+
+  name: "index_ab",
+  enable: function() {
+    if (this._log == null)
+      this._log =  Log4Moz.repository.getLogger("gloda.index_ab");
+
+    MailServices.ab.addAddressBookListener(this,
+                                           Ci.nsIAbListener.itemAdded |
+                                           Ci.nsIAbListener.itemChanged |
+                                           Ci.nsIAbListener.directoryItemRemoved);
+  },
+
+  disable: function() {
+    MailServices.ab.removeAddressBookListener(this);
+  },
+
+  // it's a getter so we can reference 'this'
+  get workers() {
+    return [
+      ["ab-card", {
+         worker: this._worker_index_card,
+       }],
+    ];
+  },
+
+  _worker_index_card: function*(aJob, aCallbackHandle) {
+    let card = aJob.id;
+
+    if (card.primaryEmail) {
+      // load the identity
+      let query = Gloda.newQuery(Gloda.NOUN_IDENTITY);
+      query.kind("email");
+      // we currently normalize all e-mail addresses to be lowercase
+      query.value(card.primaryEmail.toLowerCase());
+      let identityCollection = query.getCollection(aCallbackHandle);
+      yield Gloda.kWorkAsync;
+
+      if (identityCollection.items.length) {
+        let identity = identityCollection.items[0];
+        // force the identity to know it has an associated ab card.
+        identity._hasAddressBookCard = true;
+
+        this._log.debug("Found identity, processing card.");
+        yield aCallbackHandle.pushAndGo(
+            Gloda.grokNounItem(identity.contact, {card: card}, false, false,
+                               aCallbackHandle));
+        this._log.debug("Done processing card.");
+      }
+    }
+
+    yield GlodaIndexer.kWorkDone;
+  },
+
+  initialSweep: function() {
+  },
+
+  /* ------ nsIAbListener ------ */
+  /**
+   * When an address book card is added, update the cached GlodaIdentity
+   *  object's cached idea of whether the identity has an ab card.
+   */
+  onItemAdded: function ab_indexer_onItemAdded(aParentDir, aItem) {
+    if (!(aItem instanceof Ci.nsIAbCard))
+      return;
+
+    this._log.debug("Received Card Add Notification");
+    let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+      Gloda.NOUN_IDENTITY, "email@" + aItem.primaryEmail.toLowerCase());
+    if (identity)
+      identity._hasAddressBookCard = true;
+  },
+  /**
+   * When an address book card is added, update the cached GlodaIdentity
+   *  object's cached idea of whether the identity has an ab card.
+   */
+  onItemRemoved: function ab_indexer_onItemRemoved(aParentDir, aItem) {
+    if (!(aItem instanceof Ci.nsIAbCard))
+      return;
+
+    this._log.debug("Received Card Removal Notification");
+    let identity = GlodaCollectionManager.cacheLookupOneByUniqueValue(
+      Gloda.NOUN_IDENTITY, "email@" + aItem.primaryEmail.toLowerCase());
+    if (identity)
+      identity._hasAddressBookCard = false;
+
+  },
+  onItemPropertyChanged: function ab_indexer_onItemPropertyChanged(aItem,
+      aProperty, aOldValue, aNewValue) {
+    if (aProperty == null && aItem instanceof Ci.nsIAbCard) {
+      this._log.debug("Received Card Change Notification");
+
+      let card = aItem; // instanceof already QueryInterface'd for us.
+      let job = new IndexingJob("ab-card", card);
+      GlodaIndexer.indexJob(job);
+    }
+  }
+};
+GlodaIndexer.registerIndexer(GlodaABIndexer);
+
+var GlodaABAttrs = {
+  providerName: "gloda.ab_attr",
+  _log: null,
+
+  init: function() {
+    this._log =  Log4Moz.repository.getLogger("gloda.abattrs");
+
+    try {
+      this.defineAttributes();
+    }
+    catch (ex) {
+      this._log.error("Error in init: " + ex);
+      throw ex;
+    }
+  },
+
+  defineAttributes: function() {
+    /* ***** Contacts ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "identities",
+      singular: false,
+      special: Gloda.kSpecialColumnChildren,
+      //specialColumnName: "contactID",
+      storageAttributeName: "_identities",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_IDENTITY,
+      }); // tested-by: test_attributes_fundamental
+    this._attrContactName = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "name",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "name",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrContactPopularity = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "popularity",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "popularity",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // not-tested
+    this._attrContactFrecency = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "frecency",
+      singular: true,
+      special: Gloda.kSpecialColumn,
+      specialColumnName: "frecency",
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.NOUN_NUMBER,
+      canQuery: true,
+      }); // not-tested
+
+    /* ***** Identities ***** */
+    this._attrIdentityContact = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrDerived,
+      attributeName: "contact",
+      singular: true,
+      special: Gloda.kSpecialColumnParent,
+      specialColumnName: "contactID", // the column in the db
+      idStorageAttributeName: "_contactID",
+      valueStorageAttributeName: "_contact",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_CONTACT,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrIdentityKind = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "kind",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "kind",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+    this._attrIdentityValue = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrFundamental,
+      attributeName: "value",
+      singular: true,
+      special: Gloda.kSpecialString,
+      specialColumnName: "value",
+      subjectNouns: [Gloda.NOUN_IDENTITY],
+      objectNoun: Gloda.NOUN_STRING,
+      canQuery: true,
+      }); // tested-by: test_attributes_fundamental
+
+    /* ***** Contact Meta ***** */
+    // Freeform tags; not explicit like thunderbird's fundamental tags.
+    //  we differentiate for now because of fundamental implementation
+    //  differences.
+    this._attrFreeTag = Gloda.defineAttribute({
+      provider: this,
+      extensionName: Gloda.BUILT_IN,
+      attributeType: Gloda.kAttrExplicit,
+      attributeName: "freetag",
+      bind: true,
+      bindName: "freeTags",
+      singular: false,
+      subjectNouns: [Gloda.NOUN_CONTACT],
+      objectNoun: Gloda.lookupNoun("freetag"),
+      parameterNoun: null,
+      canQuery: true,
+      }); // not-tested
+    // we need to find any existing bound freetag attributes, and use them to
+    //  populate to FreeTagNoun's understanding
+    if ("parameterBindings" in this._attrFreeTag) {
+      for (let freeTagName in this._attrFreeTag.parameterBindings) {
+        this._log.debug("Telling FreeTagNoun about: " + freeTagName);
+        FreeTagNoun.getFreeTag(freeTagName);
+      }
+    }
+  },
+
+  process: function*(aContact, aRawReps, aIsNew, aCallbackHandle) {
+    let card = aRawReps.card;
+    if (aContact.NOUN_ID != Gloda.NOUN_CONTACT) {
+      this._log.warn("Somehow got a non-contact: " + aContact);
+      return; // this will produce an exception; we like.
+    }
+
+    // update the name
+    if (card.displayName && card.displayName != aContact.name)
+      aContact.name = card.displayName;
+
+    aContact.freeTags = [];
+
+    let tags = null;
+    try {
+      tags = card.getProperty("Categories", null);
+    } catch (ex) {
+      this._log.error("Problem accessing property: " + ex);
+    }
+    if (tags) {
+      for (let tagName of tags.split(",")) {
+        tagName = tagName.trim();
+        if (tagName) {
+          aContact.freeTags.push(FreeTagNoun.getFreeTag(tagName));
+        }
+      }
+    }
+
+    yield Gloda.kWorkDone;
+  }
+};

mailnews/db/gloda/modules/log4moz.js

@@ -0,0 +1,932 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['Log4Moz'];
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+var MODE_RDONLY   = 0x01;
+var MODE_WRONLY   = 0x02;
+var MODE_CREATE   = 0x08;
+var MODE_APPEND   = 0x10;
+var MODE_TRUNCATE = 0x20;
+
+var PERMS_FILE      = parseInt("0644", 8);
+var PERMS_DIRECTORY = parseInt("0755", 8);
+
+var ONE_BYTE = 1;
+var ONE_KILOBYTE = 1024 * ONE_BYTE;
+var ONE_MEGABYTE = 1024 * ONE_KILOBYTE;
+
+var DEFAULT_NETWORK_TIMEOUT_DELAY = 5;
+
+var CDATA_START = "<![CDATA[";
+var CDATA_END = "]]>";
+var CDATA_ESCAPED_END = CDATA_END + "]]&gt;" + CDATA_START;
+
+var Log4Moz = {
+  Level: {
+    Fatal:  70,
+    Error:  60,
+    Warn:   50,
+    Info:   40,
+    Config: 30,
+    Debug:  20,
+    Trace:  10,
+    All:    0,
+    Desc: {
+      70: "FATAL",
+      60: "ERROR",
+      50: "WARN",
+      40: "INFO",
+      30: "CONFIG",
+      20: "DEBUG",
+      10: "TRACE",
+      0:  "ALL"
+    }
+  },
+
+  /**
+   * Create a logger and configure it with dump and console appenders as
+   * specified by prefs based on the logger name.
+   *
+   * E.g., if the loggername is foo, then look for prefs
+   *   foo.logging.console
+   *   foo.logging.dump
+   *
+   * whose values can be empty: no logging of that type; or any of
+   * 'Fatal', 'Error', 'Warn', 'Info', 'Config', 'Debug', 'Trace', 'All',
+   * in which case the logging level for each appender will be set accordingly
+   *
+   * Parameters:
+   *
+   * @param loggername The name of the logger
+   * @param level (optional) the level of the logger itself
+   * @param consoleLevel (optional) the level of the console appender
+   * @param dumpLevel (optional) the level of the dump appender
+   *
+   * As described above, well-named prefs override the last two parameters
+   **/
+
+  getConfiguredLogger: function(loggername, level, consoleLevel, dumpLevel) {
+    let log = Log4Moz.repository.getLogger(loggername);
+    if (log._configured)
+      return log
+
+    let formatter = new Log4Moz.BasicFormatter();
+
+    level = level || Log4Moz.Level.Error;
+
+    consoleLevel = consoleLevel || -1;
+    dumpLevel = dumpLevel || -1;
+    let branch = Services.prefs.getBranch(loggername + ".logging.");
+    if (branch)
+    {
+      try {
+        // figure out if event-driven indexing should be enabled...
+        let consoleLevelString = branch.getCharPref("console");
+        if (consoleLevelString) {
+          // capitalize to fit with Log4Moz.Level expectations
+          consoleLevelString = consoleLevelString.charAt(0).toUpperCase() +
+             consoleLevelString.substr(1).toLowerCase();
+          consoleLevel = (consoleLevelString == 'None') ?
+                          100 : Log4Moz.Level[consoleLevelString];
+        }
+      } catch (ex) {
+        // Ignore if preference is not found
+      }
+      try {
+        let dumpLevelString = branch.getCharPref("dump");
+        if (dumpLevelString) {
+          // capitalize to fit with Log4Moz.Level expectations
+          dumpLevelString = dumpLevelString.charAt(0).toUpperCase() +
+             dumpLevelString.substr(1).toLowerCase();
+          dumpLevel = (dumpLevelString == 'None') ?
+                       100 : Log4Moz.Level[dumpLevelString];
+        }
+      } catch (ex) {
+        // Ignore if preference is not found
+      }
+    }
+
+    if (consoleLevel != 100) {
+      if (consoleLevel == -1)
+        consoleLevel = Log4Moz.Level.Error;
+      let capp = new Log4Moz.ConsoleAppender(formatter);
+      capp.level = consoleLevel;
+      log.addAppender(capp);
+    }
+
+    if (dumpLevel != 100) {
+      if (dumpLevel == -1)
+        dumpLevel = Log4Moz.Level.Error;
+      let dapp = new Log4Moz.DumpAppender(formatter);
+      dapp.level = dumpLevel;
+      log.addAppender(dapp);
+    }
+
+    log.level = Math.min(level, Math.min(consoleLevel, dumpLevel));
+
+    log._configured = true;
+
+    return log;
+  },
+
+  get repository() {
+    delete Log4Moz.repository;
+    Log4Moz.repository = new LoggerRepository();
+    return Log4Moz.repository;
+  },
+  set repository(value) {
+    delete Log4Moz.repository;
+    Log4Moz.repository = value;
+  },
+
+  get LogMessage() { return LogMessage; },
+  get Logger() { return Logger; },
+  get LoggerRepository() { return LoggerRepository; },
+
+  get Formatter() { return Formatter; },
+  get BasicFormatter() { return BasicFormatter; },
+  get XMLFormatter() { return XMLFormatter; },
+  get JSONFormatter() { return JSONFormatter; },
+  get Appender() { return Appender; },
+  get DumpAppender() { return DumpAppender; },
+  get ConsoleAppender() { return ConsoleAppender; },
+  get TimeAwareMemoryBucketAppender() { return TimeAwareMemoryBucketAppender; },
+  get FileAppender() { return FileAppender; },
+  get SocketAppender() { return SocketAppender; },
+  get RotatingFileAppender() { return RotatingFileAppender; },
+  get ThrowingAppender() { return ThrowingAppender; },
+
+  // Logging helper:
+  // let logger = Log4Moz.repository.getLogger("foo");
+  // logger.info(Log4Moz.enumerateInterfaces(someObject).join(","));
+  enumerateInterfaces: function Log4Moz_enumerateInterfaces(aObject) {
+    let interfaces = [];
+
+    for (i in Ci) {
+      try {
+        aObject.QueryInterface(Ci[i]);
+        interfaces.push(i);
+      }
+      catch(ex) {}
+    }
+
+    return interfaces;
+  },
+
+  // Logging helper:
+  // let logger = Log4Moz.repository.getLogger("foo");
+  // logger.info(Log4Moz.enumerateProperties(someObject).join(","));
+  enumerateProperties: function Log4Moz_enumerateProps(aObject,
+                                                       aExcludeComplexTypes) {
+    let properties = [];
+
+    for (var p in aObject) {
+      try {
+        if (aExcludeComplexTypes &&
+            (typeof aObject[p] == "object" || typeof aObject[p] == "function"))
+          continue;
+        properties.push(p + " = " + aObject[p]);
+      }
+      catch(ex) {
+        properties.push(p + " = " + ex);
+      }
+    }
+
+    return properties;
+  }
+};
+
+function LoggerContext() {
+  this._started = this._lastStateChange = Date.now();
+  this._state = "started";
+}
+LoggerContext.prototype = {
+  _jsonMe: true,
+  _id: "unknown",
+  setState: function LoggerContext_state(aState) {
+    this._state = aState;
+    this._lastStateChange = Date.now();
+    return this;
+  },
+  finish: function LoggerContext_finish() {
+    this._finished = Date.now();
+    this._state = "finished";
+    return this;
+  },
+  toString: function LoggerContext_toString() {
+    return "[Context: " + this._id + " state: " + this._state + "]";
+  }
+};
+
+
+/*
+ * LogMessage
+ * Encapsulates a single log event's data
+ */
+function LogMessage(loggerName, level, messageObjects){
+  this.loggerName = loggerName;
+  this.messageObjects = messageObjects;
+  this.level = level;
+  this.time = Date.now();
+}
+LogMessage.prototype = {
+  get levelDesc() {
+    if (this.level in Log4Moz.Level.Desc)
+      return Log4Moz.Level.Desc[this.level];
+    return "UNKNOWN";
+  },
+
+  toString: function LogMsg_toString(){
+    return "LogMessage [" + this.time + " " + this.level + " " +
+      this.messageObjects + "]";
+  }
+};
+
+/*
+ * Logger
+ * Hierarchical version.  Logs to all appenders, assigned or inherited
+ */
+
+function Logger(name, repository) {
+  this._init(name, repository);
+}
+Logger.prototype = {
+  _init: function Logger__init(name, repository) {
+    if (!repository)
+      repository = Log4Moz.repository;
+    this._name = name;
+    this.children = [];
+    this.ownAppenders = [];
+    this.appenders = [];
+    this._repository = repository;
+  },
+
+  get name() {
+    return this._name;
+  },
+
+  _level: null,
+  get level() {
+    if (this._level != null)
+      return this._level;
+    if (this.parent)
+      return this.parent.level;
+    dump("log4moz warning: root logger configuration error: no level defined\n");
+    return Log4Moz.Level.All;
+  },
+  set level(level) {
+    this._level = level;
+  },
+
+  _parent: null,
+  get parent() { return this._parent; },
+  set parent(parent) {
+    if (this._parent == parent) {
+      return;
+    }
+    // Remove ourselves from parent's children
+    if (this._parent) {
+      let index = this._parent.children.indexOf(this);
+      if (index != -1) {
+        this._parent.children.splice(index, 1);
+      }
+    }
+    this._parent = parent;
+    parent.children.push(this);
+    this.updateAppenders();
+  },
+
+  updateAppenders: function updateAppenders() {
+    if (this._parent) {
+      let notOwnAppenders = this._parent.appenders.filter(function(appender) {
+        return this.ownAppenders.indexOf(appender) == -1;
+      }, this);
+      this.appenders = notOwnAppenders.concat(this.ownAppenders);
+    } else {
+      this.appenders = this.ownAppenders.slice();
+    }
+
+    // Update children's appenders.
+    for (let i = 0; i < this.children.length; i++) {
+      this.children[i].updateAppenders();
+    }
+  },
+
+  addAppender: function Logger_addAppender(appender) {
+    if (this.ownAppenders.indexOf(appender) != -1) {
+      return;
+    }
+    this.ownAppenders.push(appender);
+    this.updateAppenders();
+  },
+
+  _nextContextId: 0,
+  newContext: function Logger_newContext(objWithProps) {
+    if (!("_id" in objWithProps))
+      objWithProps._id = this._name + ":" + (++this._nextContextId);
+
+    let c = new LoggerContext();
+    c._isContext = true;
+    for (let key in objWithProps) {
+      c[key] = objWithProps[key];
+    }
+    return c;
+  },
+
+  log: function Logger_log(message) {
+    if (this.level > message.level)
+      return;
+    let appenders = this.appenders;
+    for (let i = 0; i < appenders.length; i++){
+      appenders[i].append(message);
+    }
+  },
+
+  removeAppender: function Logger_removeAppender(appender) {
+    let index = this.ownAppenders.indexOf(appender);
+    if (index == -1) {
+      return;
+    }
+    this.ownAppenders.splice(index, 1);
+    this.updateAppenders();
+  },
+
+  log: function Logger_log(level, args) {
+    if (this.level > level)
+      return;
+
+    // Hold off on creating the message object until we actually have
+    // an appender that's responsible.
+    let message;
+    let appenders = this.appenders;
+    for (let i = 0; i < appenders.length; i++){
+      let appender = appenders[i];
+      if (appender.level > level)
+        continue;
+
+      if (!message)
+        message = new LogMessage(this._name, level,
+                                 Array.prototype.slice.call(args));
+
+      appender.append(message);
+    }
+  },
+
+  fatal: function Logger_fatal() {
+    this.log(Log4Moz.Level.Fatal, arguments);
+  },
+  error: function Logger_error() {
+    this.log(Log4Moz.Level.Error, arguments);
+  },
+  warn: function Logger_warn() {
+    this.log(Log4Moz.Level.Warn, arguments);
+  },
+  info: function Logger_info(string) {
+    this.log(Log4Moz.Level.Info, arguments);
+  },
+  config: function Logger_config(string) {
+    this.log(Log4Moz.Level.Config, arguments);
+  },
+  debug: function Logger_debug(string) {
+    this.log(Log4Moz.Level.Debug, arguments);
+  },
+  trace: function Logger_trace(string) {
+    this.log(Log4Moz.Level.Trace, arguments);
+  }
+};
+
+/*
+ * LoggerRepository
+ * Implements a hierarchy of Loggers
+ */
+
+function LoggerRepository() {}
+LoggerRepository.prototype = {
+  _loggers: {},
+
+  _rootLogger: null,
+  get rootLogger() {
+    if (!this._rootLogger) {
+      this._rootLogger = new Logger("root", this);
+      this._rootLogger.level = Log4Moz.Level.All;
+    }
+    return this._rootLogger;
+  },
+  set rootLogger(logger) {
+    throw "Cannot change the root logger";
+  },
+
+  _updateParents: function LogRep__updateParents(name) {
+    let pieces = name.split('.');
+    let cur, parent;
+
+    // find the closest parent
+    // don't test for the logger name itself, as there's a chance it's already
+    // there in this._loggers
+    for (let i = 0; i < pieces.length - 1; i++) {
+      if (cur)
+        cur += '.' + pieces[i];
+      else
+        cur = pieces[i];
+      if (cur in this._loggers)
+        parent = cur;
+    }
+
+    // if we didn't assign a parent above, there is no parent
+    if (!parent)
+      this._loggers[name].parent = this.rootLogger;
+    else
+      this._loggers[name].parent = this._loggers[parent];
+
+    // trigger updates for any possible descendants of this logger
+    for (let logger in this._loggers) {
+      if (logger != name && logger.indexOf(name) == 0)
+        this._updateParents(logger);
+    }
+  },
+
+  getLogger: function LogRep_getLogger(name) {
+    if (name in this._loggers)
+      return this._loggers[name];
+    this._loggers[name] = new Logger(name, this);
+    this._updateParents(name);
+    return this._loggers[name];
+  }
+};
+
+/*
+ * Formatters
+ * These massage a LogMessage into whatever output is desired
+ * Only the BasicFormatter is currently implemented
+ */
+
+// Abstract formatter
+function Formatter() {}
+Formatter.prototype = {
+  format: function Formatter_format(message) {}
+};
+
+// services' log4moz lost the date formatting default...
+function BasicFormatter(dateFormat) {
+  if (dateFormat)
+    this.dateFormat = dateFormat;
+}
+BasicFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  _dateFormat: null,
+
+  get dateFormat() {
+    if (!this._dateFormat)
+      this._dateFormat = "%Y-%m-%d %H:%M:%S";
+    return this._dateFormat;
+  },
+
+  set dateFormat(format) {
+    this._dateFormat = format;
+  },
+
+  format: function BF_format(message) {
+    let date = new Date(message.time);
+    // The trick below prevents errors further down because mo is null or
+    //  undefined.
+    let messageString = message.messageObjects.map(mo => "" + mo).join(" ");
+    return date.toLocaleFormat(this.dateFormat) + "\t" +
+      message.loggerName + "\t" + message.levelDesc + "\t" +
+      messageString + "\n";
+  }
+};
+
+/*
+ * XMLFormatter
+ * Format like log4j's XMLLayout.  The intent is that you can hook this up to
+ * a SocketAppender and point them at a Chainsaw GUI running with an
+ * XMLSocketReceiver running.  Then your output comes out in Chainsaw.
+ * (Chainsaw is log4j's GUI that displays log output with niceties such as
+ * filtering and conditional coloring.)
+ */
+
+function XMLFormatter() {}
+XMLFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  format: function XF_format(message) {
+    let cdataEscapedMessage =
+      message.messageObjects
+        .map(mo => (typeof(mo) == "object") ? mo.toString() : mo)
+        .join(" ")
+        .split(CDATA_END).join(CDATA_ESCAPED_END);
+    return "<log4j:event logger='" + message.loggerName + "' " +
+                        "level='" + message.levelDesc + "' thread='unknown' " +
+                        "timestamp='" + message.time + "'>" +
+      "<log4j:message><![CDATA[" + cdataEscapedMessage + "]]></log4j:message>" +
+      "</log4j:event>";
+  }
+};
+
+function JSONFormatter() {
+}
+JSONFormatter.prototype = {
+  __proto__: Formatter.prototype,
+
+  format: function JF_format(message) {
+    // XXX I did all kinds of questionable things in here; they should be
+    //  resolved...
+    // 1) JSON does not walk the __proto__ chain; there is no need to clobber
+    //   it.
+    // 2) Our net mutation is sorta redundant messageObjects alongside
+    //   msgObjects, although we only serialize one.
+    let origMessageObjects = message.messageObjects;
+    message.messageObjects = [];
+    let reProto = [];
+    for (let messageObject of origMessageObjects) {
+      if (messageObject)
+        if (messageObject._jsonMe) {
+          message.messageObjects.push(messageObject);
+// FIXME: the commented out code should be fixed in a better way.
+// See bug 984539: find a good way to avoid JSONing the impl in log4moz
+//          // temporarily strip the prototype to avoid JSONing the impl.
+//          reProto.push([messageObject, messageObject.__proto__]);
+//          messageObject.__proto__ = undefined;
+        }
+        else
+          message.messageObjects.push(messageObject.toString());
+      else
+        message.messageObjects.push(messageObject);
+    }
+    let encoded = JSON.stringify(message) + "\r\n";
+    message.msgObjects = origMessageObjects;
+//    for (let objectAndProtoPair of reProto) {
+//      objectAndProtoPair[0].__proto__ = objectAndProtoPair[1];
+//    }
+    return encoded;
+  }
+};
+
+
+/*
+ * Appenders
+ * These can be attached to Loggers to log to different places
+ * Simply subclass and override doAppend to implement a new one
+ */
+
+function Appender(formatter) {
+  this._name = "Appender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+Appender.prototype = {
+  _level: Log4Moz.Level.All,
+
+  append: function App_append(message) {
+    this.doAppend(this._formatter.format(message));
+  },
+  toString: function App_toString() {
+    return this._name + " [level=" + this._level +
+      ", formatter=" + this._formatter + "]";
+  },
+  doAppend: function App_doAppend(message) {}
+};
+
+/*
+ * DumpAppender
+ * Logs to standard out
+ */
+
+function DumpAppender(formatter) {
+  this._name = "DumpAppender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+DumpAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  doAppend: function DApp_doAppend(message) {
+    dump(message);
+  }
+};
+
+/**
+ * An in-memory appender that always logs to its in-memory bucket and associates
+ * each message with a timestamp.  Whoever creates us is responsible for causing
+ * us to switch to a new bucket using whatever criteria is appropriate.
+ *
+ * This is intended to be used roughly like an in-memory circular buffer.  The
+ * expectation is that we are being used for unit tests and that each unit test
+ * function will get its own bucket.  In the event that a test fails we would
+ * be asked for the contents of the current bucket and some portion of the
+ * previous bucket using up to some duration.
+ */
+function TimeAwareMemoryBucketAppender() {
+  this._name = "TimeAwareMemoryBucketAppender";
+  this._level = Log4Moz.Level.All;
+
+  this._lastBucket = null;
+  // to minimize object construction, even indices are timestamps, odd indices
+  //  are the message objects.
+  this._curBucket = [];
+  this._curBucketStartedAt = Date.now();
+}
+TimeAwareMemoryBucketAppender.prototype = {
+  get level() { return this._level; },
+  set level(level) { this._level = level; },
+
+  append: function TAMBA_append(message) {
+    if (this._level <= message.level)
+      this._curBucket.push(message);
+  },
+
+  newBucket: function() {
+    this._lastBucket = this._curBucket;
+    this._curBucketStartedAt = Date.now();
+    this._curBucket = [];
+  },
+
+  getPreviousBucketEvents: function(aNumMS) {
+    let lastBucket = this._lastBucket;
+    if (lastBucket == null || !lastBucket.length)
+      return [];
+    let timeBound = this._curBucketStartedAt - aNumMS;
+    // seek backwards through the list...
+    let i;
+    for (i = lastBucket.length - 1; i >= 0; i --) {
+      if (lastBucket[i].time < timeBound)
+        break;
+    }
+    return lastBucket.slice(i+1);
+  },
+
+  getBucketEvents: function() {
+    return this._curBucket.concat();
+  },
+
+  toString: function() {
+    return "[TimeAwareMemoryBucketAppender]";
+  },
+};
+
+/*
+ * ConsoleAppender
+ * Logs to the javascript console
+ */
+
+function ConsoleAppender(formatter) {
+  this._name = "ConsoleAppender";
+  this._formatter = formatter;
+}
+ConsoleAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  // override to send Error and higher level messages to Components.utils.reportError()
+  append: function CApp_append(message) {
+    let stringMessage = this._formatter.format(message);
+    if (message.level > Log4Moz.Level.Warn) {
+      Cu.reportError(stringMessage);
+    }
+    this.doAppend(stringMessage);
+  },
+
+  doAppend: function CApp_doAppend(message) {
+    Services.console.logStringMessage(message);
+  }
+};
+
+/*
+ * FileAppender
+ * Logs to a file
+ */
+
+function FileAppender(file, formatter) {
+  this._name = "FileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+FileAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  __fos: null,
+  get _fos() {
+    if (!this.__fos)
+      this.openStream();
+    return this.__fos;
+  },
+
+  openStream: function FApp_openStream() {
+    this.__fos = Cc["@mozilla.org/network/file-output-stream;1"].
+      createInstance(Ci.nsIFileOutputStream);
+    let flags = MODE_WRONLY | MODE_CREATE | MODE_APPEND;
+    this.__fos.init(this._file, flags, PERMS_FILE, 0);
+  },
+
+  closeStream: function FApp_closeStream() {
+    if (!this.__fos)
+      return;
+    try {
+      this.__fos.close();
+      this.__fos = null;
+    } catch(e) {
+      dump("Failed to close file output stream\n" + e);
+    }
+  },
+
+  doAppend: function FApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this._fos.write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+
+  clear: function FApp_clear() {
+    this.closeStream();
+    this._file.remove(false);
+  }
+};
+
+/*
+ * RotatingFileAppender
+ * Similar to FileAppender, but rotates logs when they become too large
+ */
+
+function RotatingFileAppender(file, formatter, maxSize, maxBackups) {
+  if (maxSize === undefined)
+    maxSize = ONE_MEGABYTE * 2;
+
+  if (maxBackups === undefined)
+    maxBackups = 0;
+
+  this._name = "RotatingFileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._maxSize = maxSize;
+  this._maxBackups = maxBackups;
+}
+RotatingFileAppender.prototype = {
+  __proto__: FileAppender.prototype,
+
+  doAppend: function RFApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this.rotateLogs();
+      this._fos.write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+  rotateLogs: function RFApp_rotateLogs() {
+    if(this._file.exists() &&
+       this._file.fileSize < this._maxSize)
+      return;
+
+    this.closeStream();
+
+    for (let i = this.maxBackups - 1; i > 0; i--){
+      let backup = this._file.parent.clone();
+      backup.append(this._file.leafName + "." + i);
+      if (backup.exists())
+        backup.moveTo(this._file.parent, this._file.leafName + "." + (i + 1));
+    }
+
+    let cur = this._file.clone();
+    if (cur.exists())
+      cur.moveTo(cur.parent, cur.leafName + ".1");
+
+    // Note: this._file still points to the same file
+  }
+};
+
+/*
+ * SocketAppender
+ * Logs via TCP to a given host and port.  Attempts to automatically reconnect
+ * when the connection drops or cannot be initially re-established.  Connection
+ * attempts will happen at most every timeoutDelay seconds (has a sane default
+ * if left blank).  Messages are dropped when there is no connection.
+ */
+
+function SocketAppender(host, port, formatter, timeoutDelay) {
+  this._name = "SocketAppender";
+  this._host = host;
+  this._port = port;
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._timeout_delay = timeoutDelay || DEFAULT_NETWORK_TIMEOUT_DELAY;
+
+  this._socketService = Cc["@mozilla.org/network/socket-transport-service;1"]
+                          .getService(Ci.nsISocketTransportService);
+  this._mainThread = Services.tm.mainThread;
+}
+SocketAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  __nos: null,
+  get _nos() {
+    if (!this.__nos)
+      this.openStream();
+    return this.__nos;
+  },
+  _nextCheck: 0,
+  openStream: function SApp_openStream() {
+    let now = Date.now();
+    if (now <= this._nextCheck) {
+      return;
+    }
+    this._nextCheck = now + this._timeout_delay * 1000;
+    try {
+      this._transport = this._socketService.createTransport(
+        null, 0, // default socket type
+        this._host, this._port,
+        null); // no proxy
+      this._transport.setTimeout(Ci.nsISocketTransport.TIMEOUT_CONNECT,
+                                 this._timeout_delay);
+      // do not set a timeout for TIMEOUT_READ_WRITE. The timeout is not
+      //  entirely intuitive; your socket will time out if no one reads or
+      //  writes to the socket within the timeout.  That, as you can imagine,
+      //  is not what we want.
+      this._transport.setEventSink(this, this._mainThread);
+
+      let outputStream = this._transport.openOutputStream(
+        0, // neither blocking nor unbuffered operation is desired
+        0, // default buffer size is fine
+        0 // default buffer count is fine
+        );
+
+      let uniOutputStream = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                              .createInstance(Ci.nsIConverterOutputStream);
+      uniOutputStream.init(outputStream, "utf-8", 0, 0x0000);
+
+      this.__nos = uniOutputStream;
+    } catch (ex) {
+      dump("Unexpected SocketAppender connection problem: " +
+           ex.fileName + ":" + ex.lineNumber + ": " + ex + "\n");
+    }
+  },
+
+  closeStream: function SApp_closeStream() {
+    if (!this._transport)
+      return;
+    try {
+      this._connected = false;
+      this._transport = null;
+      let nos = this.__nos;
+      this.__nos = null;
+      nos.close();
+    } catch(e) {
+      // this shouldn't happen, but no one cares
+    }
+  },
+
+  doAppend: function SApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      let nos = this._nos;
+      if (nos)
+        nos.writeString(message);
+    } catch(e) {
+      if (this._transport && !this._transport.isAlive()) {
+        this.closeStream();
+      }
+    }
+  },
+
+  clear: function SApp_clear() {
+    this.closeStream();
+  },
+
+  /* nsITransportEventSink */
+  onTransportStatus: function SApp_onTransportStatus(aTransport, aStatus,
+      aProgress, aProgressMax) {
+    if (aStatus == 0x804b0004) // STATUS_CONNECTED_TO is not a constant.
+      this._connected = true;
+  },
+};
+
+/**
+ * Throws an exception whenever it gets a message.  Intended to be used in
+ * automated testing situations where the code would normally log an error but
+ * not die in a fatal manner.
+ */
+function ThrowingAppender(thrower, formatter) {
+  this._name = "ThrowingAppender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+  this._thrower = thrower;
+}
+ThrowingAppender.prototype = {
+  __proto__: Appender.prototype,
+
+  doAppend: function TApp_doAppend(message) {
+    if (this._thrower)
+      this._thrower(message);
+    else
+      throw message;
+  }
+};

mailnews/db/gloda/modules/mimeTypeCategories.js

@@ -0,0 +1,204 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file wants to be a data file of some sort.  It might do better as a real
+ *  raw JSON file.  It is trying to be one right now, but it obviously is not.
+ */
+
+var EXPORTED_SYMBOLS = ['MimeCategoryMapping'];
+
+/**
+ * Input data structure to allow us to build a fast mapping from mime type to
+ *  category name.  The keys in MimeCategoryMapping are the top-level
+ *  categories.  Each value can either be a list of MIME types or a nested
+ *  object which recursively defines sub-categories.  We currently do not use
+ *  the sub-categories.  They are just there to try and organize the MIME types
+ *  a little and open the door to future enhancements.
+ *
+ * Do _not_ add additional top-level categories unless you have added
+ *  corresponding entries to gloda.properties under the
+ *  "gloda.mimetype.category" branch and are making sure localizers are aware
+ *  of the change and have time to localize it.
+ *
+ * Entries with wildcards in them are part of a fallback strategy by the
+ *  |mimeTypeNoun| and do not actually use regular expressions or anything like
+ *  that.  Everything is a straight string lookup.  Given "foo/bar" we look for
+ *  "foo/bar", then "foo/*", and finally "*".
+ */
+var MimeCategoryMapping = {
+  archives: [
+    "application/java-archive",
+    "application/x-java-archive",
+    "application/x-jar",
+    "application/x-java-jnlp-file",
+
+    "application/mac-binhex40",
+    "application/vnd.ms-cab-compressed",
+
+    "application/x-arc",
+    "application/x-arj",
+    "application/x-compress",
+    "application/x-compressed-tar",
+    "application/x-cpio",
+    "application/x-cpio-compressed",
+    "application/x-deb",
+
+    "application/x-bittorrent",
+
+    "application/x-rar",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/zip",
+    "application/x-zip-compressed",
+    "application/x-zip",
+
+    "application/x-bzip",
+    "application/x-bzip-compressed-tar",
+    "application/x-bzip2",
+    "application/x-gzip",
+    "application/x-tar",
+    "application/x-tar-gz",
+    "application/x-tarz",
+  ],
+  documents: {
+    database: [
+      "application/vnd.ms-access",
+      "application/x-msaccess",
+      "application/msaccess",
+      "application/vnd.msaccess",
+      "application/x-msaccess",
+      "application/mdb",
+      "application/x-mdb",
+
+      "application/vnd.oasis.opendocument.database",
+
+    ],
+    graphics: [
+      "application/postscript",
+      "application/x-bzpostscript",
+      "application/x-dvi",
+      "application/x-gzdvi",
+
+      "application/illustrator",
+
+      "application/vnd.corel-draw",
+      "application/cdr",
+      "application/coreldraw",
+      "application/x-cdr",
+      "application/x-coreldraw",
+      "image/cdr",
+      "image/x-cdr",
+      "zz-application/zz-winassoc-cdr",
+
+      "application/vnd.oasis.opendocument.graphics",
+      "application/vnd.oasis.opendocument.graphics-template",
+      "application/vnd.oasis.opendocument.image",
+
+      "application/x-dia-diagram",
+    ],
+    presentation: [
+      "application/vnd.ms-powerpoint.presentation.macroenabled.12",
+      "application/vnd.ms-powerpoint.template.macroenabled.12",
+      "application/vnd.ms-powerpoint",
+      "application/powerpoint",
+      "application/mspowerpoint",
+      "application/x-mspowerpoint",
+      "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+      "application/vnd.openxmlformats-officedocument.presentationml.template",
+
+      "application/vnd.oasis.opendocument.presentation",
+      "application/vnd.oasis.opendocument.presentation-template"
+    ],
+    spreadsheet: [
+      "application/vnd.lotus-1-2-3",
+      "application/x-lotus123",
+      "application/x-123",
+      "application/lotus123",
+      "application/wk1",
+
+      "application/x-quattropro",
+
+      "application/vnd.ms-excel.sheet.binary.macroenabled.12",
+      "application/vnd.ms-excel.sheet.macroenabled.12",
+      "application/vnd.ms-excel.template.macroenabled.12",
+      "application/vnd.ms-excel",
+      "application/msexcel",
+      "application/x-msexcel",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
+
+      "application/vnd.oasis.opendocument.formula",
+      "application/vnd.oasis.opendocument.formula-template",
+      "application/vnd.oasis.opendocument.chart",
+      "application/vnd.oasis.opendocument.chart-template",
+      "application/vnd.oasis.opendocument.spreadsheet",
+      "application/vnd.oasis.opendocument.spreadsheet-template",
+
+      "application/x-gnumeric",
+    ],
+    wordProcessor: [
+      "application/msword",
+      "application/vnd.ms-word",
+      "application/x-msword",
+      "application/msword-template",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+      "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
+      "application/vnd.ms-word.document.macroenabled.12",
+      "application/vnd.ms-word.template.macroenabled.12",
+      "application/x-mswrite",
+      "application/x-pocket-word",
+
+      "application/rtf",
+      "text/rtf",
+
+
+      "application/vnd.oasis.opendocument.text",
+      "application/vnd.oasis.opendocument.text-master",
+      "application/vnd.oasis.opendocument.text-template",
+      "application/vnd.oasis.opendocument.text-web",
+
+      "application/vnd.wordperfect",
+
+      "application/x-abiword",
+      "application/x-amipro",
+    ],
+    suite: [
+      "application/vnd.ms-works"
+    ],
+  },
+  images: [
+    "image/*"
+  ],
+  media: {
+    audio: [
+      "audio/*",
+    ],
+    video: [
+      "video/*",
+    ],
+    container: [
+      "application/ogg",
+
+      "application/smil",
+      "application/vnd.ms-asf",
+      "application/vnd.rn-realmedia",
+      "application/x-matroska",
+      "application/x-quicktime-media-link",
+      "application/x-quicktimeplayer",
+    ]
+  },
+  other: [
+    "*"
+  ],
+  pdf: [
+    "application/pdf",
+    "application/x-pdf",
+    "image/pdf",
+    "file/pdf",
+
+    "application/x-bzpdf",
+    "application/x-gzpdf",
+  ],
+}

mailnews/db/gloda/modules/mimemsg.js

@@ -0,0 +1,719 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['MsgHdrToMimeMessage',
+                          'MimeMessage', 'MimeContainer',
+                          'MimeBody', 'MimeUnknown',
+                          'MimeMessageAttachment'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var EMITTER_MIME_CODE = "application/x-js-mime-message";
+
+/**
+ * The URL listener is surplus because the CallbackStreamListener ends up
+ *  getting the same set of events, effectively.
+ */
+var dumbUrlListener = {
+  OnStartRunningUrl: function (aUrl) {
+  },
+  OnStopRunningUrl: function (aUrl, aExitCode) {
+  },
+};
+
+/**
+ * Maintain a list of all active stream listeners so that we can cancel them all
+ *  during shutdown.  If we don't cancel them, we risk calls into javascript
+ *  from C++ after the various XPConnect contexts have already begun their
+ *  teardown process.
+ */
+var activeStreamListeners = {};
+
+var shutdownCleanupObserver = {
+  _initialized: false,
+  ensureInitialized: function mimemsg_shutdownCleanupObserver_init() {
+    if (this._initialized)
+      return;
+
+    Services.obs.addObserver(this, "quit-application", false);
+
+    this._initialized = true;
+  },
+
+  observe: function mimemsg_shutdownCleanupObserver_observe(
+      aSubject, aTopic, aData) {
+    if (aTopic == "quit-application") {
+      Services.obs.removeObserver(this, "quit-application");
+
+      for (let uri in activeStreamListeners) {
+        let streamListener = activeStreamListeners[uri];
+        if (streamListener._request)
+          streamListener._request.cancel(Cr.NS_BINDING_ABORTED);
+      }
+    }
+  }
+};
+
+function CallbackStreamListener(aMsgHdr, aCallbackThis, aCallback) {
+  this._msgHdr = aMsgHdr;
+  let hdrURI = aMsgHdr.folder.getUriForMsg(aMsgHdr);
+  this._request = null;
+  this._stream = null;
+  if (aCallback === undefined) {
+    this._callbacksThis = [null];
+    this._callbacks = [aCallbackThis];
+  }
+  else {
+    this._callbacksThis = [aCallbackThis];
+    this._callbacks =[aCallback];
+  }
+  activeStreamListeners[hdrURI] = this;
+}
+
+CallbackStreamListener.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIStreamListener]),
+
+  // nsIRequestObserver part
+  onStartRequest: function (aRequest, aContext) {
+    this._request = aRequest;
+  },
+  onStopRequest: function (aRequest, aContext, aStatusCode) {
+    let msgURI = this._msgHdr.folder.getUriForMsg(this._msgHdr);
+    delete activeStreamListeners[msgURI];
+
+    aContext.QueryInterface(Ci.nsIURI);
+    let message = MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aContext.spec];
+    if (message === undefined)
+      message = null;
+
+    delete MsgHdrToMimeMessage.RESULT_RENDEVOUZ[aContext.spec];
+
+    for (let i = 0; i < this._callbacksThis.length; i++) {
+      try {
+        this._callbacks[i].call(this._callbacksThis[i], this._msgHdr, message);
+      } catch (e) {
+        // Most of the time, exceptions will silently disappear into the endless
+        // deeps of XPConnect, and never reach the surface ever again. At least
+        // warn the user if he has dump enabled.
+        dump("The MsgHdrToMimeMessage callback threw an exception: "+e+"\n");
+        // That one will probably never make it to the original caller.
+        throw(e);
+      }
+    }
+
+    this._msgHdr = null;
+    this._request = null;
+    this._stream = null;
+    this._callbacksThis = null;
+    this._callbacks = null;
+  },
+
+  /* okay, our onDataAvailable should actually never be called.  the stream
+     converter is actually eating everything except the start and stop
+     notification. */
+  // nsIStreamListener part
+  onDataAvailable: function (aRequest,aContext,aInputStream,aOffset,aCount) {
+    dump("this should not be happening! arrgggggh!\n");
+    if (this._stream === null) {
+      this._stream = Cc["@mozilla.org/scriptableinputstream;1"].
+                    createInstance(Ci.nsIScriptableInputStream);
+      this._stream.init(aInputStream);
+    }
+    this._stream.read(aCount);
+
+  },
+};
+
+var gMessenger = Cc["@mozilla.org/messenger;1"].
+                   createInstance(Ci.nsIMessenger);
+
+function stripEncryptedParts(aPart) {
+  if (aPart.parts && aPart.isEncrypted) {
+    aPart.parts = []; // Show an empty container.
+  } else if (aPart.parts) {
+    aPart.parts = aPart.parts.map(stripEncryptedParts);
+  }
+  return aPart;
+}
+
+/**
+ * Starts retrieval of a MimeMessage instance for the given message header.
+ *  Your callback will be called with the message header you provide and the
+ *
+ * @param aMsgHdr The message header to retrieve the body for and build a MIME
+ *     representation of the message.
+ * @param aCallbackThis The (optional) 'this' to use for your callback function.
+ * @param aCallback The callback function to invoke on completion of message
+ *     parsing or failure.  The first argument passed will be the nsIMsgDBHdr
+ *     you passed to this function.  The second argument will be the MimeMessage
+ *     instance resulting from the processing on success, and null on failure.
+ * @param [aAllowDownload=false] Should we allow the message to be downloaded
+ *     for this streaming request?  The default is false, which means that we
+ *     require that the message be available offline.  If false is passed and
+ *     the message is not available offline, we will propagate an exception
+ *     thrown by the underlying code.
+ * @param [aOptions] Optional options.
+ * @param [aOptions.saneBodySize] Limit body sizes to a 'reasonable' size in
+ *     order to combat corrupt offline/message stores creating pathological
+ *     situtations where we have erroneously multi-megabyte messages.  This
+ *     also likely reduces the impact of legitimately ridiculously large
+ *     messages.
+ * @param [aOptions.partsOnDemand] If this is a message stored on an IMAP
+ *     server, and for whatever reason, it isn't available locally, then setting
+ *     this option to true will make sure that attachments aren't downloaded.
+ *     This makes sure the message is available quickly.
+ * @param [aOptions.examineEncryptedParts] By default, we won't reveal the
+ *     contents of multipart/encrypted parts to the consumers, unless explicitly
+ *     requested. In the case of MIME/PGP messages, for instance, the message
+ *     will appear as an empty multipart/encrypted container, unless this option
+ *     is used.
+ */
+function MsgHdrToMimeMessage(aMsgHdr, aCallbackThis, aCallback,
+                             aAllowDownload, aOptions) {
+  shutdownCleanupObserver.ensureInitialized();
+
+  let requireOffline = !aAllowDownload;
+
+  let msgURI = aMsgHdr.folder.getUriForMsg(aMsgHdr);
+  let msgService = gMessenger.messageServiceFromURI(msgURI);
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = aOptions;
+  let partsOnDemandStr = (aOptions && aOptions.partsOnDemand)
+    ? "&fetchCompleteMessage=false"
+    : "";
+  // By default, Enigmail only decrypts a message streamed via libmime if it's
+  // the one currently on display in the message reader. With this option, we're
+  // letting Enigmail know that it should decrypt the message since the client
+  // explicitly asked for it.
+  let encryptedStr = (aOptions && aOptions.examineEncryptedParts)
+    ? "&examineEncryptedParts=true"
+    : "";
+
+  // S/MIME, our other encryption backend, is not that smart, and always
+  // decrypts data. In order to protect sensitive data (e.g. not index it in
+  // Gloda), unless the client asked for encrypted data, we pass to the client
+  // callback a stripped-down version of the MIME structure where encrypted
+  // parts have been removed.
+  let wrapCallback = function (aCallback, aCallbackThis) {
+    if (aOptions && aOptions.examineEncryptedParts)
+      return aCallback;
+    else
+      return ((aMsgHdr, aMimeMsg) =>
+        aCallback.call(aCallbackThis, aMsgHdr, stripEncryptedParts(aMimeMsg))
+      );
+  };
+
+  // Apparently there used to be an old syntax where the callback was the second
+  // argument...
+  let callback = aCallback ? aCallback : aCallbackThis;
+  let callbackThis = aCallback ? aCallbackThis : null;
+
+  // if we're already streaming this msg, just add the callback
+  // to the listener.
+  let listenerForURI = activeStreamListeners[msgURI];
+  if (listenerForURI != undefined) {
+    listenerForURI._callbacks.push(wrapCallback(callback, callbackThis));
+    listenerForURI._callbacksThis.push(callbackThis);
+    return;
+  }
+  let streamListener = new CallbackStreamListener(
+    aMsgHdr,
+    callbackThis,
+    wrapCallback(callback, callbackThis)
+  );
+
+  try {
+    let streamURI = msgService.streamMessage(
+      msgURI,
+      streamListener, // consumer
+      null, // nsIMsgWindow
+      dumbUrlListener, // nsIUrlListener
+      true, // have them create the converter
+      // additional uri payload, note that "header=" is prepended automatically
+      "filter&emitter=js"+partsOnDemandStr+encryptedStr,
+      requireOffline);
+  } catch (ex) {
+    // If streamMessage throws an exception, we should make sure to clear the
+    // activeStreamListener, or any subsequent attempt at sreaming this URI
+    // will silently fail
+    if (activeStreamListeners[msgURI]) {
+      delete activeStreamListeners[msgURI];
+    }
+    MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+    throw(ex);
+  }
+
+  MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+}
+
+/**
+ * Let the jsmimeemitter provide us with results.  The poor emitter (if I am
+ *  understanding things correctly) is evaluated outside of the C.u.import
+ *  world, so if we were to import him, we would not see him, but rather a new
+ *  copy of him.  This goes for his globals, etc.  (and is why we live in this
+ *  file right here).  Also, it appears that the XPCOM JS wrappers aren't
+ *  magically unified so that we can try and pass data as expando properties
+ *  on things like the nsIUri instances either.  So we have the jsmimeemitter
+ *  import us and poke things into RESULT_RENDEVOUZ.  We put it here on this
+ *  function to try and be stealthy and avoid polluting the namespaces (or
+ *  encouraging bad behaviour) of our importers.
+ *
+ * If you can come up with a prettier way to shuttle this data, please do.
+ */
+MsgHdrToMimeMessage.RESULT_RENDEVOUZ = {};
+/**
+ * Cram rich options here for the MimeMessageEmitter to grab from.  We
+ *  leverage the known control-flow to avoid needing a whole dictionary here.
+ *  We set this immediately before constructing the emitter and clear it
+ *  afterwards.  Control flow is never yielded during the process and reentrancy
+ *  cannot happen via any other means.
+ */
+MsgHdrToMimeMessage.OPTION_TUNNEL = null;
+
+var HeaderHandlerBase = {
+  /**
+   * Look-up a header that should be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @param aDefaultValue The value to return if the header was not found, null
+   *     if left unspecified.
+   * @return the value of the header if present, and the default value if not
+   *  (defaults to null).  If the header was present multiple times, the first
+   *  instance of the header is returned.  Use getAll if you want all of the
+   *  values for the multiply-defined header.
+   */
+  get: function MimeMessage_get(aHeaderName, aDefaultValue) {
+    if (aDefaultValue === undefined) {
+      aDefaultValue = null;
+    }
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers)
+      // we require that the list cannot be empty if present
+      return this.headers[lowerHeader][0];
+    else
+      return aDefaultValue;
+  },
+  /**
+   * Look-up a header that can be present multiple times.  Use get for headers
+   *  that you only expect to be present at most once.
+   *
+   * @param aHeaderName The header name to retrieve, case does not matter.
+   * @return An array containing the values observed, which may mean a zero
+   *     length array.
+   */
+  getAll: function MimeMessage_getAll(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    if (lowerHeader in this.headers)
+      return this.headers[lowerHeader];
+    else
+      return [];
+  },
+  /**
+   * @param aHeaderName Header name to test for its presence.
+   * @return true if the message has (at least one value for) the given header
+   *     name.
+   */
+  has: function MimeMessage_has(aHeaderName) {
+    let lowerHeader = aHeaderName.toLowerCase();
+    return lowerHeader in this.headers;
+  },
+  _prettyHeaderString: function MimeMessage__prettyHeaderString(aIndent) {
+    if (aIndent === undefined)
+      aIndent = "";
+    let s = "";
+    for (let header in this.headers) {
+      let values = this.headers[header];
+      s += "\n        " + aIndent + header + ": " + values;
+    }
+    return s;
+  }
+};
+
+/**
+ * @ivar partName The MIME part, ex "1.2.2.1".  The partName of a (top-level)
+ *     message is "1", its first child is "1.1", its second child is "1.2",
+ *     its first child's first child is "1.1.1", etc.
+ * @ivar headers Maps lower-cased header field names to a list of the values
+ *     seen for the given header.  Use get or getAll as convenience helpers.
+ * @ivar parts The list of the MIME part children of this message.  Children
+ *     will be either MimeMessage instances, MimeMessageAttachment instances,
+ *     MimeContainer instances, or MimeUnknown instances.  The latter two are
+ *     the result of limitations in the Javascript representation generation
+ *     at this time, combined with the need to most accurately represent the
+ *     MIME structure.
+ */
+function MimeMessage() {
+  this.partName = null;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeMessage.prototype = {
+  __proto__: HeaderHandlerBase,
+  contentType: "message/rfc822",
+
+  /**
+   * @return a list of all attachments contained in this message and all its
+   *     sub-messages.  Only MimeMessageAttachment instances will be present in
+   *     the list (no sub-messages).
+   */
+  get allAttachments() {
+    let results = []; // messages are not attachments, don't include self
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+
+  /**
+   * @return a list of all attachments contained in this message, with
+   *    included/forwarded messages treated as real attachments. Attachments
+   *    contained in inner messages won't be shown.
+   */
+  get allUserAttachments() {
+    if (this.url)
+      // The jsmimeemitter camouflaged us as a MimeAttachment
+      return [this];
+    else
+      // Why is there no flatten method for arrays?
+      return this.parts.map(child => child.allUserAttachments)
+        .reduce((a, b) => a.concat(b), []);
+  },
+
+  /**
+   * @return the total size of this message, that is, the size of all subparts
+   */
+  get size () {
+    return this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+
+  /**
+   * In the case of attached messages, libmime considers them as attachments,
+   * and if the body is, say, quoted-printable encoded, then libmime will start
+   * counting bytes and notify the js mime emitter about it. The JS mime emitter
+   * being a nice guy, it will try to set a size on us. While this is the
+   * expected behavior for MimeMsgAttachments, we must make sure we can handle
+   * that (failing to write a setter results in exceptions being thrown).
+   */
+  set size (whatever) {
+    // nop
+  },
+
+  /**
+   * @param aMsgFolder A message folder, any message folder.  Because this is
+   *    a hack.
+   * @return The concatenation of all of the body parts where parts
+   *    available as text/plain are pulled as-is, and parts only available
+   *    as text/html are converted to plaintext form first.  In other words,
+   *    if we see a multipart/alternative with a text/plain, we take the
+   *    text/plain.  If we see a text/html without an alternative, we convert
+   *    that to text.
+   */
+  coerceBodyToPlaintext:
+      function MimeMessage_coerceBodyToPlaintext(aMsgFolder) {
+    let bodies = [];
+    for (let part of this.parts) {
+      // an undefined value for something not having the method is fine
+      let body = part.coerceBodyToPlaintext &&
+                 part.coerceBodyToPlaintext(aMsgFolder);
+      if (body)
+        bodies.push(body);
+    }
+    if (bodies)
+      return bodies.join("");
+    else
+      return "";
+  },
+
+  /**
+   * Convert the message and its hierarchy into a "pretty string".  The message
+   *  and each MIME part get their own line.  The string never ends with a
+   *  newline.  For a non-multi-part message, only a single line will be
+   *  returned.
+   * Messages have their subject displayed, attachments have their filename and
+   *  content-type (ex: image/jpeg) displayed.  "Filler" classes simply have
+   *  their class displayed.
+   */
+  prettyString: function MimeMessage_prettyString(aVerbose, aIndent,
+                                                  aDumpBody) {
+    if (aIndent === undefined)
+      aIndent = "";
+    let nextIndent = aIndent + "  ";
+
+    let s = "Message "+(this.isEncrypted ? "[encrypted] " : "") +
+      "(" + this.size + " bytes): " +
+      "subject" in this.headers ? this.headers.subject : "";
+    if (aVerbose)
+      s += this._prettyHeaderString(nextIndent);
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+};
+
+
+/**
+ * @ivar contentType The content-type of this container.
+ * @ivar parts The parts held by this container.  These can be instances of any
+ *      of the classes found in this file.
+ */
+function MimeContainer(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.parts = [];
+  this.isEncrypted = false;
+}
+
+MimeContainer.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    let results = [];
+    for (let iChild = 0; iChild < this.parts.length; iChild++) {
+      let child = this.parts[iChild];
+      results = results.concat(child.allAttachments);
+    }
+    return results;
+  },
+  get allUserAttachments () {
+    return this.parts.map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size () {
+    return this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+  set size (whatever) {
+    // nop
+  },
+  coerceBodyToPlaintext:
+      function MimeContainer_coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "multipart/alternative") {
+      let htmlPart;
+      // pick the text/plain if we can find one, otherwise remember the HTML one
+      for (let part of this.parts) {
+        if (part.contentType == "text/plain")
+          return part.body;
+        if (part.contentType == "text/html")
+          htmlPart = part;
+        // text/enriched gets transformed into HTML, use it if we don't already
+        //  have an HTML part.
+        else if (!htmlPart && part.contentType == "text/enriched")
+	  htmlPart = part;
+      }
+      // convert the HTML part if we have one
+      if (htmlPart)
+        return aMsgFolder.convertMsgSnippetToPlainText(htmlPart.body);
+    }
+    // if it's not alternative, recurse/aggregate using MimeMessage logic
+    return MimeMessage.prototype.coerceBodyToPlaintext.call(this, aMsgFolder);
+  },
+  prettyString: function MimeContainer_prettyString(aVerbose, aIndent,
+                                                    aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s = "Container "+(this.isEncrypted ? "[encrypted] " : "")+
+      "(" + this.size + " bytes): " + this.contentType;
+    if (aVerbose)
+      s += this._prettyHeaderString(nextIndent);
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        part.prettyString(aVerbose, nextIndent, aDumpBody);
+    }
+
+    return s;
+  },
+  toString: function MimeContainer_toString() {
+    return "Container: " + this.contentType;
+  }
+};
+
+/**
+ * @class Represents a body portion that we understand and do not believe to be
+ *  a proper attachment.  This means text/plain or text/html and it has no
+ *  filename.  (A filename suggests an attachment.)
+ *
+ * @ivar contentType The content type of this body materal; text/plain or
+ *     text/html.
+ * @ivar body The actual body content.
+ */
+function MimeBody(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  this.body = "";
+  this.isEncrypted = false;
+}
+
+MimeBody.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return []; // we are a leaf
+  },
+  get allUserAttachments() {
+    return []; // we are a leaf
+  },
+  get size() {
+    return this.body.length;
+  },
+  set size (whatever) {
+    // nop
+  },
+  appendBody: function MimeBody_append(aBuf) {
+    this.body += aBuf;
+  },
+  coerceBodyToPlaintext:
+      function MimeBody_coerceBodyToPlaintext(aMsgFolder) {
+    if (this.contentType == "text/plain")
+      return this.body;
+    // text/enriched gets transformed into HTML by libmime
+    if (this.contentType == "text/html" ||
+        this.contentType == "text/enriched")
+      return aMsgFolder.convertMsgSnippetToPlainText(this.body);
+    return "";
+  },
+  prettyString: function MimeBody_prettyString(aVerbose, aIndent, aDumpBody) {
+    let s = "Body: "+(this.isEncrypted ? "[encrypted] " : "")+
+      "" + this.contentType + " (" + this.body.length + " bytes" +
+      (aDumpBody ? (": '" + this.body + "'") : "") + ")";
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+    return s;
+  },
+  toString: function MimeBody_toString() {
+    return "Body: " + this.contentType + " (" + this.body.length + " bytes)";
+  }
+};
+
+/**
+ * @class A MIME Leaf node that doesn't have a filename so we assume it's not
+ *  intended to be an attachment proper.  This is probably meant for inline
+ *  display or is the result of someone amusing themselves by composing messages
+ *  by hand or a bad client.  This class should probably be renamed or we should
+ *  introduce a better named class that we try and use in preference to this
+ *  class.
+ *
+ * @ivar contentType The content type of this part.
+ */
+function MimeUnknown(aContentType) {
+  this.partName = null;
+  this.contentType = aContentType;
+  this.headers = {};
+  // Looks like libmime does not always intepret us as an attachment, which
+  //  means we'll have to have a default size. Returning undefined would cause
+  //  the recursive size computations to fail.
+  this._size = 0;
+  this.isEncrypted = false;
+  // We want to make sure MimeUnknown has a part property: S/MIME encrypted
+  // messages have a topmost MimeUnknown part, with the encrypted bit set to 1,
+  // and we need to ensure all other encrypted parts are children of this
+  // topmost part.
+  this.parts = [];
+}
+
+MimeUnknown.prototype = {
+  __proto__: HeaderHandlerBase,
+  get allAttachments() {
+    return this.parts.map(child => child.allAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get allUserAttachments() {
+    return this.parts.map(child => child.allUserAttachments)
+      .reduce((a, b) => a.concat(b), []);
+  },
+  get size() {
+    return this._size + this.parts.map(child => child.size)
+      .reduce((a, b) => a + Math.max(b, 0), 0);
+  },
+  set size(aSize) {
+    this._size = aSize;
+  },
+  prettyString: function MimeUnknown_prettyString(aVerbose, aIndent,
+                                                  aDumpBody) {
+    let nextIndent = aIndent + "  ";
+
+    let s = "Unknown: "+(this.isEncrypted ? "[encrypted] " : "")+
+      "" + this.contentType + " (" + this.size + " bytes)";
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+
+    for (let iPart = 0; iPart < this.parts.length; iPart++) {
+      let part = this.parts[iPart];
+      s += "\n" + nextIndent + (iPart+1) + " " +
+        (part ? part.prettyString(aVerbose, nextIndent, aDumpBody) : "NULL");
+    }
+    return s;
+  },
+  toString: function MimeUnknown_toString() {
+    return "Unknown: " + this.contentType;
+  }
+};
+
+/**
+ * @class An attachment proper.  We think it's an attachment because it has a
+ *  filename that libmime was able to figure out.
+ *
+ * @ivar partName @see{MimeMessage.partName}
+ * @ivar name The filename of this attachment.
+ * @ivar contentType The MIME content type of this part.
+ * @ivar url The URL to stream if you want the contents of this part.
+ * @ivar isExternal Is the attachment stored someplace else than in the message?
+ * @ivar size The size of the attachment if available, -1 otherwise (size is set
+ *  after initialization by jsmimeemitter.js)
+ */
+function MimeMessageAttachment(aPartName, aName, aContentType, aUrl,
+                               aIsExternal) {
+  this.partName = aPartName;
+  this.name = aName;
+  this.contentType = aContentType;
+  this.url = aUrl;
+  this.isExternal = aIsExternal;
+  this.headers = {};
+  this.isEncrypted = false;
+  // parts is copied over from the part instance that preceded us
+  // headers is copied over from the part instance that preceded us
+  // isEncrypted is copied over from the part instance that preceded us
+}
+
+MimeMessageAttachment.prototype = {
+  __proto__: HeaderHandlerBase,
+  // This is a legacy property.
+  get isRealAttachment() {
+    return true;
+  },
+  get allAttachments() {
+    return [this]; // we are a leaf, so just us.
+  },
+  get allUserAttachments() {
+    return [this];
+  },
+  prettyString: function MimeMessageAttachment_prettyString(aVerbose, aIndent,
+                                                            aDumpBody) {
+    let s = "Attachment "+(this.isEncrypted ? "[encrypted] " : "")+
+      "(" + this.size+" bytes): "
+      + this.name + ", " + this.contentType;
+    if (aVerbose)
+      s += this._prettyHeaderString(aIndent + "  ");
+    return s;
+  },
+  toString: function MimeMessageAttachment_toString() {
+    return this.prettyString(false, "");
+  },
+};

mailnews/db/gloda/modules/moz.build

@@ -0,0 +1,32 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXTRA_JS_MODULES.gloda += [
+    'collection.js',
+    'connotent.js',
+    'databind.js',
+    'datamodel.js',
+    'datastore.js',
+    'dbview.js',
+    'everybody.js',
+    'explattr.js',
+    'facet.js',
+    'fundattr.js',
+    'gloda.js',
+    'index_ab.js',
+    'index_msg.js',
+    'indexer.js',
+    'log4moz.js',
+    'mimemsg.js',
+    'mimeTypeCategories.js',
+    'msg_search.js',
+    'noun_freetag.js',
+    'noun_mimetype.js',
+    'noun_tag.js',
+    'public.js',
+    'query.js',
+    'suffixtree.js',
+    'utils.js',
+]

mailnews/db/gloda/modules/msg_search.js

@@ -0,0 +1,346 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaMsgSearcher"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource:///modules/gloda/public.js");
+
+/**
+ * How much time boost should a 'score point' amount to?  The authoritative,
+ *  incontrivertible answer, across all time and space, is a week.
+ *  Note that gloda stores timestamps as PRTimes for no exceedingly good
+ *  reason.
+ */
+var FUZZSCORE_TIMESTAMP_FACTOR = 1000 * 1000 * 60 * 60 * 24 * 7;
+
+var RANK_USAGE =
+  "glodaRank(matchinfo(messagesText), 1.0, 2.0, 2.0, 1.5, 1.5)";
+
+var DASCORE =
+  "(((" + RANK_USAGE + " + messages.notability) * " +
+    FUZZSCORE_TIMESTAMP_FACTOR +
+   ") + messages.date)";
+
+/**
+ * A new optimization decision we are making is that we do not want to carry
+ *  around any data in our ephemeral tables that is not used for whittling the
+ *  result set.  The idea is that the btree page cache or OS cache is going to
+ *  save us from the disk seeks and carrying around the extra data is just going
+ *  to be CPU/memory churn that slows us down.
+ *
+ * Additionally, we try and avoid row lookups that would have their results
+ *  discarded by the LIMIT.  Because of limitations in FTS3 (which might
+ *  be addressed in FTS4 by a feature request), we can't avoid the 'messages'
+ *  lookup since that has the message's date and static notability but we can
+ *  defer the 'messagesText' lookup.
+ *
+ * This is the access pattern we are after here:
+ * 1) Order the matches with minimized lookup and result storage costs.
+ * - The innermost MATCH does the doclist magic and provides us with
+ *    matchinfo() support which does not require content row retrieval
+ *    from messagesText.  Unfortunately, this is not enough to whittle anything
+ *    because we still need static interestingness, so...
+ * - Based on the match we retrieve the date and notability for that row from
+ *    'messages' using this in conjunction with matchinfo() to provide a score
+ *    that we can then use to LIMIT our results.
+ * 2) We reissue the MATCH query so that we will be able to use offsets(), but
+ *    we intersect the results of this MATCH against our LIMITed results from
+ *    step 1.
+ * - We use 'docid IN (phase 1 query)' to accomplish this because it results in
+ *    efficient lookup.  If we just use a join, we get O(mn) performance because
+ *    a cartesian join ends up being performed where either we end up performing
+ *    the fulltext query M times and table scan intersect with the results from
+ *    phase 1 or we do the fulltext once but traverse the entire result set from
+ *    phase 1 N times.
+ * - We believe that the re-execution of the MATCH query should have no disk
+ *    costs because it should still be cached by SQLite or the OS.  In the case
+ *    where memory is so constrained this is not true our behavior is still
+ *    probably preferable than the old way because that would have caused lots
+ *    of swapping.
+ * - This part of the query otherwise resembles the basic gloda query but with
+ *    the inclusion of the offsets() invocation.  The messages table lookup
+ *    should not involve any disk traffic because the pages should still be
+ *    cached (SQLite or OS) from phase 1.  The messagesText lookup is new, and
+ *    this is the major disk-seek reduction optimization we are making.  (Since
+ *    we avoid this lookup for all of the documents that were excluded by the
+ *    LIMIT.)  Since offsets() also needs to retrieve the row from messagesText
+ *    there is a nice synergy there.
+ */
+var NUEVO_FULLTEXT_SQL =
+  "SELECT messages.*, messagesText.*, offsets(messagesText) AS osets " +
+  "FROM messagesText, messages " +
+  "WHERE" +
+    " messagesText MATCH ?1 " +
+    " AND messagesText.docid IN (" +
+       "SELECT docid " +
+       "FROM messagesText JOIN messages ON messagesText.docid = messages.id " +
+       "WHERE messagesText MATCH ?1 " +
+       "ORDER BY " + DASCORE + " DESC " +
+       "LIMIT ?2" +
+    " )" +
+    " AND messages.id = messagesText.docid " +
+    " AND +messages.deleted = 0" +
+    " AND +messages.folderID IS NOT NULL" +
+    " AND +messages.messageKey IS NOT NULL";
+
+function identityFunc(x) {
+  return x;
+}
+
+function oneLessMaxZero(x) {
+  if (x <= 1)
+    return 0;
+  else
+    return x - 1;
+}
+
+function reduceSum(accum, curValue) {
+  return accum + curValue;
+}
+
+/*
+ * Columns are: body, subject, attachment names, author, recipients
+ */
+
+/**
+ * Scores if all search terms match in a column.  We bias against author
+ *  slightly and recipient a bit more in this case because a search that
+ *  entirely matches just on a person should give a mention of that person
+ *  in the subject or attachment a fighting chance.
+ * Keep in mind that because of our indexing in the face of address book
+ *  contacts (namely, we index the name used in the e-mail as well as the
+ *  display name on the address book card associated with the e-mail adress)
+ *  a contact is going to bias towards matching multiple times.
+ */
+var COLUMN_ALL_MATCH_SCORES = [4, 20, 20, 16, 12];
+/**
+ * Score for each distinct term that matches in the column.  This is capped
+ *  by COLUMN_ALL_SCORES.
+ */
+var COLUMN_PARTIAL_PER_MATCH_SCORES = [1, 4, 4, 4, 3];
+/**
+ * If a term matches multiple times, what is the marginal score for each
+ *  additional match.  We count the total number of matches beyond the
+ *  first match for each term.  In other words, if we have 3 terms which
+ *  matched 5, 3, and 0 times, then the total from our perspective is
+ *  (5 - 1) + (3 - 1) + 0 = 4 + 2 + 0 = 6.  We take the minimum of that value
+ *  and the value in COLUMN_MULTIPLE_MATCH_LIMIT and multiply by the value in
+ *  COLUMN_MULTIPLE_MATCH_SCORES.
+ */
+var COLUMN_MULTIPLE_MATCH_SCORES = [1, 0, 0, 0, 0];
+var COLUMN_MULTIPLE_MATCH_LIMIT = [10, 0, 0, 0, 0];
+
+/**
+ * Score the message on its offsets (from stashedColumns).
+ */
+function scoreOffsets(aMessage, aContext) {
+  let score = 0;
+
+  let termTemplate = aContext.terms.map(_ => 0);
+  // for each column, a list of the incidence of each term
+  let columnTermIncidence = [termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat(),
+                             termTemplate.concat()];
+
+  // we need a friendlyParseInt because otherwise the radix stuff happens
+  //  because of the extra arguments map parses.  curse you, map!
+  let offsetNums =
+    aContext.stashedColumns[aMessage.id][0].split(" ").map(x => parseInt(x));
+  for (let i=0; i < offsetNums.length; i += 4) {
+    let columnIndex = offsetNums[i];
+    let termIndex = offsetNums[i+1];
+    columnTermIncidence[columnIndex][termIndex]++;
+  }
+
+  for (let iColumn = 0; iColumn < COLUMN_ALL_MATCH_SCORES.length; iColumn++) {
+    let termIncidence = columnTermIncidence[iColumn];
+    // bestow all match credit
+    if (termIncidence.every(identityFunc))
+      score += COLUMN_ALL_MATCH_SCORES[iColumn];
+    // bestow partial match credit
+    else if (termIncidence.some(identityFunc))
+      score += Math.min(COLUMN_ALL_MATCH_SCORES[iColumn],
+                        COLUMN_PARTIAL_PER_MATCH_SCORES[iColumn] *
+                          termIncidence.filter(identityFunc).length);
+    // bestow multiple match credit
+    score += Math.min(termIncidence.map(oneLessMaxZero).reduce(reduceSum, 0),
+                      COLUMN_MULTIPLE_MATCH_LIMIT[iColumn]) *
+             COLUMN_MULTIPLE_MATCH_SCORES[iColumn];
+  }
+
+  return score;
+}
+
+/**
+ * The searcher basically looks like a query, but is specialized for fulltext
+ *  search against messages.  Most of the explicit specialization involves
+ *  crafting a SQL query that attempts to order the matches by likelihood that
+ *  the user was looking for it.  This is based on full-text matches combined
+ *  with an explicit (generic) interest score value placed on the message at
+ *  indexing time.  This is followed by using the more generic gloda scoring
+ *  mechanism to explicitly score the messages given the search context in
+ *  addition to the more generic score adjusting rules.
+ */
+function GlodaMsgSearcher(aListener, aSearchString, aAndTerms) {
+  this.listener = aListener;
+
+  this.searchString = aSearchString;
+  this.fulltextTerms = this.parseSearchString(aSearchString);
+  this.andTerms = (aAndTerms != null) ? aAndTerms : true;
+
+  this.query = null;
+  this.collection = null;
+
+  this.scores = null;
+}
+GlodaMsgSearcher.prototype = {
+  /**
+   * Number of messages to retrieve initially.
+   */
+  get retrievalLimit() {
+    return Services.prefs.getIntPref(
+      "mailnews.database.global.search.msg.limit"
+    );
+  },
+
+  /**
+   * Parse the string into terms/phrases by finding matching double-quotes.
+   */
+  parseSearchString: function GlodaMsgSearcher_parseSearchString(aSearchString) {
+    aSearchString = aSearchString.trim();
+    let terms = [];
+
+    /*
+     * Add the term as long as the trim on the way in didn't obliterate it.
+     *
+     * In the future this might have other helper logic; it did once before.
+     */
+    function addTerm(aTerm) {
+      if (aTerm)
+        terms.push(aTerm);
+    }
+
+    while (aSearchString) {
+      if (aSearchString.startsWith('"')) {
+        let endIndex = aSearchString.indexOf(aSearchString[0], 1);
+        // eat the quote if it has no friend
+        if (endIndex == -1) {
+          aSearchString = aSearchString.substring(1);
+          continue;
+        }
+
+        addTerm(aSearchString.substring(1, endIndex).trim());
+        aSearchString = aSearchString.substring(endIndex + 1);
+        continue;
+      }
+
+      let spaceIndex = aSearchString.indexOf(" ");
+      if (spaceIndex == -1) {
+        addTerm(aSearchString);
+        break;
+      }
+
+      addTerm(aSearchString.substring(0, spaceIndex));
+      aSearchString = aSearchString.substring(spaceIndex+1);
+    }
+
+    return terms;
+  },
+
+  buildFulltextQuery: function GlodaMsgSearcher_buildFulltextQuery() {
+    let query = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
+      noMagic: true,
+      explicitSQL: NUEVO_FULLTEXT_SQL,
+      limitClauseAlreadyIncluded: true,
+      // osets is 0-based column number 14 (volatile to column changes)
+      // save the offset column for extra analysis
+      stashColumns: [14]
+    });
+
+    let fulltextQueryString = "";
+
+    for (let [iTerm, term] of this.fulltextTerms.entries()) {
+      if (iTerm)
+        fulltextQueryString += this.andTerms ? " " : " OR ";
+
+      // Put our term in quotes.  This is needed for the tokenizer to be able
+      //  to do useful things.  The exception is people clever enough to use
+      //  NEAR.
+      if (/^NEAR(\/\d+)?$/.test(term))
+        fulltextQueryString += term;
+      // Check if this is a single-character CJK search query.  If so, we want
+      //  to add a wildcard.
+      // Our tokenizer treats anything at/above 0x2000 as CJK for now.
+      else if (term.length == 1 && term.charCodeAt(0) >= 0x2000)
+        fulltextQueryString += term + "*";
+      else if (
+          term.length == 2 &&
+            term.charCodeAt(0) >= 0x2000 &&
+            term.charCodeAt(1) >= 0x2000
+          || term.length >= 3
+      )
+        fulltextQueryString += '"' + term + '"';
+
+    }
+
+    query.fulltextMatches(fulltextQueryString);
+    query.limit(this.retrievalLimit);
+
+    return query;
+  },
+
+  getCollection: function GlodaMsgSearcher_getCollection(
+      aListenerOverride, aData) {
+    if (aListenerOverride)
+      this.listener = aListenerOverride;
+
+    this.query = this.buildFulltextQuery();
+    this.collection = this.query.getCollection(this, aData);
+    this.completed = false;
+
+    return this.collection;
+  },
+
+  sortBy: '-dascore',
+
+  onItemsAdded: function GlodaMsgSearcher_onItemsAdded(aItems, aCollection) {
+    let newScores = Gloda.scoreNounItems(
+      aItems,
+      {
+        terms: this.fulltextTerms,
+        stashedColumns: aCollection.stashedColumns
+      },
+      [scoreOffsets]);
+    if (this.scores)
+      this.scores = this.scores.concat(newScores);
+    else
+      this.scores = newScores;
+
+    if (this.listener)
+      this.listener.onItemsAdded(aItems, aCollection);
+  },
+  onItemsModified: function GlodaMsgSearcher_onItemsModified(aItems,
+                                                             aCollection) {
+    if (this.listener)
+      this.listener.onItemsModified(aItems, aCollection);
+  },
+  onItemsRemoved: function GlodaMsgSearcher_onItemsRemoved(aItems,
+                                                           aCollection) {
+    if (this.listener)
+      this.listener.onItemsRemoved(aItems, aCollection);
+  },
+  onQueryCompleted: function GlodaMsgSearcher_onQueryCompleted(aCollection) {
+    this.completed = true;
+    if (this.listener)
+      this.listener.onQueryCompleted(aCollection);
+  },
+};

mailnews/db/gloda/modules/noun_freetag.js

@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['FreeTag', 'FreeTagNoun'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+function FreeTag(aTagName) {
+  this.name = aTagName;
+}
+
+FreeTag.prototype = {
+  toString: function () {
+    return this.name;
+  }
+};
+
+/**
+ * @namespace Tag noun provider.  Since the tag unique value is stored as a
+ *  parameter, we are an odd case and semantically confused.
+ */
+var FreeTagNoun = {
+  _log: Log4Moz.repository.getLogger("gloda.noun.freetag"),
+
+  name: "freetag",
+  clazz: FreeTag,
+  allowsArbitraryAttrs: false,
+  usesParameter: true,
+
+  _listeners: [],
+  addListener: function(aListener) {
+    this._listeners.push(aListener);
+  },
+  removeListener: function(aListener) {
+    let index = this._listeners.indexOf(aListener);
+    if (index >=0)
+      this._listeners.splice(index, 1);
+  },
+
+  populateKnownFreeTags: function() {
+    for (let attr of this.objectNounOfAttributes) {
+      let attrDB = attr.dbDef;
+      for (let param in attrDB.parameterBindings) {
+        this.getFreeTag(param);
+      }
+    }
+  },
+
+  knownFreeTags: {},
+  getFreeTag: function(aTagName) {
+    let tag = this.knownFreeTags[aTagName];
+    if (!tag) {
+      tag = this.knownFreeTags[aTagName] = new FreeTag(aTagName);
+      for (let listener of this._listeners)
+        listener.onFreeTagAdded(tag);
+    }
+    return tag;
+  },
+
+  comparator: function gloda_noun_freetag_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.name.localeCompare(b.name);
+  },
+
+  toParamAndValue: function gloda_noun_freetag_toParamAndValue(aTag) {
+    return [aTag.name, null];
+  },
+
+  toJSON: function gloda_noun_freetag_toJSON(aTag) {
+    return aTag.name;
+  },
+  fromJSON: function gloda_noun_freetag_fromJSON(aTagName) {
+    return this.getFreeTag(aTagName);
+  },
+};
+
+Gloda.defineNoun(FreeTagNoun);

mailnews/db/gloda/modules/noun_mimetype.js

@@ -0,0 +1,365 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['MimeType', 'MimeTypeNoun'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+Cu.import("resource:///modules/StringBundle.js");
+
+var LOG = Log4Moz.repository.getLogger("gloda.noun.mimetype");
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+var CategoryStringMap = {};
+
+/**
+ * Mime type abstraction that exists primarily so we can map mime types to
+ *  integer id's.
+ *
+ * Instances of this class should only be retrieved via |MimeTypeNoun|; no one
+ *  should ever create an instance directly.
+ */
+function MimeType(aID, aType, aSubType, aFullType, aCategory) {
+  this._id = aID;
+  this._type = aType;
+  this._subType = aSubType;
+  this._fullType = aFullType;
+  this._category = aCategory;
+}
+
+MimeType.prototype = {
+  /**
+   * The integer id we have associated with the mime type.  This is stable for
+   *  the lifetime of the database, which means that anything in the Gloda
+   *  database can use this without fear.  Things not persisted in the database
+   *  should use the actual string mime type, retrieval via |fullType|.
+   */
+  get id() { return this._id; },
+  /**
+   * The first part of the MIME type; "text/plain" gets you "text".
+   */
+  get type() { return this._type; },
+  set fullType(aFullType) {
+    if (!this._fullType) {
+      this._fullType = aFullType;
+      [this._type, this._subType] = this._fullType.split("/");
+      this._category =
+        MimeTypeNoun._getCategoryForMimeType(aFullType, this._type);
+    }
+  },
+  /**
+   * If the |fullType| is "text/plain", subType is "plain".
+   */
+  get subType() { return this._subType; },
+  /**
+   * The full MIME type; "text/plain" returns "text/plain".
+   */
+  get fullType() { return this._fullType; },
+  toString: function () {
+    return this.fullType;
+  },
+
+  /**
+   * @return the category we believe this mime type belongs to.  This category
+   *     name should never be shown directly to the user.  Instead, use
+   *     |categoryLabel| to get the localized name for the category.  The
+   *     category mapping comes from mimeTypesCategories.js.
+   */
+  get category() {
+    return this._category;
+  },
+  /**
+   * @return The localized label for the category from gloda.properties in the
+   *     "gloda.mimetype.category.CATEGORY.label" definition using the value
+   *     from |category|.
+   */
+  get categoryLabel() {
+    return CategoryStringMap[this._category];
+  }
+};
+
+/**
+ * Mime type noun provider.
+ *
+ * The set of MIME Types is sufficiently limited that we can keep them all in
+ *  memory.  In theory it is also sufficiently limited that we could use the
+ *  parameter mechanism in the database.  However, it is more efficient, for
+ *  both space and performance reasons, to store the specific mime type as a
+ *  value.  For future-proofing reasons, we opt to use a database table to
+ *  persist the mapping rather than a hard-coded list.  A preferences file or
+ *  other text file would arguably suffice, but for consistency reasons, the
+ *  database is not a bad thing.
+ */
+var MimeTypeNoun = {
+  name: "mime-type",
+  clazz: MimeType, // gloda supports clazz as well as class
+  allowsArbitraryAttrs: false,
+
+  _strings: new StringBundle("chrome://messenger/locale/gloda.properties"),
+
+  // note! update test_noun_mimetype if you change our internals!
+  _mimeTypes: {},
+  _mimeTypesByID: {},
+  TYPE_BLOCK_SIZE: 16384,
+  _mimeTypeHighID: {},
+  _mimeTypeRangeDummyObjects: {},
+  _highID: 0,
+
+  // we now use the exciting 'schema' mechanism of defineNoun to get our table
+  //  created for us, plus some helper methods that we simply don't use.
+  schema: {
+    name: 'mimeTypes',
+    columns: [['id', 'INTEGER PRIMARY KEY', '_id'],
+              ['mimeType', 'TEXT', 'fullType']],
+  },
+
+  _init: function() {
+    LOG.debug("loading MIME types");
+    this._loadCategoryMapping();
+    this._loadMimeTypes();
+  },
+
+  /**
+   * A map from MIME type to category name.
+   */
+  _mimeTypeToCategory: {},
+  /**
+   * Load the contents of mimeTypeCategories.js and populate
+   */
+  _loadCategoryMapping: function MimeTypeNoun__loadCategoryMapping() {
+    let mimecatNS = {};
+    Cu.import("resource:///modules/gloda/mimeTypeCategories.js",
+              mimecatNS);
+    let mcm = mimecatNS.MimeCategoryMapping;
+
+    let mimeTypeToCategory = this._mimeTypeToCategory;
+
+    function procMapObj(aSubTree, aCategories) {
+      for (let key in aSubTree) {
+        let value = aSubTree[key];
+        // Add this category to our nested categories list.  Use concat since
+        //  the list will be long-lived and each list needs to be distinct.
+        let categories = aCategories.concat();
+        categories.push(key);
+
+        if (categories.length == 1) {
+          CategoryStringMap[key] =
+            MimeTypeNoun._strings.get(
+              "gloda.mimetype.category." + key + ".label");
+        }
+
+        // Is it an array? If so, just process this depth
+        if (Array.isArray(value)) {
+          for (let mimeTypeStr of value) {
+            mimeTypeToCategory[mimeTypeStr] = categories;
+          }
+        }
+        // it's yet another sub-tree branch
+        else {
+          procMapObj(value, categories);
+        }
+      }
+    }
+
+    procMapObj(mimecatNS.MimeCategoryMapping, []);
+  },
+
+  /**
+   * Lookup the category associated with a MIME type given its full type and
+   *  type.  (So, "foo/bar" and "foo" for "foo/bar".)
+   */
+  _getCategoryForMimeType:
+      function MimeTypeNoun__getCategoryForMimeType(aFullType, aType) {
+    if (aFullType in this._mimeTypeToCategory)
+      return this._mimeTypeToCategory[aFullType][0];
+    let wildType = aType + "/*";
+    if (wildType in this._mimeTypeToCategory)
+      return this._mimeTypeToCategory[wildType][0];
+    return this._mimeTypeToCategory["*"][0];
+  },
+
+  /**
+   * In order to allow the gloda query mechanism to avoid hitting the database,
+   *  we need to either define the noun type as cachable and have a super-large
+   *  cache or simply have a collection with every MIME type in it that stays
+   *  alive forever.
+   * This is that collection.  It is initialized by |_loadMimeTypes|.  As new
+   *  MIME types are created, we add them to the collection.
+   */
+  _universalCollection: null,
+
+  /**
+   * Kick off a query of all the mime types in our database, leaving
+   *  |_processMimeTypes| to actually do the legwork.
+   */
+  _loadMimeTypes: function MimeTypeNoun__loadMimeTypes() {
+    // get all the existing mime types!
+    let query = Gloda.newQuery(this.id);
+    let nullFunc = function() {};
+    this._universalCollection = query.getCollection({
+      onItemsAdded: nullFunc, onItemsModified: nullFunc,
+      onItemsRemoved: nullFunc,
+      onQueryCompleted: function (aCollection) {
+        MimeTypeNoun._processMimeTypes(aCollection.items);
+      }
+    }, null);
+  },
+
+  /**
+   * For the benefit of our Category queryHelper, we need dummy ranged objects
+   *  that cover the numerical address space allocated to the category.  We
+   *  can't use a real object for the upper-bound because the upper-bound is
+   *  constantly growing and there is the chance the query might get persisted,
+   *  which means these values need to be long-lived.  Unfortunately, our
+   *  solution to this problem (dummy objects) complicates the second case,
+   *  should it ever occur.  (Because the dummy objects cannot be persisted
+   *  on their own... but there are other issues that will come up that we will
+   *  just have to deal with then.)
+   */
+  _createCategoryDummies: function (aId, aCategory) {
+    let blockBottom = aId - (aId % this.TYPE_BLOCK_SIZE);
+    let blockTop = blockBottom + this.TYPE_BLOCK_SIZE - 1;
+    this._mimeTypeRangeDummyObjects[aCategory] = [
+      new MimeType(blockBottom, "!category-dummy!", aCategory,
+                   "!category-dummy!/" + aCategory, aCategory),
+      new MimeType(blockTop, "!category-dummy!", aCategory,
+                   "!category-dummy!/" + aCategory, aCategory)
+    ];
+  },
+
+  _processMimeTypes: function MimeTypeNoun__processMimeTypes(aMimeTypes) {
+    for (let mimeType of aMimeTypes) {
+      if (mimeType.id > this._highID)
+        this._highID = mimeType.id;
+      this._mimeTypes[mimeType] = mimeType;
+      this._mimeTypesByID[mimeType.id] = mimeType;
+
+      let typeBlock = mimeType.id - (mimeType.id % this.TYPE_BLOCK_SIZE);
+      let blockHighID = (mimeType.category in this._mimeTypeHighID) ?
+                          this._mimeTypeHighID[mimeType.category] : undefined;
+      // create the dummy range objects
+      if (blockHighID === undefined)
+        this._createCategoryDummies(mimeType.id, mimeType.category);
+      if ((blockHighID === undefined) || mimeType.id > blockHighID)
+        this._mimeTypeHighID[mimeType.category] = mimeType.id;
+    }
+  },
+
+  _addNewMimeType: function MimeTypeNoun__addNewMimeType(aMimeTypeName) {
+    let [typeName, subTypeName] = aMimeTypeName.split("/");
+    let category = this._getCategoryForMimeType(aMimeTypeName, typeName);
+
+    if (!(category in this._mimeTypeHighID)) {
+      let nextID = this._highID - (this._highID % this.TYPE_BLOCK_SIZE) +
+        this.TYPE_BLOCK_SIZE;
+      this._mimeTypeHighID[category] = nextID;
+      this._createCategoryDummies(nextID, category);
+    }
+
+    let nextID = ++this._mimeTypeHighID[category];
+
+    let mimeType = new MimeType(nextID, typeName, subTypeName, aMimeTypeName,
+                                category);
+    if (mimeType.id > this._highID)
+      this._highID = mimeType.id;
+
+    this._mimeTypes[aMimeTypeName] = mimeType;
+    this._mimeTypesByID[nextID] = mimeType;
+
+    // As great as the gloda extension mechanisms are, we don't think it makes
+    //  a lot of sense to use them in this case.  So we directly trigger object
+    //  insertion without any of the grokNounItem stuff.
+    this.objInsert.call(this.datastore, mimeType);
+    // Since we bypass grokNounItem and its fun, we need to explicitly add the
+    //  new MIME-type to _universalCollection ourselves.  Don't try this at
+    //  home, kids.
+    this._universalCollection._onItemsAdded([mimeType]);
+
+    return mimeType;
+  },
+
+  /**
+   * Map a mime type to a |MimeType| instance, creating it if necessary.
+   *
+   * @param aMimeTypeName The mime type.  It may optionally include parameters
+   *     (which will be ignored).  A mime type is of the form "type/subtype".
+   *     A type with parameters would look like 'type/subtype; param="value"'.
+   */
+  getMimeType: function MimeTypeNoun_getMimeType(aMimeTypeName) {
+    // first, lose any parameters
+    let semiIndex = aMimeTypeName.indexOf(";");
+    if (semiIndex >= 0)
+      aMimeTypeName = aMimeTypeName.substring(0, semiIndex);
+    aMimeTypeName = aMimeTypeName.trim().toLowerCase();
+
+    if (aMimeTypeName in this._mimeTypes)
+      return this._mimeTypes[aMimeTypeName];
+    else
+      return this._addNewMimeType(aMimeTypeName);
+  },
+
+  /**
+   * Query helpers contribute additional functions to the query object for the
+   *  attributes that use the noun type.  For example, we define Category, so
+   *  for the "attachmentTypes" attribute, "attachmentTypesCategory" would be
+   *  exposed.
+   */
+  queryHelpers: {
+    /**
+     * Query for MIME type categories based on one or more MIME type objects
+     *  passed in.  We want the range to span the entire block allocated to the
+     *  category.
+     *
+     * @param aAttrDef The attribute that is using us.
+     * @param aArguments The actual arguments object that
+     */
+    Category: function(aAttrDef, aArguments) {
+      let rangePairs = [];
+      // If there are no arguments then we want to fall back to the 'in'
+      //  constraint which matches on any attachment.
+      if (aArguments.length == 0)
+        return this._inConstraintHelper(aAttrDef, []);
+
+      for (let iArg = 0; iArg < aArguments.length; iArg++) {
+        let arg = aArguments[iArg];
+        rangePairs.push(MimeTypeNoun._mimeTypeRangeDummyObjects[arg.category]);
+      }
+      return this._rangedConstraintHelper(aAttrDef, rangePairs);
+    }
+  },
+
+  comparator: function gloda_noun_mimeType_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.fullType.localeCompare(b.fullType);
+  },
+
+  toParamAndValue: function gloda_noun_mimeType_toParamAndValue(aMimeType) {
+    return [null, aMimeType.id];
+  },
+  toJSON: function gloda_noun_mimeType_toJSON(aMimeType) {
+    return aMimeType.id;
+  },
+  fromJSON: function gloda_noun_mimeType_fromJSON(aMimeTypeID) {
+    return this._mimeTypesByID[aMimeTypeID];
+  },
+};
+Gloda.defineNoun(MimeTypeNoun, Gloda.NOUN_MIME_TYPE);
+try {
+MimeTypeNoun._init();
+} catch (ex) {
+  LOG.error("problem init-ing: " + ex.fileName + ":" + ex.lineNumber + ": " + ex);
+}

mailnews/db/gloda/modules/noun_tag.js

@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['TagNoun'];
+
+Components.utils.import("resource:///modules/mailServices.js");
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/gloda.js");
+
+/**
+ * @namespace Tag noun provider.
+ */
+var TagNoun = {
+  name: "tag",
+  clazz: Ci.nsIMsgTag,
+  usesParameter: true,
+  allowsArbitraryAttrs: false,
+  idAttr: "key",
+  _msgTagService: null,
+  _tagMap: null,
+  _tagList: null,
+
+  _init: function () {
+    this._msgTagService = MailServices.tags;
+    this._updateTagMap();
+  },
+
+  getAllTags: function gloda_noun_tag_getAllTags() {
+    if (this._tagList == null)
+      this._updateTagMap();
+    return this._tagList;
+  },
+
+  _updateTagMap: function gloda_noun_tag_updateTagMap() {
+    this._tagMap = {};
+    let tagArray = this._tagList = this._msgTagService.getAllTags({});
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      this._tagMap[tag.key] = tag;
+    }
+  },
+
+  comparator: function gloda_noun_tag_comparator(a, b) {
+    if (a == null) {
+      if (b == null)
+        return 0;
+      else
+        return 1;
+    }
+    else if (b == null) {
+      return -1;
+    }
+    return a.tag.localeCompare(b.tag);
+  },
+  userVisibleString: function gloda_noun_tag_userVisibleString(aTag) {
+    return aTag.tag;
+  },
+
+  // we cannot be an attribute value
+
+  toParamAndValue: function gloda_noun_tag_toParamAndValue(aTag) {
+    return [aTag.key, null];
+  },
+  toJSON: function gloda_noun_tag_toJSON(aTag) {
+    return aTag.key;
+  },
+  fromJSON: function gloda_noun_tag_fromJSON(aTagKey, aIgnored) {
+    let tag = this._tagMap.hasOwnProperty(aTagKey) ? this._tagMap[aTagKey]
+                : undefined;
+    // you will note that if a tag is removed, we are unable to aggressively
+    //  deal with this.  we are okay with this, but it would be nice to be able
+    //  to listen to the message tag service to know when we should rebuild.
+    if ((tag === undefined) && this._msgTagService.isValidKey(aTagKey)) {
+      this._updateTagMap();
+      tag = this._tagMap[aTagKey];
+    }
+    // we intentionally are returning undefined if the tag doesn't exist
+    return tag;
+  },
+  /**
+   * Convenience helper to turn a tag key into a tag name.
+   */
+  getTag: function gloda_noun_tag_getTag(aTagKey) {
+    return this.fromJSON(aTagKey);
+  }
+};
+
+TagNoun._init();
+Gloda.defineNoun(TagNoun, Gloda.NOUN_TAG);

mailnews/db/gloda/modules/public.js

@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["Gloda"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/gloda.js");
+Cu.import("resource:///modules/gloda/everybody.js");
+Cu.import("resource:///modules/gloda/indexer.js");
+// initialize the indexer! (who was actually imported as a nested dep by the
+//  things everybody.js imported.)  We waited until now so it could know about
+//  its indexers.
+GlodaIndexer._init();
+Cu.import("resource:///modules/gloda/index_msg.js");
+
+/**
+ * Expose some junk
+ */
+function proxy(aSourceObj, aSourceAttr, aDestObj, aDestAttr) {
+  aDestObj[aDestAttr] = function() {
+    return aSourceObj[aSourceAttr].apply(aSourceObj, arguments);
+  };
+}
+
+proxy(GlodaIndexer, "addListener", Gloda, "addIndexerListener");
+proxy(GlodaIndexer, "removeListener", Gloda, "removeIndexerListener");
+proxy(GlodaMsgIndexer, "isMessageIndexed", Gloda, "isMessageIndexed");
+proxy(GlodaMsgIndexer, "setFolderIndexingPriority", Gloda,
+      "setFolderIndexingPriority");
+proxy(GlodaMsgIndexer, "resetFolderIndexingPriority", Gloda,
+      "resetFolderIndexingPriority");

mailnews/db/gloda/modules/query.js

@@ -0,0 +1,618 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["GlodaQueryClassFactory"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/gloda/log4moz.js");
+
+// GlodaDatastore has some constants we need, and oddly enough, there was no
+//  load dependency preventing us from doing this.
+Cu.import("resource:///modules/gloda/datastore.js");
+
+/**
+ * @class Query class core; each noun gets its own sub-class where attributes
+ *  have helper methods bound.
+ *
+ * @param aOptions A dictionary of options.  Current legal options are:
+ *     - noMagic: Indicates that the noun's dbQueryJoinMagic should be ignored.
+ *                Currently, this means that messages will not have their
+ *                full-text indexed values re-attached.  This is planned to be
+ *                offset by having queries/cache lookups that do not request
+ *                noMagic to ensure that their data does get loaded.
+ *     - explicitSQL: A hand-rolled alternate representation for the core
+ *           SELECT portion of the SQL query.  The queryFromQuery logic still
+ *           generates its normal query, we just ignore its result in favor of
+ *           your provided value.  This means that the positional parameter
+ *           list is still built and you should/must rely on those bound
+ *           parameters (using '?').  The replacement occurs prior to the
+ *           outerWrapColumns, ORDER BY, and LIMIT contributions to the query.
+ *     - outerWrapColumns: If provided, wraps the query in a "SELECT *,blah
+ *           FROM (actual query)" where blah is your list of outerWrapColumns
+ *           made comma-delimited.  The idea is that this allows you to
+ *           reference the result of expressions inside the query using their
+ *           names rather than having to duplicate the logic.  In practice,
+ *           this makes things more readable but is unlikely to improve
+ *           performance.  (Namely, my use of 'offsets' for full-text stuff
+ *           ends up in the EXPLAIN plan twice despite this.)
+ *     - noDbQueryValidityConstraints: Indicates that any validity constraints
+ *           should be ignored. This should be used when you need to get every
+ *           match regardless of whether it's valid.
+ *
+ * @property _owner The query instance that holds the list of unions...
+ * @property _constraints A list of (lists of OR constraints) that are ANDed
+ *     together.  For example [[FROM bob, FROM jim], [DATE last week]] would
+ *     be requesting us to find all the messages from either bob or jim, and
+ *     sent in the last week.
+ * @property _unions A list of other queries whose results are unioned with our
+ *     own.  There is no concept of nesting or sub-queries apart from this
+ *     mechanism.
+ */
+function GlodaQueryClass(aOptions) {
+  this.options = (aOptions != null) ? aOptions : {};
+
+  // if we are an 'or' clause, who is our parent whom other 'or' clauses should
+  //  spawn from...
+  this._owner = null;
+  // our personal chain of and-ing.
+  this._constraints = [];
+  // the other instances we union with
+  this._unions = [];
+
+  this._order = [];
+  this._limit = 0;
+}
+
+GlodaQueryClass.prototype = {
+  WILDCARD: {},
+
+  get constraintCount() {
+    return this._constraints.length;
+  },
+
+  or: function gloda_query_or() {
+    let owner = this._owner || this;
+    let orQuery = new this._queryClass();
+    orQuery._owner = owner;
+    owner._unions.push(orQuery);
+    return orQuery;
+  },
+
+  orderBy: function gloda_query_orderBy() {
+    for (let iArg = 0; iArg < arguments.length; iArg++) {
+      let arg = arguments[iArg];
+      this._order.push(arg);
+    }
+    return this;
+  },
+
+  limit: function gloda_query_limit(aLimit) {
+    this._limit = aLimit;
+    return this;
+  },
+
+  /**
+   * Return a collection asynchronously populated by this collection.  You must
+   *  provide a listener to receive notifications from the collection as it
+   *  receives updates.  The listener object should implement onItemsAdded,
+   *  onItemsModified, and onItemsRemoved methods, all of which take a single
+   *  argument which is the list of items which have been added, modified, or
+   *  removed respectively.
+   *
+   * @param aListener The collection listener.
+   * @param [aData] The data attribute to set on the collection.
+   * @param [aArgs.becomeExplicit] Make the collection explicit so that the
+   *     collection will only ever contain results found from the database
+   *     query and the query will not be updated as new items are indexed that
+   *     also match the query.
+   * @param [aArgs.becomeNull] Change the collection's query to a null query so
+   *     that it will never receive any additional added/modified/removed events
+   *     apart from the underlying database query.  This is really only intended
+   *     for gloda internal use but may be acceptable for non-gloda use.  Please
+   *     ask on mozilla.dev.apps.thunderbird first to make sure there isn't a
+   *     better solution for your use-case.  (Note: removals will still happen
+   *     when things get fully deleted.)
+   */
+  getCollection: function gloda_query_getCollection(aListener, aData, aArgs) {
+    this.completed = false;
+    return this._nounDef.datastore.queryFromQuery(this, aListener, aData,
+             /* aExistingCollection */ null, /* aMasterCollection */ null,
+             aArgs);
+  },
+
+  /**
+   * Test whether the given first-class noun instance satisfies this query.
+   *
+   * @testpoint gloda.query.test
+   */
+  test: function gloda_query_test(aObj) {
+    // when changing this method, be sure that GlodaDatastore's queryFromQuery
+    //  method likewise has any required changes made.
+    let unionQueries = [this].concat(this._unions);
+
+    for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
+      let curQuery = unionQueries[iUnion];
+
+      // assume success until a specific (or) constraint proves us wrong
+      let querySatisfied = true;
+      for (let iConstraint = 0; iConstraint < curQuery._constraints.length;
+           iConstraint++) {
+        let constraint = curQuery._constraints[iConstraint];
+        let [constraintType, attrDef] = constraint;
+        let boundName = attrDef ? attrDef.boundName : "id";
+        if ((boundName in aObj) &&
+            aObj[boundName] === GlodaDatastore.IGNORE_FACET) {
+          querySatisfied = false;
+          break;
+        }
+
+        let constraintValues = constraint.slice(2);
+
+        if (constraintType === GlodaDatastore.kConstraintIdIn) {
+          if (constraintValues.indexOf(aObj.id) == -1) {
+            querySatisfied = false;
+            break;
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintIn
+        else if ((constraintType === GlodaDatastore.kConstraintIn) ||
+                 (constraintType === GlodaDatastore.kConstraintEquals)) {
+          let objectNounDef = attrDef.objectNounDef;
+
+          // if they provide an equals comparator, use that.
+          // (note: the next case has better optimization possibilities than
+          //  this mechanism, but of course has higher initialization costs or
+          //  code complexity costs...)
+          if (objectNounDef.equals) {
+            let testValues;
+            if (!(boundName in aObj))
+              testValues = [];
+            else if (attrDef.singular)
+              testValues = [aObj[boundName]];
+            else
+              testValues = aObj[boundName];
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (constraintValues.length == 0 && testValues.length &&
+                testValues[0] != null)
+              continue;
+
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier)
+                continue;
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              for (let value of constraintValues) {
+                if (objectNounDef.equals(testValue, value)) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch)
+                break;
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          }
+          // otherwise, we need to convert everyone to their param/value form
+          //  in order to test for equality
+          else {
+            // let's just do the simple, obvious thing for now.  which is
+            //  what we did in the prior case but exploding values using
+            //  toParamAndValue, and then comparing.
+            let testValues;
+            if (!(boundName in aObj))
+              testValues = [];
+            else if (attrDef.singular)
+              testValues = [aObj[boundName]];
+            else
+              testValues = aObj[boundName];
+
+            // If there are no constraints, then we are just testing for there
+            //  being a value.  Succeed (continue) in that case.
+            if (constraintValues.length == 0 && testValues.length &&
+                testValues[0] != null)
+              continue;
+            // If there are no test values and the empty set is significant,
+            //  then check if any of the constraint values are null (our
+            //  empty indicator.)
+            if (testValues.length == 0 && attrDef.emptySetIsSignificant) {
+              let foundEmptySetSignifier = false;
+              for (let constraintValue of constraintValues) {
+                if (constraintValue == null) {
+                  foundEmptySetSignifier = true;
+                  break;
+                }
+              }
+              if (foundEmptySetSignifier)
+                continue;
+            }
+
+            let foundMatch = false;
+            for (let testValue of testValues) {
+              let [aParam, aValue] = objectNounDef.toParamAndValue(testValue);
+              for (let value of constraintValues) {
+                // skip empty set check sentinel values
+                if (value == null && attrDef.emptySetIsSignificant)
+                  continue;
+                let [bParam, bValue] = objectNounDef.toParamAndValue(value);
+                if (aParam == bParam && aValue == bValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              if (foundMatch)
+                break;
+            }
+            if (!foundMatch) {
+              querySatisfied = false;
+              break;
+            }
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintRanges
+        else if (constraintType === GlodaDatastore.kConstraintRanges) {
+          let objectNounDef = attrDef.objectNounDef;
+
+          let testValues;
+          if (!(boundName in aObj))
+              testValues = [];
+          else if (attrDef.singular)
+            testValues = [aObj[boundName]];
+          else
+            testValues = aObj[boundName];
+
+          let foundMatch = false;
+          for (let testValue of testValues) {
+            let [tParam, tValue] = objectNounDef.toParamAndValue(testValue);
+            for (let rangeTuple of constraintValues) {
+              let [lowerRValue, upperRValue] = rangeTuple;
+              if (lowerRValue == null) {
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                if (tParam == upperParam && tValue <= upperValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              else if (upperRValue == null) {
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if (tParam == lowerParam && tValue >= lowerValue) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+              else { // no one is null
+                let [upperParam, upperValue] =
+                  objectNounDef.toParamAndValue(upperRValue);
+                let [lowerParam, lowerValue] =
+                  objectNounDef.toParamAndValue(lowerRValue);
+                if ((tParam == lowerParam) && (tValue >= lowerValue) &&
+                    (tParam == upperParam) && (tValue <= upperValue)) {
+                  foundMatch = true;
+                  break;
+                }
+              }
+            }
+            if (foundMatch)
+              break;
+          }
+          if (!foundMatch) {
+            querySatisfied = false;
+            break;
+          }
+        }
+        // @testpoint gloda.query.test.kConstraintStringLike
+        else if (constraintType === GlodaDatastore.kConstraintStringLike) {
+          let curIndex = 0;
+          let value = (boundName in aObj) ? aObj[boundName] : "";
+          // the attribute must be singular, we don't support arrays of strings.
+          for (let valuePart of constraintValues) {
+            if (typeof valuePart == "string") {
+              let index = value.indexOf(valuePart);
+              // if curIndex is null, we just need any match
+              // if it's not null, it must match the offset of our found match
+              if (curIndex === null) {
+                if (index == -1)
+                  querySatisfied = false;
+                else
+                  curIndex = index + valuePart.length;
+              }
+              else {
+                if (index != curIndex)
+                  querySatisfied = false;
+                else
+                  curIndex = index + valuePart.length;
+              }
+              if (!querySatisfied)
+                break;
+            }
+            else // wild!
+              curIndex = null;
+          }
+          // curIndex must be null or equal to the length of the string
+          if (querySatisfied && curIndex !== null && curIndex != value.length)
+            querySatisfied = false;
+        }
+        // @testpoint gloda.query.test.kConstraintFulltext
+        else if (constraintType === GlodaDatastore.kConstraintFulltext) {
+          // this is beyond our powers. Even if we have the fulltext content in
+          //  memory, which we may not, the tokenization and such to perform
+          //  the testing gets very complicated in the face of i18n, etc.
+          // so, let's fail if the item is not already in the collection, and
+          //  let the testing continue if it is.  (some other constraint may no
+          //  longer apply...)
+          if (!(aObj.id in this.collection._idMap))
+            querySatisfied = false;
+        }
+
+        if (!querySatisfied)
+          break;
+      }
+
+      if (querySatisfied)
+        return true;
+    }
+    return false;
+  },
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  traditional in/equals constraint.  The goal is to let them build a range
+   *  without having to know how we structure |_constraints|.
+   *
+   * @protected
+   */
+  _inConstraintHelper:
+      function gloda_query__discreteConstraintHelper(aAttrDef, aValues) {
+    let constraint =
+      [GlodaDatastore.kConstraintIn, aAttrDef].concat(aValues);
+    this._constraints.push(constraint);
+    return this;
+  },
+
+  /**
+   * Helper code for noun definitions of queryHelpers that want to build a
+   *  range.  The goal is to let them build a range without having to know how
+   *  we structure |_constraints| or requiring them to mark themselves as
+   *  continuous to get a "Range".
+   *
+   * @protected
+   */
+  _rangedConstraintHelper:
+      function gloda_query__rangedConstraintHelper(aAttrDef, aRanges) {
+    let constraint =
+      [GlodaDatastore.kConstraintRanges, aAttrDef].concat(aRanges);
+    this._constraints.push(constraint);
+    return this;
+  }
+};
+
+/**
+ * @class A query that never matches anything.
+ *
+ * Collections corresponding to this query are intentionally frozen in time and
+ *  do not want to be notified of any updates.  We need the collection to be
+ *  registered with the collection manager so that the noun instances in the
+ *  collection are always 'reachable' via the collection for as long as we might
+ *  be handing out references to the instances.  (The other way to avoid updates
+ *  would be to not register the collection, but then items might not be
+ *  reachable.)
+ * This is intended to be used in implementation details behind the gloda
+ *  abstraction barrier.  For example, the message indexer likes to be able
+ *  to represent 'ghost' and deleted messages, but these should never be exposed
+ *  to the user.  For code simplicity, it wants to be able to use the query
+ *  mechanism.  But it doesn't want updates that are effectively
+ *  nonsensical.  For example, a ghost message that is reused by message
+ *  indexing may already be present in a collection; when the collection manager
+ *  receives an itemsAdded event, a GlodaExplicitQueryClass would result in
+ *  an item added notification in that case, which would wildly not be desired.
+ */
+function GlodaNullQueryClass() {
+}
+
+GlodaNullQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for null queries.
+   */
+  options: {},
+
+  /**
+   * Provide a duck-typing way of indicating to GlodaCollectionManager that our
+   *  associated collection just doesn't want anything to change.  Our test
+   *  function is able to convey most of it, but special-casing has to happen
+   *  somewhere, so it happens here.
+   */
+  frozen: true,
+
+  /**
+   * Since our query never matches anything, it doesn't make sense to let
+   *  someone attempt to construct a boolean OR involving us.
+   *
+   * @returns null
+   */
+  or: function() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on a null query.  This method is normally used (on a normal query)
+   *  to return a collection populated by the constraints of the query.  We
+   *  match nothing, so we should return nothing.  More importantly, you are
+   *  currently doing something wrong if you try and do this, so null is
+   *  appropriate.  It may turn out that it makes sense for us to return an
+   *  empty collection in the future for sentinel value purposes, but we'll
+   *  cross that bridge when we come to it.
+   *
+   * @returns null
+   */
+  getCollection: function() {
+    return null;
+  },
+
+  /**
+   * Never matches anything.
+   *
+   * @param aObj The object someone wants us to test for relevance to our
+   *     associated collection.  But we don't care!  Not a fig!
+   * @returns false
+   */
+  test: function gloda_query_null_test(aObj) {
+    return false;
+  }
+};
+
+/**
+ * @class A query that only 'tests' for already belonging to the collection.
+ *
+ * This type of collection is useful for when you (or rather your listener)
+ *  are interested in hearing about modifications to your collection or removals
+ *  from your collection because of deletion, but do not want to be notified
+ *  about newly indexed items matching your normal query constraints.
+ *
+ * @param aCollection The collection this query belongs to.  This needs to be
+ *     passed-in here or the collection should set the attribute directly when
+ *     the query is passed in to a collection's constructor.
+ */
+function GlodaExplicitQueryClass(aCollection) {
+  this.collection = aCollection;
+}
+
+GlodaExplicitQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation, which
+   *  does not happen for explicit queries.
+   */
+  options: {},
+
+  /**
+   * Since our query is intended to only match the contents of our collection,
+   *  it doesn't make sense to let someone attempt to construct a boolean OR
+   *  involving us.
+   *
+   * @returns null
+   */
+  or: function() {
+    return null;
+  },
+
+  /**
+   * Return nothing (null) because it does not make sense to create a collection
+   *  based on an explicit query.  This method is normally used (on a normal
+   *  query) to return a collection populated by the constraints of the query.
+   *  In the case of an explicit query, we expect it will be associated with
+   *  either a hand-created collection or the results of a normal query that is
+   *  immediately converted into an explicit query.  In all likelihood, calling
+   *  this method on an instance of this type is an error, so it is helpful to
+   *  return null because people will error hard.
+   *
+   * @returns null
+   */
+  getCollection: function() {
+    return null;
+  },
+
+  /**
+   * Matches only items that are already in the collection associated with this
+   *  query (by id).
+   *
+   * @param aObj The object/item to test for already being in the associated
+   *     collection.
+   * @returns true when the object is in the associated collection, otherwise
+   *     false.
+   */
+  test: function gloda_query_explicit_test(aObj) {
+    return (aObj.id in this.collection._idMap);
+  }
+};
+
+/**
+ * @class A query that 'tests' true for everything.  Intended for debugging purposes
+ *  only.
+ */
+function GlodaWildcardQueryClass() {
+}
+
+GlodaWildcardQueryClass.prototype = {
+  /**
+   * No options; they are currently only needed for SQL query generation.
+   */
+  options: {},
+
+  // don't let people try and mess with us
+  or: function() { return null; },
+  // don't let people try and query on us (until we have a real use case for
+  //  that...)
+  getCollection: function() { return null; },
+  /**
+   * Everybody wins!
+   */
+  test: function gloda_query_explicit_test(aObj) {
+    return true;
+  }
+};
+
+/**
+ * Factory method to effectively create per-noun subclasses of GlodaQueryClass,
+ *  GlodaNullQueryClass, GlodaExplicitQueryClass, and GlodaWildcardQueryClass.
+ *  For GlodaQueryClass this allows us to add per-noun helpers.  For the others,
+ *  this is merely a means of allowing us to attach the (per-noun) nounDef to
+ *  the 'class'.
+ */
+function GlodaQueryClassFactory(aNounDef) {
+  let newQueryClass = function(aOptions) {
+    GlodaQueryClass.call(this, aOptions);
+  };
+  newQueryClass.prototype = new GlodaQueryClass();
+  newQueryClass.prototype._queryClass = newQueryClass;
+  newQueryClass.prototype._nounDef = aNounDef;
+
+  let newNullClass = function(aCollection) {
+    GlodaNullQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newNullClass.prototype = new GlodaNullQueryClass();
+  newNullClass.prototype._queryClass = newNullClass;
+  newNullClass.prototype._nounDef = aNounDef;
+
+  let newExplicitClass = function(aCollection) {
+    GlodaExplicitQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newExplicitClass.prototype = new GlodaExplicitQueryClass();
+  newExplicitClass.prototype._queryClass = newExplicitClass;
+  newExplicitClass.prototype._nounDef = aNounDef;
+
+  let newWildcardClass = function(aCollection) {
+    GlodaWildcardQueryClass.call(this);
+    this.collection = aCollection;
+  };
+  newWildcardClass.prototype = new GlodaWildcardQueryClass();
+  newWildcardClass.prototype._queryClass = newWildcardClass;
+  newWildcardClass.prototype._nounDef = aNounDef;
+
+  return [newQueryClass, newNullClass, newExplicitClass, newWildcardClass];
+}

mailnews/db/gloda/modules/suffixtree.js

@@ -0,0 +1,340 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["SuffixTree", "MultiSuffixTree"];
+
+/**
+ * Given a list of strings and a corresponding map of items that those strings
+ *  correspond to, build a suffix tree.
+ */
+function MultiSuffixTree(aStrings, aItems) {
+  if (aStrings.length != aItems.length)
+    throw new Error("Array lengths need to be the same.");
+
+  let s = '';
+  let offsetsToItems = [];
+  let lastLength = 0;
+  for (let i = 0; i < aStrings.length; i++) {
+    s += aStrings[i];
+    offsetsToItems.push(lastLength, s.length, aItems[i]);
+    lastLength = s.length;
+  }
+  
+  this._construct(s);
+  this._offsetsToItems = offsetsToItems;
+  this._numItems = aItems.length;
+}
+
+/**
+ * @constructor
+ */
+function State(aStartIndex, aEndIndex, aSuffix) {
+  this.start = aStartIndex;
+  this.end = aEndIndex;
+  this.suffix = aSuffix;
+}
+
+var dump;
+if (dump === undefined) {
+  dump = function(a) {
+    print(a.slice(0, -1));
+  };
+}
+
+/**
+ * Since objects are basically hash-tables anyways, we simply create an
+ *  attribute whose name is the first letter of the edge string.  (So, the
+ *  edge string can conceptually be a multi-letter string, but since we would
+ *  split it were there any ambiguity, it's okay to just use the single letter.)
+ *  This avoids having to update the attribute name or worry about tripping our
+ *  implementation up.
+ */
+State.prototype = {
+  get isExplicit() {
+    // our end is not inclusive...
+    return (this.end <= this.start);
+  },
+  get isImplicit() {
+    // our end is not inclusive...
+    return (this.end > this.start);
+  },
+  
+  get length() {
+    return this.end - this.start;
+  },
+  
+  toString: function State_toString() {
+    return "[Start: " + this.start + " End: " + this.end +
+           (this.suffix ? " non-null suffix]" : " null suffix]"); 
+  }
+};
+
+/**
+ * Suffix tree implemented using Ukkonen's algorithm.
+ * @constructor
+ */
+function SuffixTree(aStr) {
+  this._construct(aStr);
+}
+
+/**
+ * States are 
+ */
+SuffixTree.prototype = {
+  /**
+   * Find all items matching the provided substring.
+   */
+  findMatches: function findMatches(aSubstring) {
+    let results = [];
+    let state = this._root;
+    let index=0;
+    let end = aSubstring.length;
+    while(index < end) {
+      state = state[aSubstring[index]];
+      // bail if there was no edge
+      if (state === undefined)
+        return results;
+      // bail if the portion of the edge we traversed is not equal to that
+      //  portion of our pattern
+      let actualTraverseLength = Math.min(state.length,
+                                          end - index);
+      if (this._str.substring(state.start,
+                              state.start + actualTraverseLength) !=
+          aSubstring.substring(index, index + actualTraverseLength))
+        return results;
+      index += state.length;
+    }
+    
+    // state should now be the node which itself and all its children match...
+    // The delta is to adjust us to the offset of the last letter of our match;
+    //  the edge we traversed to get here may have found us traversing more
+    //  than we wanted.
+    // index - end captures the over-shoot of the edge traversal,
+    // index - end + 1 captures the fact that we want to find the last letter
+    //  that matched, not just the first letter beyond it
+    // However, if this state is a leaf node (end == 'infinity'), then 'end'
+    //  isn't describing an edge at all and we want to avoid accounting for it.
+    let delta;
+    /*
+    if (state.end != this._infinity)
+      //delta = index - end + 1;
+      delta = end - (index - state.length); 
+    else */
+    delta = index - state.length - end + 1;
+ 
+    this._resultGather(state, results, {}, end, delta, true);
+    return results;
+  },
+  
+  _resultGather: function resultGather(aState, aResults, aPresence,
+                                       aPatLength, aDelta, alreadyAdjusted) {
+    // find the item that this state originated from based on the state's
+    //  start character.  offsetToItem holds [string start index, string end
+    //  index (exclusive), item reference].  So we want to binary search to
+    //  find the string whose start/end index contains the state's start index.
+    let low = 0;
+    let high = this._numItems-1;
+    let mid, stringStart, stringEnd;
+    
+    let patternLast = aState.start - aDelta;
+    while (low <= high) {
+      mid = low + Math.floor((high - low) / 2); // excessive, especially with js nums
+      stringStart = this._offsetsToItems[mid*3];
+      let startDelta = stringStart - patternLast;
+      stringEnd = this._offsetsToItems[mid*3+1];
+      let endDelta = stringEnd - patternLast;
+      if (startDelta > 0)
+        high = mid - 1;
+      else if (endDelta <= 0)
+        low = mid + 1;
+      else {
+        break;
+      }
+    }
+    
+    // - The match occurred completely inside a source string.  Success.
+    // - The match spans more than one source strings, and is therefore not
+    //   a match.
+    
+    // at this point, we have located the origin string that corresponds to the
+    //  start index of this state.
+    // - The match terminated with the end of the preceding string, and does
+    //   not match us at all.  We, and potentially our children, are merely
+    //   serving as a unique terminal.
+    // - The 
+
+  let patternFirst = patternLast - (aPatLength - 1);
+
+  if (patternFirst >= stringStart) {
+    if (!(stringStart in aPresence)) {
+      aPresence[stringStart] = true;
+      aResults.push(this._offsetsToItems[mid*3+2]);
+    }
+  }
+    
+    // bail if we had it coming OR
+    // if the result terminates at/part-way through this state, meaning any
+    //  of its children are not going to be actual results, just hangers
+    //  on.
+/*
+    if (bail || (end <= aState.end)) {
+dump("  bailing! (bail was: " + bail + ")\n");
+      return;
+    }
+*/    
+    // process our children...
+    for (let key in aState) {
+      // edges have attributes of length 1...
+      if (key.length == 1) {
+        let statePrime = aState[key];
+        this._resultGather(statePrime, aResults, aPresence, aPatLength,
+                           aDelta + aState.length, //(alreadyAdjusted ? 0 : aState.length),
+                           false);
+      }
+    }
+  },
+
+  /**
+   * Given a reference 'pair' of a state and a string (may be 'empty'=explicit,
+   *  which means no work to do and we return immediately) follow that state
+   *  (and then the successive states)'s transitions until we run out of
+   *  transitions.  This happens either when we find an explicit state, or 
+   *  find ourselves partially along an edge (conceptually speaking).  In
+   *  the partial case, we return the state prior to the edge traversal.
+   * (The information about the 'edge' is contained on its target State;
+   *  we can do this because a state is only referenced by one other state.) 
+   */
+  _canonize: function canonize(aState, aStart, aEnd) {
+    if (aEnd <= aStart) {
+      return [aState, aStart];
+    }
+  
+    let statePrime;
+    // we treat an aState of null as 'bottom', which has transitions for every
+    //  letter in the alphabet to 'root'.  rather than create all those
+    //  transitions, we special-case here.
+    if (aState === null)
+      statePrime = this._root;
+    else
+      statePrime = aState[this._str[aStart]];
+    while (statePrime.length <= aEnd - aStart) { // (no 1 adjustment required)
+      aStart += statePrime.length;
+      aState = statePrime;
+      if (aStart < aEnd) {
+        statePrime = aState[this._str[aStart]];
+      }
+    }
+    return [aState, aStart]; 
+  },
+
+  /**
+   * Given a reference 'pair' whose state may or may not be explicit (and for
+   *  which we will perform the required splitting to make it explicit), test
+   *  whether it already possesses a transition corresponding to the provided
+   *  character.
+   * @return A list of: whether we had to make it explicit, the (potentially)
+   *    new explicit state.
+   */
+  _testAndSplit: function testAndSplit(aState, aStart, aEnd, aChar) {
+    if (aStart < aEnd) { // it's not explicit
+      let statePrime = aState[this._str[aStart]];
+      let length = aEnd - aStart;
+      if (aChar == this._str[statePrime.start + length]) {
+        return [true, aState];
+      }
+      else {
+        // do splitting... aState -> rState -> statePrime
+        let rState = new State(statePrime.start, statePrime.start + length);
+        aState[this._str[statePrime.start]] = rState;
+        statePrime.start += length;
+        rState[this._str[statePrime.start]] = statePrime;
+        return [false, rState];
+      }
+    }
+    else { // it's already explicit
+      if (aState === null) { // bottom case... shouldn't happen, but hey. 
+        return [true, aState];
+      }
+      return [(aChar in aState), aState];
+    }
+      
+  },
+
+  _update: function update(aState, aStart, aIndex) {
+    let oldR = this._root;
+    let textAtIndex = this._str[aIndex]; // T sub i (0-based corrected...)
+    // because of the way we store the 'end' value as a one-past form, we do
+    //  not need to subtract 1 off of aIndex.
+    let [endPoint, rState] = this._testAndSplit(aState, aStart, aIndex, //no -1
+                                                textAtIndex);
+    while (!endPoint) {
+      let rPrime = new State(aIndex, this._infinity);
+      rState[textAtIndex] = rPrime;
+      if (oldR !== this._root)
+        oldR.suffix = rState;
+      oldR = rState;
+      [aState, aStart] = this._canonize(aState.suffix, aStart, aIndex); // no -1
+      [endPoint, rState] = this._testAndSplit(aState, aStart, aIndex, // no -1
+                                              textAtIndex);
+    }
+    if (oldR !== this._root)
+      oldR.suffix = aState;
+    
+    return [aState, aStart];
+  },
+  
+  _construct: function construct(aStr) {
+    this._str = aStr;
+    // just needs to be longer than the string.
+    this._infinity = aStr.length + 1;
+    
+    //this._bottom = new State(0, -1, null);
+    this._root = new State(-1, 0, null); // null === bottom
+    let state = this._root;
+    let start = 0;
+  
+    for (let i = 0; i < aStr.length; i++) {
+      [state, start] = this._update(state, start, i); // treat as flowing -1...
+      [state, start] = this._canonize(state, start, i+1); // 1-length string
+    }
+  },
+  
+  dump: function SuffixTree_show(aState, aIndent, aKey) {
+    if (aState === undefined)
+      aState = this._root;
+    if (aIndent === undefined) {
+      aIndent = "";
+      aKey = ".";
+    }
+    
+    if (aState.isImplicit) {
+      let snip;
+      if (aState.length > 10)
+        snip = this._str.slice(aState.start,
+                           Math.min(aState.start+10, this._str.length)) + "...";
+      else
+        snip =  this._str.slice(aState.start,
+                                Math.min(aState.end, this._str.length)); 
+      dump(aIndent + aKey + ":" + snip + "(" +
+           aState.start + ":" + aState.end + ")\n");
+    }
+    else
+      dump(aIndent + aKey + ": (explicit:" + aState.start + ":" + aState.end +")\n");
+    let nextIndent = aIndent + "  ";
+    let keys = Object.keys(aState).filter(c => c.length == 1);
+    for (let key of keys) {
+      this.dump(aState[key], nextIndent, key);
+    }
+  }
+};
+MultiSuffixTree.prototype = SuffixTree.prototype;
+
+function examplar() {
+  let names = ["AndrewSmith", "AndrewJones", "MarkSmith", "BryanClark",
+               "MarthaJones", "DavidAscher", "DanMosedale", "DavidBienvenu",
+               "JanetDavis", "JosephBryant"];
+  let b = new MultiSuffixTree(names, names);
+  b.dump();
+  dump(b.findMatches("rya") + "\n");
+}

mailnews/db/gloda/modules/utils.js

@@ -0,0 +1,155 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['GlodaUtils'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource:///modules/mailServices.js");
+
+/**
+ * @namespace A holding place for logic that is not gloda-specific and should
+ *  reside elsewhere.
+ */
+var GlodaUtils = {
+
+  /**
+   * This Regexp is super-complicated and used at least in two different parts of
+   * the code, so let's expose it from one single location.
+   */
+  PART_RE: new RegExp("^[^?]+\\?(?:/;section=\\d+\\?)?(?:[^&]+&)*part=([^&]+)(?:&[^&]+)*$"),
+
+  deMime: function gloda_utils_deMime(aString) {
+    return MailServices.mimeConverter.decodeMimeHeader(aString, null, false, true);
+  },
+
+  _headerParser: MailServices.headerParser,
+
+  /**
+   * Parses an RFC 2822 list of e-mail addresses and returns an object with
+   *  4 attributes, as described below.  We will use the example of the user
+   *  passing an argument of '"Bob Smith" <bob@example.com>'.
+   *
+   * This method (by way of nsIMsgHeaderParser) takes care of decoding mime
+   *  headers, but is not aware of folder-level character set overrides.
+   *
+   * count: the number of addresses parsed. (ex: 1)
+   * addresses: a list of e-mail addresses (ex: ["bob@example.com"])
+   * names: a list of names (ex: ["Bob Smith"])
+   * fullAddresses: aka the list of name and e-mail together (ex: ['"Bob Smith"
+   *  <bob@example.com>']).
+   *
+   * This method is a convenience wrapper around nsIMsgHeaderParser.
+   */
+  parseMailAddresses: function gloda_utils_parseMailAddresses(aMailAddresses) {
+    let addresses = {}, names = {}, fullAddresses = {};
+    this._headerParser.parseHeadersWithArray(aMailAddresses, addresses,
+                                             names, fullAddresses);
+    return {names: names.value, addresses: addresses.value,
+            fullAddresses: fullAddresses.value,
+            count: names.value.length};
+  },
+
+  /**
+   * MD5 hash a string and return the hex-string result. Impl from nsICryptoHash
+   *  docs.
+   */
+  md5HashString: function gloda_utils_md5hash(aString) {
+    let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+                    createInstance(Ci.nsIScriptableUnicodeConverter);
+    let trash = {};
+    converter.charset = "UTF-8";
+    let data = converter.convertToByteArray(aString, trash);
+
+    let hasher = Cc['@mozilla.org/security/hash;1'].
+                 createInstance(Ci.nsICryptoHash);
+    hasher.init(Ci.nsICryptoHash.MD5);
+    hasher.update(data, data.length);
+    let hash = hasher.finish(false);
+
+     // return the two-digit hexadecimal code for a byte
+    function toHexString(charCode) {
+      return ("0" + charCode.toString(16)).slice(-2);
+    }
+
+    // convert the binary hash data to a hex string.
+    let hex = Object.keys(hash).map(i => toHexString(hash.charCodeAt(i)));
+    return hex.join("");
+  },
+
+  getCardForEmail: function gloda_utils_getCardForEmail(aAddress) {
+    // search through all of our local address books looking for a match.
+    let enumerator = MailServices.ab.directories;
+    let cardForEmailAddress;
+    let addrbook;
+    while (!cardForEmailAddress && enumerator.hasMoreElements())
+    {
+      addrbook = enumerator.getNext().QueryInterface(Ci.nsIAbDirectory);
+      try
+      {
+        cardForEmailAddress = addrbook.cardForEmailAddress(aAddress);
+        if (cardForEmailAddress)
+          return cardForEmailAddress;
+      } catch (ex) {}
+    }
+
+    return null;
+  },
+
+  _FORCE_GC_AFTER_NUM_HEADERS: 4096,
+  _headersSeen: 0,
+  /**
+   * As |forceGarbageCollection| says, once XPConnect sees a header, it likes
+   *  to hold onto that reference.  This method is used to track the number of
+   *  headers we have seen and force a GC when we have to.
+   *
+   * Ideally the indexer's idle-biased GC mechanism would take care of all the
+   *  GC; we are just a failsafe to make sure that our memory usage is bounded
+   *  based on the number of headers we have seen rather than just time.
+   *  Since holding onto headers can keep databases around too, this also
+   *  helps avoid keeping file handles open, etc.
+   *
+   * |forceGarbageCollection| will zero our tracking variable when a GC happens
+   *  so we are informed by the indexer's GC triggering.
+   *
+   * And of course, we don't want to trigger collections willy nilly because
+   *  they have a cost even if there is no garbage.
+   *
+   * @param aNumHeadersSeen The number of headers code has seen.  A granularity
+   *     of hundreds of messages should be fine.
+   */
+  considerHeaderBasedGC: function(aNumHeadersSeen) {
+    this._headersSeen += aNumHeadersSeen;
+    if (this._headersSeen >= this._FORCE_GC_AFTER_NUM_HEADERS)
+      this.forceGarbageCollection();
+  },
+
+  /**
+   * Force a garbage-collection sweep.  Gloda has to force garbage collection
+   *  periodically because XPConnect's XPCJSRuntime::DeferredRelease mechanism
+   *  can end up holding onto a ridiculously high number of XPConnect objects in
+   *  between normal garbage collections.  This has mainly posed a problem
+   *  because nsAutolock is a jerk in DEBUG builds in 1.9.1, but in theory this
+   *  also helps us even out our memory usage.
+   * We also are starting to do this more to try and keep the garbage collection
+   *  durations acceptable.  We intentionally avoid triggering the cycle
+   *  collector in those cases, as we do presume a non-trivial fixed cost for
+   *  cycle collection.  (And really all we want is XPConnect to not be a jerk.)
+   * This method exists mainly to centralize our GC activities and because if
+   *  we do start involving the cycle collector, that is a non-trivial block of
+   *  code to copy-and-paste all over the place (at least in a module).
+   *
+   * @param aCycleCollecting Do we need the cycle collector to run?  Currently
+   *     unused / unimplemented, but we would use
+   *     nsIDOMWindowUtils.garbageCollect() to do so.
+   */
+  forceGarbageCollection:
+      function gloda_utils_garbageCollection(aCycleCollecting) {
+    Cu.forceGC();
+    this._headersSeen = 0;
+  }
+};