Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

mailnews/extensions/fts3/data/README

@@ -0,0 +1,5 @@
+The data files in this directory come from the ICU project:
+http://bugs.icu-project.org/trac/browser/icu/trunk/source/data/unidata/norm2
+
+They are intended to be consumed by the ICU project's gennorm2 script.  We have
+our own script that processes them.

mailnews/extensions/fts3/data/generate_table.py

@@ -0,0 +1,264 @@
+#!/usr/bin/python
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Mozilla Thunderbird.
+#
+# The Initial Developer of the Original Code is Mozilla Japan.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Makoto Kato <m_kato@ga2.so-net.ne.jp>
+#   Andrew Sutherland <asutherland@asutherland.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import re
+
+def printTable(f, t):
+	i = f
+	while i <= t:
+		c = array[i]
+		print "0x%04x," % c,
+		i = i + 1
+		if not i % 8:
+			print "\n\t",
+
+print '''/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is mozilla.org code.
+ *
+ * The Initial Developer of the Original Code is Mozilla Japan.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Makoto Kato <m_kato@ga2.so-net.ne.jp>
+ *   Andrew Sutherland <asutherland@asutherland.org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either of the GNU General Public License Version 2 or later (the "GPL"),
+ * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+/* THIS FILE IS GENERATED BY generate_table.py.  DON'T EDIT THIS */
+'''
+
+p = re.compile('([0-9A-F]{4,5})(?:\.\.([0-9A-F]{4,5}))?[=\>]([0-9A-F]{4,5})?')
+G_FROM = 1
+G_TO = 2
+G_FIRSTVAL = 3
+
+# Array whose value at index i is the unicode value unicode character i should
+# map to.
+array = []
+# Contents of gNormalizeTable.  We insert zero entries for sub-pages where we
+# have no mappings.  We insert references to the tables where we do have
+# such tables.
+globalTable = []
+globalTable.append("0")
+# The (exclusive) upper bound of the conversion table, unicode character-wise.
+# This is 0x10000 because our generated table is only 16-bit.  This also limits
+# the values we can map to; we perform an identity mapping for target values
+# that >= maxmapping.
+maxmapping = 0x10000
+sizePerTable = 64
+
+# Map characters that the mapping tells us to obliterate to the NUKE_CHAR
+# (such lines look like "FFF0..FFF8>")
+# We do this because if we didn't do this, we would emit these characters as
+# part of a token, which we definitely don't want.
+NUKE_CHAR = 0x20
+
+# --- load case folding table
+# entries in the file look like:
+#  0041>0061
+#  02D8>0020 0306
+#  2000..200A>0020
+#
+# The 0041 (uppercase A) tells us it lowercases to 0061 (lowercase a).
+# The 02D8 is a "spacing clone[s] of diacritic" breve which gets decomposed into
+#  a space character and a breve.  This entry/type of entry also shows up in
+#  'nfkc.txt'.
+# The 2000..200A covers a range of space characters and maps them down to the
+#  'normal' space character.
+
+file = open('nfkc_cf.txt')
+
+m = None
+line = "\n"
+i = 0x0
+while i < maxmapping and line:
+        if not m:
+                line = file.readline()
+                m = p.match(line)
+                if not m:
+                        continue
+                low = int(m.group(G_FROM), 16)
+                # if G_TO is present, use it, otherwise fallback to low
+                high = m.group(G_TO) and int(m.group(G_TO), 16) or low
+                # if G_FIRSTVAL is present use it, otherwise use NUKE_CHAR
+                val = (m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16)
+                                           or NUKE_CHAR)
+		continue
+
+
+        if i >= low and i <= high:
+		if val >= maxmapping:
+			array.append(i)
+		else:
+			array.append(val)
+                if i == high:
+                        m = None
+	else:
+		array.append(i)
+	i = i + 1
+file.close()
+
+# --- load normalization / decomposition table
+# It is important that this file gets processed second because the other table
+# will tell us about mappings from uppercase U with diaeresis to lowercase u
+# with diaeresis.  We obviously don't want that clobbering our value.  (Although
+# this would work out if we propagated backwards rather than forwards...)
+#
+# - entries in this file that we care about look like:
+#  00A0>0020
+#  0100=0041 0304
+#
+# They are found in the "Canonical and compatibility decomposition mappings"
+# section.
+#
+# The 00A0 is mapping NBSP to the normal space character.
+# The 0100 (a capital A with a bar over top of) is equivalent to 0041 (capital
+#  A) plus a 0304 (combining overline).  We do not care about the combining
+#  marks which is why our regular expression does not capture it.
+#
+#
+# - entries that we do not care about look like:
+#  0300..0314:230
+#
+# These map marks to their canonical combining class which appears to be a way
+# of specifying the precedence / order in which marks should be combined.  The
+# key thing is we don't care about them.
+file = open('nfkc.txt')
+line = file.readline()
+m = p.match(line)
+while line:
+	if not m:
+		line = file.readline()
+		m = p.match(line)
+		continue
+
+        low = int(m.group(G_FROM), 16)
+        # if G_TO is present, use it, otherwise fallback to low
+        high = m.group(G_TO) and int(m.group(G_TO), 16) or low
+        # if G_FIRSTVAL is present use it, otherwise fall back to NUKE_CHAR
+        val = m.group(G_FIRSTVAL) and int(m.group(G_FIRSTVAL), 16) or NUKE_CHAR
+        for i in range(low, high+1):
+                if i < maxmapping and val < maxmapping:
+                        array[i] = val
+	m = None
+file.close()
+
+# --- generate a normalized table to support case and accent folding
+
+i = 0
+needTerm = False;
+while i < maxmapping:
+	if not i % sizePerTable:
+		# table is empty?
+		j = i
+		while j < i + sizePerTable:
+			if array[j] != j:
+				break
+			j += 1
+
+		if j == i + sizePerTable:
+			if i:
+				globalTable.append("0")
+			i += sizePerTable
+			continue
+
+		if needTerm:
+			print "};\n"
+		globalTable.append("gNormalizeTable%04x" % i)
+		print "static const unsigned short gNormalizeTable%04x[] = {\n\t" % i,
+		print "/* U+%04x */\n\t" % i,
+		needTerm = True
+        # Decomposition does not case-fold, so we want to compensate by
+        # performing a lookup here.  Because decomposition chains can be
+        # example: 01d5, a capital U with a diaeresis and a bar. yes, really.
+        # 01d5 -> 00dc -> 0055 (U) -> 0075 (u)
+        c = array[i]
+        while c != array[c]:
+                c = array[c]
+        if c >= 0x41 and c <= 0x5a:
+                raise Exception('got an uppercase character somehow: %x => %x'
+                                % (i, c))
+	print "0x%04x," % c,
+	i = i + 1
+	if not i % 8:
+		print "\n\t",
+
+print "};\n\nstatic const unsigned short* gNormalizeTable[] = {",
+i = 0
+while i < (maxmapping / sizePerTable):
+	if not i % 4:
+		print "\n\t",
+	print globalTable[i] + ",", 
+	i += 1
+
+print '''
+};
+
+unsigned int normalize_character(const unsigned int c)
+{
+  if (c >= ''' + ('0x%x' % (maxmapping,)) + ''' || !gNormalizeTable[c >> 6])
+    return c;
+  return gNormalizeTable[c >> 6][c & 0x3f];
+}
+'''

mailnews/extensions/fts3/public/moz.build

@@ -0,0 +1,11 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPIDL_SOURCES += [
+    'nsIFts3Tokenizer.idl',
+]
+
+XPIDL_MODULE = 'fts3tok'
+

mailnews/extensions/fts3/public/nsIFts3Tokenizer.idl

@@ -0,0 +1,15 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+interface mozIStorageConnection;
+
+[scriptable, uuid(136c88ea-7003-4fe8-8835-333fd18e598c)]
+interface nsIFts3Tokenizer : nsISupports {
+    // register FTS3 tokenizer module for "mozporter" tokenizer
+    // mozporter is based by porter tokenizer with bi-gram tokenizer for CJK
+    void registerTokenizer(in mozIStorageConnection connection);
+};

mailnews/extensions/fts3/src/README.mozilla

@@ -0,0 +1,3 @@
+fts3_porter.c code is from SQLite3.
+
+This customized tokenizer "mozporter" by Mozilla supports CJK indexing using bi-gram. So you have to use bi-gram search string if you wanto to search CJK character.

mailnews/extensions/fts3/src/fts3_tokenizer.h

@@ -0,0 +1,148 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an 
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+  /*
+  ** Structure version. Should always be set to 0.
+  */
+  int iVersion;
+
+  /*
+  ** Create a new tokenizer. The values in the argv[] array are the
+  ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+  ** TABLE statement that created the fts3 table. For example, if
+  ** the following SQL is executed:
+  **
+  **   CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+  **
+  ** then argc is set to 2, and the argv[] array contains pointers
+  ** to the strings "arg1" and "arg2".
+  **
+  ** This method should return either SQLITE_OK (0), or an SQLite error 
+  ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+  ** to point at the newly created tokenizer structure. The generic
+  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** this callback. The caller will do so.
+  */
+  int (*xCreate)(
+    int argc,                           /* Size of argv array */
+    const char *const*argv,             /* Tokenizer argument strings */
+    sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
+  );
+
+  /*
+  ** Destroy an existing tokenizer. The fts3 module calls this method
+  ** exactly once for each successful call to xCreate().
+  */
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Create a tokenizer cursor to tokenize an input buffer. The caller
+  ** is responsible for ensuring that the input buffer remains valid
+  ** until the cursor is closed (using the xClose() method). 
+  */
+  int (*xOpen)(
+    sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
+    const char *pInput, int nBytes,      /* Input buffer */
+    sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
+  );
+
+  /*
+  ** Destroy an existing tokenizer cursor. The fts3 module calls this 
+  ** method exactly once for each successful call to xOpen().
+  */
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+  /*
+  ** Retrieve the next token from the tokenizer cursor pCursor. This
+  ** method should either return SQLITE_OK and set the values of the
+  ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+  ** the end of the buffer has been reached, or an SQLite error code.
+  **
+  ** *ppToken should be set to point at a buffer containing the 
+  ** normalized version of the token (i.e. after any case-folding and/or
+  ** stemming has been performed). *pnBytes should be set to the length
+  ** of this buffer in bytes. The input text that generated the token is
+  ** identified by the byte offsets returned in *piStartOffset and
+  ** *piEndOffset. *piStartOffset should be set to the index of the first
+  ** byte of the token in the input buffer. *piEndOffset should be set
+  ** to the index of the first byte just past the end of the token in
+  ** the input buffer.
+  **
+  ** The buffer *ppToken is set to point at is managed by the tokenizer
+  ** implementation. It is only required to be valid until the next call
+  ** to xNext() or xClose(). 
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xNext)(
+    sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
+    const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
+    int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
+    int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
+    int *piPosition      /* OUT: Number of tokens returned before this one */
+  );
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+#endif /* _FTS3_TOKENIZER_H_ */

mailnews/extensions/fts3/src/moz.build

@@ -0,0 +1,18 @@
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SOURCES += [
+    'fts3_porter.c',
+    'Normalize.c',
+]
+
+SOURCES += [
+    'nsFts3Tokenizer.cpp',
+    'nsGlodaRankerFunction.cpp',
+]
+
+FINAL_LIBRARY = 'mail'
+
+CXXFLAGS += CONFIG['SQLITE_CFLAGS']

mailnews/extensions/fts3/src/nsFts3Tokenizer.cpp

@@ -0,0 +1,72 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsFts3Tokenizer.h"
+
+#include "nsGlodaRankerFunction.h"
+
+#include "nsIFts3Tokenizer.h"
+#include "mozIStorageConnection.h"
+#include "mozIStorageStatement.h"
+#include "nsStringGlue.h"
+
+extern "C" void sqlite3Fts3PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule);
+
+extern "C" void glodaRankFunc(sqlite3_context *pCtx,
+                              int nVal,
+                              sqlite3_value **apVal);
+
+NS_IMPL_ISUPPORTS(nsFts3Tokenizer,nsIFts3Tokenizer)
+
+nsFts3Tokenizer::nsFts3Tokenizer()
+{
+}
+
+nsFts3Tokenizer::~nsFts3Tokenizer()
+{
+}
+
+NS_IMETHODIMP
+nsFts3Tokenizer::RegisterTokenizer(mozIStorageConnection *connection)
+{
+  nsresult rv;
+  nsCOMPtr<mozIStorageStatement> selectStatement;
+
+  // -- register the tokenizer
+  rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT fts3_tokenizer(?1, ?2)"),
+    getter_AddRefs(selectStatement));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  const sqlite3_tokenizer_module* module = nullptr;
+  sqlite3Fts3PorterTokenizerModule(&module);
+  if (!module)
+    return NS_ERROR_FAILURE;
+
+  rv = selectStatement->BindUTF8StringParameter(
+         0, NS_LITERAL_CSTRING("mozporter"));
+  NS_ENSURE_SUCCESS(rv, rv);
+  rv = selectStatement->BindBlobParameter(1,
+                                          (uint8_t*)&module,
+                                          sizeof(module));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  bool hasMore;
+  rv = selectStatement->ExecuteStep(&hasMore);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // -- register the ranking function
+  nsCOMPtr<mozIStorageFunction> func = new nsGlodaRankerFunction();
+  NS_ENSURE_TRUE(func, NS_ERROR_OUT_OF_MEMORY);
+  rv = connection->CreateFunction(
+         NS_LITERAL_CSTRING("glodaRank"),
+         -1, // variable argument support
+         func
+       );
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return rv;
+}

mailnews/extensions/fts3/src/nsFts3Tokenizer.h

@@ -0,0 +1,26 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef nsFts3Tokenizer_h__
+#define nsFts3Tokenizer_h__
+
+#include "nsCOMPtr.h"
+#include "nsIFts3Tokenizer.h"
+#include "fts3_tokenizer.h"
+
+extern const sqlite3_tokenizer_module* getWindowsTokenizer();
+
+class nsFts3Tokenizer final : public nsIFts3Tokenizer {
+public:
+    NS_DECL_ISUPPORTS
+    NS_DECL_NSIFTS3TOKENIZER
+
+    nsFts3Tokenizer();
+
+private:
+  ~nsFts3Tokenizer();
+};
+
+#endif

mailnews/extensions/fts3/src/nsFts3TokenizerCID.h

@@ -0,0 +1,16 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+ 
+#ifndef nsFts3TokenizerCID_h__
+#define nsFts3TokenizerCID_h__
+
+#define NS_FTS3TOKENIZER_CONTRACTID \
+  "@mozilla.org/messenger/fts3tokenizer;1"
+#define NS_FTS3TOKENIZER_CID			\
+{ /* a67d724d-0015-4e2e-8cad-b84775330924 */	\
+  0xa67d724d, 0x0015, 0x4e2e,			\
+  { 0x8c, 0xad, 0xb8, 0x47, 0x75, 0x33, 0x09, 0x24 }}
+
+#endif /* nsFts3TokenizerCID_h__ */

mailnews/extensions/fts3/src/nsGlodaRankerFunction.cpp

@@ -0,0 +1,145 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ *   Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Thunderbird Global Database.
+ *
+ * The Initial Developer of the Original Code is the Mozilla Foundation.
+ * Portions created by the Initial Developer are Copyright (C) 2010
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Andrew Sutherland <asutherland@asutherland.org>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include "nsGlodaRankerFunction.h"
+#include "mozIStorageValueArray.h"
+
+#include "sqlite3.h"
+
+#include "nsCOMPtr.h"
+#include "nsVariant.h"
+#include "nsComponentManagerUtils.h"
+
+#ifndef SQLITE_VERSION_NUMBER
+#error "We need SQLITE_VERSION_NUMBER defined!"
+#endif
+
+NS_IMPL_ISUPPORTS(nsGlodaRankerFunction, mozIStorageFunction)
+
+nsGlodaRankerFunction::nsGlodaRankerFunction()
+{
+}
+
+nsGlodaRankerFunction::~nsGlodaRankerFunction()
+{
+}
+
+static uint32_t COLUMN_SATURATION[] = {10, 1, 1, 1, 1};
+
+/**
+ * Our ranking function basically just multiplies the weight of the column
+ * against the number of (saturating) matches.
+ *
+ * The original code is a SQLite example ranking function, although somewhat
+ * rather modified at this point.  All SQLite code is public domain, so we are
+ * subsuming it to MPL1.1/LGPL2/GPL2.
+ */
+NS_IMETHODIMP
+nsGlodaRankerFunction::OnFunctionCall(mozIStorageValueArray *aArguments,
+                                      nsIVariant **_result)
+{
+  // all argument names are maintained from the original SQLite code.
+  uint32_t nVal;
+  nsresult rv = aArguments->GetNumEntries(&nVal);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  /* Check that the number of arguments passed to this function is correct.
+   * If not, return an error. Set aArgsData to point to the array
+   * of unsigned integer values returned by FTS3 function. Set nPhrase
+   * to contain the number of reportable phrases in the users full-text
+   * query, and nCol to the number of columns in the table.
+   */
+  if (nVal < 1)
+    return NS_ERROR_INVALID_ARG;
+
+  uint32_t lenArgsData;
+  uint32_t *aArgsData = (uint32_t *)aArguments->AsSharedBlob(0, &lenArgsData);
+
+  uint32_t nPhrase = aArgsData[0];
+  uint32_t nCol = aArgsData[1];
+  if (nVal != (1 + nCol))
+    return NS_ERROR_INVALID_ARG;
+
+  double score = 0.0;
+
+  // SQLite 3.6.22 has a different matchinfo layout than SQLite 3.6.23+
+#if SQLITE_VERSION_NUMBER <= 3006022
+  /* Iterate through each phrase in the users query. */
+  for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) {
+    // in SQ
+    for (uint32_t iCol = 0; iCol < nCol; iCol++) {
+      uint32_t nHitCount = aArgsData[2 + (iPhrase+1)*nCol + iCol];
+      double weight = aArguments->AsDouble(iCol+1);
+      if (nHitCount > 0) {
+        score += (nHitCount > COLUMN_SATURATION[iCol]) ?
+          (COLUMN_SATURATION[iCol] * weight) :
+          (nHitCount * weight);
+      }
+    }
+  }
+#else
+  /* Iterate through each phrase in the users query. */
+  for (uint32_t iPhrase = 0; iPhrase < nPhrase; iPhrase++) {
+    /* Now iterate through each column in the users query. For each column,
+    ** increment the relevancy score by:
+    **
+    **   (<hit count> / <global hit count>) * <column weight>
+    **
+    ** aPhraseinfo[] points to the start of the data for phrase iPhrase. So
+    ** the hit count and global hit counts for each column are found in 
+    ** aPhraseinfo[iCol*3] and aPhraseinfo[iCol*3+1], respectively.
+    */
+    uint32_t *aPhraseinfo = &aArgsData[2 + iPhrase*nCol*3];
+    for (uint32_t iCol = 0; iCol < nCol; iCol++) {
+      uint32_t nHitCount = aPhraseinfo[3 * iCol];
+      double weight = aArguments->AsDouble(iCol+1);
+      if (nHitCount > 0) {
+        score += (nHitCount > COLUMN_SATURATION[iCol]) ?
+          (COLUMN_SATURATION[iCol] * weight) :
+          (nHitCount * weight);
+      }
+    }
+  }
+#endif
+
+  nsCOMPtr<nsIWritableVariant> result = new nsVariant();
+   
+  rv = result->SetAsDouble(score);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  NS_ADDREF(*_result = result);
+  return NS_OK;
+}

mailnews/extensions/fts3/src/nsGlodaRankerFunction.h

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef _nsGlodaRankerFunction_h_
+#define _nsGlodaRankerFunction_h_
+
+#include "mozIStorageFunction.h"
+
+/**
+ * Basically a port of the example FTS3 ranking function to mozStorage's
+ * view of the universe.  This might get fancier at some point.
+ */
+class nsGlodaRankerFunction final : public mozIStorageFunction
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+
+  nsGlodaRankerFunction();
+private:
+  ~nsGlodaRankerFunction();
+};
+
+#endif // _nsGlodaRankerFunction_h_