Issue #1498 - Part 1: Stop using HSTS preload lists.

security/manager/ssl/nsSiteSecurityService.cpp

@@ -29,15 +29,6 @@
 #include "ScopedNSSTypes.h"
 #include "SharedCertVerifier.h"
 
-// A note about the preload list:
-// When a site specifically disables HSTS by sending a header with
-// 'max-age: 0', we keep a "knockout" value that means "we have no information
-// regarding the HSTS state of this host" (any ancestor of "this host" can still
-// influence its HSTS status via include subdomains, however).
-// This prevents the preload list from overriding the site's current
-// desired HSTS status.
-#include "nsSTSPreloadList.inc"
-
 using namespace mozilla;
 using namespace mozilla::psm;
 
@@ -393,23 +384,10 @@ nsSiteSecurityService::RemoveState(uint32_t aType, nsIURI* aURI,
   mozilla::DataStorageType storageType = isPrivate
                                          ? mozilla::DataStorage_Private
                                          : mozilla::DataStorage_Persistent;
-  // If this host is in the preload list, we have to store a knockout entry
+  SSSLOG(("SSS: removing entry for %s", hostname.get()));
-  // if it's explicitly forced to not be HSTS anymore
+  nsAutoCString storageKey;
-  if (force && GetPreloadListEntry(hostname.get())) {
+  SetStorageKey(storageKey, hostname, aType);
-    SSSLOG(("SSS: storing knockout entry for %s", hostname.get()));
+  mSiteStateStorage->Remove(storageKey, storageType);
-    SiteHSTSState siteState(0, SecurityPropertyKnockout, false);
-    nsAutoCString stateString;
-    siteState.ToString(stateString);
-    nsAutoCString storageKey;
-    SetStorageKey(storageKey, hostname, aType);
-    rv = mSiteStateStorage->Put(storageKey, stateString, storageType);
-    NS_ENSURE_SUCCESS(rv, rv);
-  } else {
-    SSSLOG(("SSS: removing entry for %s", hostname.get()));
-    nsAutoCString storageKey;
-    SetStorageKey(storageKey, hostname, aType);
-    mSiteStateStorage->Remove(storageKey, storageType);
-  }
 
   return NS_OK;
 }
@@ -969,31 +947,6 @@ nsSiteSecurityService::IsSecureURI(uint32_t aType, nsIURI* aURI,
   return IsSecureHost(aType, hostname.get(), aFlags, aCached, aResult);
 }
 
-int STSPreloadCompare(const void *key, const void *entry)
-{
-  const char *keyStr = (const char *)key;
-  const nsSTSPreload *preloadEntry = (const nsSTSPreload *)entry;
-  return strcmp(keyStr, preloadEntry->mHost);
-}
-
-// Returns the preload list entry for the given host, if it exists.
-// Only does exact host matching - the user must decide how to use the returned
-// data. May return null.
-const nsSTSPreload *
-nsSiteSecurityService::GetPreloadListEntry(const char *aHost)
-{
-  PRTime currentTime = PR_Now() + (mPreloadListTimeOffset * PR_USEC_PER_SEC);
-  if (mUsePreloadList && currentTime < gPreloadListExpirationTime) {
-    return (const nsSTSPreload *) bsearch(aHost,
-                                          kSTSPreloadList,
-                                          mozilla::ArrayLength(kSTSPreloadList),
-                                          sizeof(nsSTSPreload),
-                                          STSPreloadCompare);
-  }
-
-  return nullptr;
-}
-
 NS_IMETHODIMP
 nsSiteSecurityService::IsSecureHost(uint32_t aType, const char* aHost,
                                     uint32_t aFlags, bool* aCached,
@@ -1053,8 +1006,6 @@ nsSiteSecurityService::IsSecureHost(uint32_t aType, const char* aHost,
     return NS_OK;
   }
 
-  const nsSTSPreload *preload = nullptr;
-
   // First check the exact host. This involves first checking for an entry in
   // site security storage. If that entry exists, we don't want to check
   // in the preload list. We only want to use the stored value if it is not a
@@ -1086,21 +1037,11 @@ nsSiteSecurityService::IsSecureHost(uint32_t aType, const char* aHost,
       }
     }
 
-    // If the entry is expired and not in the preload list, we can remove it.
+    // If the entry is expired we can remove it.
-    if (expired && !GetPreloadListEntry(host.get())) {
+    if (expired) {
       mSiteStateStorage->Remove(storageKey, storageType);
     }
   }
-  // Finally look in the preloaded list. This is the exact host,
-  // so if an entry exists at all, this host is HSTS.
-  else if (GetPreloadListEntry(host.get())) {
-    SSSLOG(("%s is a preloaded STS host", host.get()));
-    *aResult = true;
-    if (aCached) {
-      *aCached = true;
-    }
-    return NS_OK;
-  }
 
   SSSLOG(("no HSTS data for %s found, walking up domain", host.get()));
   const char *subdomain;
@@ -1144,23 +1085,11 @@ nsSiteSecurityService::IsSecureHost(uint32_t aType, const char* aHost,
         }
       }
 
-      // If the entry is expired and not in the preload list, we can remove it.
+      // If the entry is expired we can remove it.
-      if (expired && !GetPreloadListEntry(subdomain)) {
+      if (expired) {
         mSiteStateStorage->Remove(storageKey, storageType);
       }
     }
-    // This is an ancestor, so if we get a match, we have to check if the
-    // preloaded entry includes subdomains.
-    else if ((preload = GetPreloadListEntry(subdomain)) != nullptr) {
-      if (preload->mIncludeSubdomains) {
-        SSSLOG(("%s is a preloaded STS host", subdomain));
-        *aResult = true;
-        if (aCached) {
-          *aCached = true;
-        }
-        break;
-      }
-    }
 
     SSSLOG(("no HSTS data for %s found, walking up domain", subdomain));
   }

security/manager/ssl/nsSiteSecurityService.h

@@ -110,8 +110,6 @@ public:
   void ToString(nsCString &aString);
 };
 
-class nsSTSPreload;
-
 class nsSiteSecurityService : public nsISiteSecurityService
                             , public nsIObserver
 {
@@ -146,8 +144,6 @@ private:
   nsresult SetHPKPState(const char* aHost, SiteHPKPState& entry, uint32_t flags,
                         bool aIsPreload);
 
-  const nsSTSPreload *GetPreloadListEntry(const char *aHost);
-
   uint64_t mMaxMaxAge;
   bool mUsePreloadList;
   bool mUseStsService;

security/manager/tools/getHSTSPreloadList.js

@@ -1,457 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-"use strict";
-
-// How to run this file:
-// 1. [obtain firefox source code]
-// 2. [build/obtain firefox binaries]
-// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
-//                                  [path to]/getHSTSPreloadlist.js \
-//                                  [absolute path to]/nsSTSPreloadlist.inc'
-// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
-//       working directory.
-
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
-var Cr = Components.results;
-
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/FileUtils.jsm");
-Cu.import("resource:///modules/XPCOMUtils.jsm");
-
-const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT";
-const OUTPUT = "nsSTSPreloadList.inc";
-const ERROR_OUTPUT = "nsSTSPreloadList.errors";
-const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
-const MAX_CONCURRENT_REQUESTS = 15;
-const MAX_RETRIES = 3;
-const REQUEST_TIMEOUT = 30 * 1000;
-const ERROR_NONE = "no error";
-const ERROR_CONNECTING_TO_HOST = "could not connect to host";
-const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
-const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
-const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
-" * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
-" * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
-"\n" +
-"/*****************************************************************************/\n" +
-"/* This is an automatically generated file. If you're not                    */\n" +
-"/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
-"/*****************************************************************************/\n" +
-"\n" +
-"#include <stdint.h>\n";
-const PREFIX = "\n" +
-"class nsSTSPreload\n" +
-"{\n" +
-"  public:\n" +
-"    const char *mHost;\n" +
-"    const bool mIncludeSubdomains;\n" +
-"};\n" +
-"\n" +
-"static const nsSTSPreload kSTSPreloadList[] = {\n";
-const POSTFIX =  "};\n";
-
-function download() {
-  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-            .createInstance(Ci.nsIXMLHttpRequest);
-  req.open("GET", SOURCE, false); // doing the request synchronously
-  try {
-    req.send();
-  }
-  catch (e) {
-    throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
-  }
-
-  if (req.status != 200) {
-    throw new Error("ERROR: problem downloading '" + SOURCE + "': status " +
-                    req.status);
-  }
-
-  var resultDecoded;
-  try {
-    resultDecoded = atob(req.responseText);
-  }
-  catch (e) {
-    throw new Error("ERROR: could not decode data as base64 from '" + SOURCE +
-                    "': " + e);
-  }
-
-  // we have to filter out '//' comments, while not mangling the json
-  var result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
-  var data = null;
-  try {
-    data = JSON.parse(result);
-  }
-  catch (e) {
-    throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
-  }
-  return data;
-}
-
-function getHosts(rawdata) {
-  var hosts = [];
-
-  if (!rawdata || !rawdata.entries) {
-    throw new Error("ERROR: source data not formatted correctly: 'entries' " +
-                    "not found");
-  }
-
-  for (let entry of rawdata.entries) {
-    if (entry.mode && entry.mode == "force-https") {
-      if (entry.name) {
-        // We trim the entry name here to avoid malformed URI exceptions when we
-        // later try to connect to the domain.
-        entry.name = entry.name.trim();
-        entry.retries = MAX_RETRIES;
-        entry.originalIncludeSubdomains = entry.include_subdomains;
-        hosts.push(entry);
-      } else {
-        throw new Error("ERROR: entry not formatted correctly: no name found");
-      }
-    }
-  }
-
-  return hosts;
-}
-
-var gSSService = Cc["@mozilla.org/ssservice;1"]
-                   .getService(Ci.nsISiteSecurityService);
-
-function processStsHeader(host, header, status, securityInfo) {
-  var maxAge = { value: 0 };
-  var includeSubdomains = { value: false };
-  var error = ERROR_NONE;
-  if (header != null && securityInfo != null) {
-    try {
-      var uri = Services.io.newURI("https://" + host.name, null, null);
-      var sslStatus = securityInfo.QueryInterface(Ci.nsISSLStatusProvider)
-                                  .SSLStatus;
-      gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
-                               uri, header, sslStatus, 0, maxAge,
-                               includeSubdomains);
-    }
-    catch (e) {
-      dump("ERROR: could not process header '" + header + "' from " +
-           host.name + ": " + e + "\n");
-      error = e;
-    }
-  } else if (status == 0) {
-    error = ERROR_CONNECTING_TO_HOST;
-  } else {
-    error = ERROR_NO_HSTS_HEADER;
-  }
-
-  let forceInclude = (host.forceInclude || host.pins == "google");
-
-  if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
-    error = ERROR_MAX_AGE_TOO_LOW;
-  }
-
-  return { name: host.name,
-           maxAge: maxAge.value,
-           includeSubdomains: includeSubdomains.value,
-           error: error,
-           retries: host.retries - 1,
-           forceInclude: forceInclude,
-           originalIncludeSubdomains: host.originalIncludeSubdomains };
-}
-
-// RedirectAndAuthStopper prevents redirects and HTTP authentication
-function RedirectAndAuthStopper() {}
-
-RedirectAndAuthStopper.prototype = {
-  // nsIChannelEventSink
-  asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
-    throw new Error(Cr.NS_ERROR_ENTITY_CHANGED);
-  },
-
-  // nsIAuthPrompt2
-  promptAuth: function(channel, level, authInfo) {
-    return false;
-  },
-
-  asyncPromptAuth: function(channel, callback, context, level, authInfo) {
-    throw new Error(Cr.NS_ERROR_NOT_IMPLEMENTED);
-  },
-
-  getInterface: function(iid) {
-    return this.QueryInterface(iid);
-  },
-
-  QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink,
-                                         Ci.nsIAuthPrompt2])
-};
-
-function getHSTSStatus(host, resultList) {
-  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-            .createInstance(Ci.nsIXMLHttpRequest);
-  var inResultList = false;
-  var uri = "https://" + host.name + "/";
-  req.open("GET", uri, true);
-  req.timeout = REQUEST_TIMEOUT;
-
-  let errorhandler = (evt) => {
-    dump(`ERROR: error making request to ${host.name} (type=${evt.type})\n`);
-    if (!inResultList) {
-      inResultList = true;
-      resultList.push(processStsHeader(host, null, req.status,
-                                       req.channel.securityInfo));
-    }
-  };
-  req.onerror = errorhandler;
-  req.ontimeout = errorhandler;
-  req.onabort = errorhandler;
-
-  req.onload = function(event) {
-    if (!inResultList) {
-      inResultList = true;
-      var header = req.getResponseHeader("strict-transport-security");
-      resultList.push(processStsHeader(host, header, req.status,
-                                       req.channel.securityInfo));
-    }
-  };
-
-  try {
-    req.channel.notificationCallbacks = new RedirectAndAuthStopper();
-    req.send();
-  }
-  catch (e) {
-    dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
-  }
-}
-
-function compareHSTSStatus(a, b) {
-  if (a.name > b.name) {
-    return 1;
-  }
-  if (a.name < b.name) {
-    return -1;
-  }
-  return 0;
-}
-
-function writeTo(string, fos) {
-  fos.write(string, string.length);
-}
-
-// Determines and returns a string representing a declaration of when this
-// preload list should no longer be used.
-// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
-function getExpirationTimeString() {
-  var now = new Date();
-  var nowMillis = now.getTime();
-  // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
-  var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
-  var expirationMicros = expirationMillis * 1000;
-  return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
-}
-
-function errorToString(status) {
-  return (status.error == ERROR_MAX_AGE_TOO_LOW
-          ? status.error + status.maxAge
-          : status.error);
-}
-
-function writeEntry(status, outputStream) {
-  let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
-                           ? status.originalIncludeSubdomains
-                           : status.includeSubdomains);
-  let includeSubdomains = (incSubdomainsBool ? "true" : "false");
-  writeTo("  { \"" + status.name + "\", " + includeSubdomains + " },\n",
-          outputStream);
-}
-
-function output(sortedStatuses, currentList) {
-  try {
-    var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
-    var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
-    var fos = FileUtils.openSafeFileOutputStream(file);
-    var eos = FileUtils.openSafeFileOutputStream(errorFile);
-    writeTo(HEADER, fos);
-    writeTo(getExpirationTimeString(), fos);
-    writeTo(PREFIX, fos);
-
-    dump("INFO: Removing error-state sites from list\n");
-    for (let status in sortedStatuses) {
-      // If we've encountered an error for this entry (other than the site not
-      // sending an HSTS header), be safe and remove it from the list
-      // (preventing stale entries from accumulating).
-      if (status.error != ERROR_NONE &&
-          status.error != ERROR_NO_HSTS_HEADER &&
-          status.error != ERROR_MAX_AGE_TOO_LOW &&
-          status.name in currentList) {
-        // dump("INFO: error connecting to or processing " + status.name + " - dropping from list\n");
-        writeTo(status.name + ": " + errorToString(status) + "\n", eos);
-        status.maxAge = 0;
-      }
-    }
-
-    // Filter out entries we aren't including.
-    dump("INFO: Filtering out entries we aren't including...\n");
-    var includedStatuses = sortedStatuses.filter(function (status) {
-      if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
-        // dump("INFO: " + status.name + " NOT ON the preload list\n");
-        writeTo(status.name + ": " + errorToString(status) + "\n", eos);
-        return false;
-      }
-
-      // dump("INFO: " + status.name + " ON the preload list\n");
-      if (status.forceInclude && status.error != ERROR_NONE) {
-        writeTo(status.name + ": " + errorToString(status) + " (error "
-                + "ignored - included regardless)\n", eos);
-      }
-      return true;
-    });
-    
-    dump("INFO: Writing statuses to file...\n");
-    for (var status of includedStatuses) {
-      writeEntry(status, fos);
-    }
-    writeTo(POSTFIX, fos);
-    FileUtils.closeSafeFileOutputStream(fos);
-    FileUtils.closeSafeFileOutputStream(eos);
-  }
-  catch (e) {
-    dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
-  }
-}
-
-function shouldRetry(response) {
-  return (response.error != ERROR_NO_HSTS_HEADER &&
-          response.error != ERROR_MAX_AGE_TOO_LOW &&
-          response.error != ERROR_NONE && response.retries > 0);
-}
-
-function getHSTSStatuses(inHosts, outStatuses) {
-  var expectedOutputLength = inHosts.length;
-  var tmpOutput = [];
-  for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
-    let host = inHosts.shift();
-    dump("spinning off request to '" + host.name + "' (remaining retries: " +
-         host.retries + ")\n");
-    getHSTSStatus(host, tmpOutput);
-  }
-
-  while (outStatuses.length != expectedOutputLength) {
-    waitForAResponse(tmpOutput);
-    var response = tmpOutput.shift();
-    dump("request to '" + response.name + "' finished\n");
-    if (shouldRetry(response)) {
-      inHosts.push(response);
-    } else {
-      outStatuses.push(response);
-    }
-
-    if (inHosts.length > 0) {
-      let host = inHosts.shift();
-      dump("spinning off request to '" + host.name + "' (remaining retries: " +
-           host.retries + ")\n");
-      getHSTSStatus(host, tmpOutput);
-    }
-  }
-}
-
-// Since all events are processed on the main thread, and since event
-// handlers are not preemptible, there shouldn't be any concurrency issues.
-function waitForAResponse(outputList) {
-  // From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
-  var threadManager = Cc["@mozilla.org/thread-manager;1"]
-                      .getService(Ci.nsIThreadManager);
-  var mainThread = threadManager.currentThread;
-  while (outputList.length == 0) {
-    mainThread.processNextEvent(true);
-  }
-}
-
-function readCurrentList(filename) {
-  var currentHosts = {};
-  var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
-  file.initWithPath(filename);
-  var fis = Cc["@mozilla.org/network/file-input-stream;1"]
-              .createInstance(Ci.nsILineInputStream);
-  fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
-  var line = {};
-  var entryRegex = /  { "([^"]*)", (true|false) },/;
-  while (fis.readLine(line)) {
-    var match = entryRegex.exec(line.value);
-    if (match) {
-      currentHosts[match[1]] = (match[2] == "true");
-    }
-  }
-  return currentHosts;
-}
-
-function combineLists(newHosts, currentHosts) {
-  for (let currentHost in currentHosts) {
-    let found = false;
-    for (let newHost of newHosts) {
-      if (newHost.name == currentHost) {
-        found = true;
-        break;
-      }
-    }
-    if (!found) {
-      newHosts.push({ name: currentHost, retries: MAX_RETRIES });
-    }
-  }
-}
-
-const TEST_ENTRIES = [
-  { name: "includesubdomains.preloaded.test", includeSubdomains: true },
-  { name: "includesubdomains2.preloaded.test", includeSubdomains: true },
-  { name: "noincludesubdomains.preloaded.test", includeSubdomains: false },
-];
-
-function deleteTestHosts(currentHosts) {
-  for (let testEntry of TEST_ENTRIES) {
-    delete currentHosts[testEntry.name];
-  }
-}
-
-function insertTestHosts(hstsStatuses) {
-  for (let testEntry of TEST_ENTRIES) {
-    hstsStatuses.push({
-      name: testEntry.name,
-      maxAge: MINIMUM_REQUIRED_MAX_AGE,
-      includeSubdomains: testEntry.includeSubdomains,
-      error: ERROR_NONE,
-      // This deliberately doesn't have a value for `retries` (because we should
-      // never attempt to connect to this host).
-      forceInclude: true,
-      originalIncludeSubdomains: testEntry.includeSubdomains,
-    });
-  }
-}
-
-// ****************************************************************************
-// This is where the action happens:
-if (arguments.length != 1) {
-  throw new Error("Usage: getHSTSPreloadList.js " +
-                  "<absolute path to current nsSTSPreloadList.inc>");
-}
-// get the current preload list
-var currentHosts = readCurrentList(arguments[0]);
-// delete any hosts we use in tests so we don't actually connect to them
-deleteTestHosts(currentHosts);
-// disable the current preload list so it won't interfere with requests we make
-Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
-// download and parse the raw json file from the Chromium source
-var rawdata = download();
-// get just the hosts with mode: "force-https"
-var hosts = getHosts(rawdata);
-// add hosts in the current list to the new list (avoiding duplicates)
-combineLists(hosts, currentHosts);
-// get the HSTS status of each host
-var hstsStatuses = [];
-getHSTSStatuses(hosts, hstsStatuses);
-// add the hosts we use in tests
-insertTestHosts(hstsStatuses);
-// sort the hosts alphabetically
-hstsStatuses.sort(compareHSTSStatus);
-// write the results to a file (this is where we filter out hosts that we
-// either couldn't connect to, didn't receive an HSTS header from, couldn't
-// parse the header, or had a header with too short a max-age)
-output(hstsStatuses, currentHosts);
-// ****************************************************************************