From 90df4cfb70959431c75f6d11ca6c94283e52a7e1 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Fri, 14 May 2021 00:02:03 +0000 Subject: [PATCH 1/7] Revert "Issue #21 - Remove TelemertyVFS" This: - reverts commit 83ecae9ff7ee1469da1675435dbcf26d094aa1c6 - adds a failure check to Connection::GetQuotaObjects This resolves #1768. --- storage/TelemetryVFS.cpp | 879 ++++++++++++++++++ storage/moz.build | 1 + storage/mozStorageConnection.cpp | 16 + storage/mozStorageService.cpp | 20 +- storage/mozStorageService.h | 3 + storage/test/unit/test_telemetry_vfs.js | 30 + storage/test/unit/xpcshell.ini | 1 + toolkit/components/telemetry/Histograms.json | 259 ++++++ .../telemetry/histogram-whitelists.json | 78 ++ 9 files changed, 1286 insertions(+), 1 deletion(-) create mode 100644 storage/TelemetryVFS.cpp create mode 100644 storage/test/unit/test_telemetry_vfs.js diff --git a/storage/TelemetryVFS.cpp b/storage/TelemetryVFS.cpp new file mode 100644 index 0000000000..eb102a0461 --- /dev/null +++ b/storage/TelemetryVFS.cpp @@ -0,0 +1,879 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ : + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include "mozilla/Telemetry.h" +#include "mozilla/Preferences.h" +#include "sqlite3.h" +#include "nsThreadUtils.h" +#include "mozilla/dom/quota/PersistenceType.h" +#include "mozilla/dom/quota/QuotaManager.h" +#include "mozilla/dom/quota/QuotaObject.h" +#include "mozilla/IOInterposer.h" + +// The last VFS version for which this file has been updated. +#define LAST_KNOWN_VFS_VERSION 3 + +// The last io_methods version for which this file has been updated. +#define LAST_KNOWN_IOMETHODS_VERSION 3 + +/** + * This preference is a workaround to allow users/sysadmins to identify + * that the profile exists on an NFS share whose implementation + * is incompatible with SQLite's default locking implementation. + * Bug 433129 attempted to automatically identify such file-systems, + * but a reliable way was not found and it was determined that the fallback + * locking is slower than POSIX locking, so we do not want to do it by default. +*/ +#define PREF_NFS_FILESYSTEM "storage.nfs_filesystem" + +namespace { + +using namespace mozilla; +using namespace mozilla::dom::quota; + +struct Histograms { + const char *name; + const Telemetry::ID readB; + const Telemetry::ID writeB; + const Telemetry::ID readMS; + const Telemetry::ID writeMS; + const Telemetry::ID syncMS; +}; + +#define SQLITE_TELEMETRY(FILENAME, HGRAM) \ + { FILENAME, \ + Telemetry::MOZ_SQLITE_ ## HGRAM ## _READ_B, \ + Telemetry::MOZ_SQLITE_ ## HGRAM ## _WRITE_B, \ + Telemetry::MOZ_SQLITE_ ## HGRAM ## _READ_MS, \ + Telemetry::MOZ_SQLITE_ ## HGRAM ## _WRITE_MS, \ + Telemetry::MOZ_SQLITE_ ## HGRAM ## _SYNC_MS \ + } + +Histograms gHistograms[] = { + SQLITE_TELEMETRY("places.sqlite", PLACES), + SQLITE_TELEMETRY("cookies.sqlite", COOKIES), + SQLITE_TELEMETRY("webappsstore.sqlite", WEBAPPS), + SQLITE_TELEMETRY(nullptr, OTHER) +}; +#undef SQLITE_TELEMETRY + +/** RAII class for measuring how long io takes on/off main thread + */ +class IOThreadAutoTimer { +public: + /** + * IOThreadAutoTimer measures time spent in IO. Additionally it + * automatically determines whether IO is happening on the main + * thread and picks an appropriate histogram. + * + * @param id takes a telemetry histogram id. The id+1 must be an + * equivalent histogram for the main thread. Eg, MOZ_SQLITE_OPEN_MS + * is followed by MOZ_SQLITE_OPEN_MAIN_THREAD_MS. + * + * @param aOp optionally takes an IO operation to report through the + * IOInterposer. Filename will be reported as NULL, and reference will be + * either "sqlite-mainthread" or "sqlite-otherthread". + */ + explicit IOThreadAutoTimer(Telemetry::ID aId, + IOInterposeObserver::Operation aOp = IOInterposeObserver::OpNone) + : start(TimeStamp::Now()), + id(aId), + op(aOp) + { + } + + /** + * This constructor is for when we want to report an operation to + * IOInterposer but do not require a telemetry probe. + * + * @param aOp IO Operation to report through the IOInterposer. + */ + explicit IOThreadAutoTimer(IOInterposeObserver::Operation aOp) + : start(TimeStamp::Now()), + id(Telemetry::HistogramCount), + op(aOp) + { + } + + ~IOThreadAutoTimer() + { + // We don't report SQLite I/O on Windows because we have a comprehensive + // mechanism for intercepting I/O on that platform that captures a superset + // of the data captured here. + } + +private: + const TimeStamp start; + const Telemetry::ID id; + IOInterposeObserver::Operation op; +}; + +struct telemetry_file { + // Base class. Must be first + sqlite3_file base; + + // histograms pertaining to this file + Histograms *histograms; + + // quota object for this file + RefPtr quotaObject; + + // The chunk size for this file. See the documentation for + // sqlite3_file_control() and FCNTL_CHUNK_SIZE. + int fileChunkSize; + + // This contains the vfs that actually does work + sqlite3_file pReal[1]; +}; + +const char* +DatabasePathFromWALPath(const char *zWALName) +{ + /** + * Do some sketchy pointer arithmetic to find the parameter key. The WAL + * filename is in the middle of a big allocated block that contains: + * + * - Random Values + * - Main Database Path + * - \0 + * - Multiple URI components consisting of: + * - Key + * - \0 + * - Value + * - \0 + * - \0 + * - Journal Path + * - \0 + * - WAL Path (zWALName) + * - \0 + * + * Because the main database path is preceded by a random value we have to be + * careful when trying to figure out when we should terminate this loop. + */ + MOZ_ASSERT(zWALName); + + nsDependentCSubstring dbPath(zWALName, strlen(zWALName)); + + // Chop off the "-wal" suffix. + NS_NAMED_LITERAL_CSTRING(kWALSuffix, "-wal"); + MOZ_ASSERT(StringEndsWith(dbPath, kWALSuffix)); + + dbPath.Rebind(zWALName, dbPath.Length() - kWALSuffix.Length()); + MOZ_ASSERT(!dbPath.IsEmpty()); + + // We want to scan to the end of the key/value URI pairs. Skip the preceding + // null and go to the last char of the journal path. + const char* cursor = zWALName - 2; + + // Make sure we just skipped a null. + MOZ_ASSERT(!*(cursor + 1)); + + // Walk backwards over the journal path. + while (*cursor) { + cursor--; + } + + // There should be another null here. + cursor--; + MOZ_ASSERT(!*cursor); + + // Back up one more char to the last char of the previous string. It may be + // the database path or it may be a key/value URI pair. + cursor--; + +#ifdef DEBUG + { + // Verify that we just walked over the journal path. Account for the two + // nulls we just skipped. + const char *journalStart = cursor + 3; + + nsDependentCSubstring journalPath(journalStart, + strlen(journalStart)); + + // Chop off the "-journal" suffix. + NS_NAMED_LITERAL_CSTRING(kJournalSuffix, "-journal"); + MOZ_ASSERT(StringEndsWith(journalPath, kJournalSuffix)); + + journalPath.Rebind(journalStart, + journalPath.Length() - kJournalSuffix.Length()); + MOZ_ASSERT(!journalPath.IsEmpty()); + + // Make sure that the database name is a substring of the journal name. + MOZ_ASSERT(journalPath == dbPath); + } +#endif + + // Now we're either at the end of the key/value URI pairs or we're at the + // end of the database path. Carefully walk backwards one character at a + // time to do this safely without running past the beginning of the database + // path. + const char *const dbPathStart = dbPath.BeginReading(); + const char *dbPathCursor = dbPath.EndReading() - 1; + bool isDBPath = true; + + while (true) { + MOZ_ASSERT(*dbPathCursor, "dbPathCursor should never see a null char!"); + + if (isDBPath) { + isDBPath = dbPathStart <= dbPathCursor && + *dbPathCursor == *cursor && + *cursor; + } + + if (!isDBPath) { + // This isn't the database path so it must be a value. Scan past it and + // the key also. + for (size_t stringCount = 0; stringCount < 2; stringCount++) { + // Scan past the string to the preceding null character. + while (*cursor) { + cursor--; + } + + // Back up one more char to the last char of preceding string. + cursor--; + } + + // Reset and start again. + dbPathCursor = dbPath.EndReading() - 1; + isDBPath = true; + + continue; + } + + MOZ_ASSERT(isDBPath); + MOZ_ASSERT(*cursor); + + if (dbPathStart == dbPathCursor) { + // Found the full database path, we're all done. + MOZ_ASSERT(nsDependentCString(cursor) == dbPath); + return cursor; + } + + // Change the cursors and go through the loop again. + cursor--; + dbPathCursor--; + } + + MOZ_CRASH("Should never get here!"); +} + +already_AddRefed +GetQuotaObjectFromNameAndParameters(const char *zName, + const char *zURIParameterKey) +{ + MOZ_ASSERT(zName); + MOZ_ASSERT(zURIParameterKey); + + const char *persistenceType = + sqlite3_uri_parameter(zURIParameterKey, "persistenceType"); + if (!persistenceType) { + return nullptr; + } + + const char *group = sqlite3_uri_parameter(zURIParameterKey, "group"); + if (!group) { + NS_WARNING("SQLite URI had 'persistenceType' but not 'group'?!"); + return nullptr; + } + + const char *origin = sqlite3_uri_parameter(zURIParameterKey, "origin"); + if (!origin) { + NS_WARNING("SQLite URI had 'persistenceType' and 'group' but not " + "'origin'?!"); + return nullptr; + } + + QuotaManager *quotaManager = QuotaManager::Get(); + MOZ_ASSERT(quotaManager); + + return quotaManager->GetQuotaObject( + PersistenceTypeFromText(nsDependentCString(persistenceType)), + nsDependentCString(group), + nsDependentCString(origin), + NS_ConvertUTF8toUTF16(zName)); +} + +void +MaybeEstablishQuotaControl(const char *zName, + telemetry_file *pFile, + int flags) +{ + MOZ_ASSERT(pFile); + MOZ_ASSERT(!pFile->quotaObject); + + if (!(flags & (SQLITE_OPEN_URI | SQLITE_OPEN_WAL))) { + return; + } + + MOZ_ASSERT(zName); + + const char *zURIParameterKey = (flags & SQLITE_OPEN_WAL) ? + DatabasePathFromWALPath(zName) : + zName; + + MOZ_ASSERT(zURIParameterKey); + + pFile->quotaObject = + GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey); +} + +/* +** Close a telemetry_file. +*/ +int +xClose(sqlite3_file *pFile) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + { // Scope for IOThreadAutoTimer + IOThreadAutoTimer ioTimer(IOInterposeObserver::OpClose); + rc = p->pReal->pMethods->xClose(p->pReal); + } + if( rc==SQLITE_OK ){ + delete p->base.pMethods; + p->base.pMethods = nullptr; + p->quotaObject = nullptr; +#ifdef DEBUG + p->fileChunkSize = 0; +#endif + } + return rc; +} + +/* +** Read data from a telemetry_file. +*/ +int +xRead(sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst) +{ + telemetry_file *p = (telemetry_file *)pFile; + IOThreadAutoTimer ioTimer(p->histograms->readMS, IOInterposeObserver::OpRead); + int rc; + rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + // sqlite likes to read from empty files, this is normal, ignore it. + return rc; +} + +/* +** Return the current file-size of a telemetry_file. +*/ +int +xFileSize(sqlite3_file *pFile, sqlite_int64 *pSize) +{ + IOThreadAutoTimer ioTimer(IOInterposeObserver::OpStat); + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + return rc; +} + +/* +** Write data to a telemetry_file. +*/ +int +xWrite(sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst) +{ + telemetry_file *p = (telemetry_file *)pFile; + IOThreadAutoTimer ioTimer(p->histograms->writeMS, IOInterposeObserver::OpWrite); + int rc; + if (p->quotaObject) { + MOZ_ASSERT(INT64_MAX - iOfst >= iAmt); + if (!p->quotaObject->MaybeUpdateSize(iOfst + iAmt, /* aTruncate */ false)) { + return SQLITE_FULL; + } + } + rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); + if (p->quotaObject && rc != SQLITE_OK) { + NS_WARNING("xWrite failed on a quota-controlled file, attempting to " + "update its current size..."); + sqlite_int64 currentSize; + if (xFileSize(pFile, ¤tSize) == SQLITE_OK) { + p->quotaObject->MaybeUpdateSize(currentSize, /* aTruncate */ true); + } + } + return rc; +} + +/* +** Truncate a telemetry_file. +*/ +int +xTruncate(sqlite3_file *pFile, sqlite_int64 size) +{ + IOThreadAutoTimer ioTimer(Telemetry::MOZ_SQLITE_TRUNCATE_MS); + telemetry_file *p = (telemetry_file *)pFile; + int rc; + if (p->quotaObject) { + if (p->fileChunkSize > 0) { + // Round up to the smallest multiple of the chunk size that will hold all + // the data. + size = + ((size + p->fileChunkSize - 1) / p->fileChunkSize) * p->fileChunkSize; + } + if (!p->quotaObject->MaybeUpdateSize(size, /* aTruncate */ true)) { + return SQLITE_FULL; + } + } + rc = p->pReal->pMethods->xTruncate(p->pReal, size); + if (p->quotaObject) { + if (rc == SQLITE_OK) { +#ifdef DEBUG + // Make sure xTruncate set the size exactly as we calculated above. + sqlite_int64 newSize; + MOZ_ASSERT(xFileSize(pFile, &newSize) == SQLITE_OK); + MOZ_ASSERT(newSize == size); +#endif + } else { + NS_WARNING("xTruncate failed on a quota-controlled file, attempting to " + "update its current size..."); + if (xFileSize(pFile, &size) == SQLITE_OK) { + p->quotaObject->MaybeUpdateSize(size, /* aTruncate */ true); + } + } + } + return rc; +} + +/* +** Sync a telemetry_file. +*/ +int +xSync(sqlite3_file *pFile, int flags) +{ + telemetry_file *p = (telemetry_file *)pFile; + IOThreadAutoTimer ioTimer(p->histograms->syncMS, IOInterposeObserver::OpFSync); + return p->pReal->pMethods->xSync(p->pReal, flags); +} + +/* +** Lock a telemetry_file. +*/ +int +xLock(sqlite3_file *pFile, int eLock) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + return rc; +} + +/* +** Unlock a telemetry_file. +*/ +int +xUnlock(sqlite3_file *pFile, int eLock) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xUnlock(p->pReal, eLock); + return rc; +} + +/* +** Check if another file-handle holds a RESERVED lock on a telemetry_file. +*/ +int +xCheckReservedLock(sqlite3_file *pFile, int *pResOut) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); + return rc; +} + +/* +** File control method. For custom operations on a telemetry_file. +*/ +int +xFileControl(sqlite3_file *pFile, int op, void *pArg) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + // Hook SQLITE_FCNTL_SIZE_HINT for quota-controlled files and do the necessary + // work before passing to the SQLite VFS. + if (op == SQLITE_FCNTL_SIZE_HINT && p->quotaObject) { + sqlite3_int64 hintSize = *static_cast(pArg); + sqlite3_int64 currentSize; + rc = xFileSize(pFile, ¤tSize); + if (rc != SQLITE_OK) { + return rc; + } + if (hintSize > currentSize) { + rc = xTruncate(pFile, hintSize); + if (rc != SQLITE_OK) { + return rc; + } + } + } + rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + // Grab the file chunk size after the SQLite VFS has approved. + if (op == SQLITE_FCNTL_CHUNK_SIZE && rc == SQLITE_OK) { + p->fileChunkSize = *static_cast(pArg); + } +#ifdef DEBUG + if (op == SQLITE_FCNTL_SIZE_HINT && p->quotaObject && rc == SQLITE_OK) { + sqlite3_int64 hintSize = *static_cast(pArg); + if (p->fileChunkSize > 0) { + hintSize = + ((hintSize + p->fileChunkSize - 1) / p->fileChunkSize) * + p->fileChunkSize; + } + sqlite3_int64 currentSize; + MOZ_ASSERT(xFileSize(pFile, ¤tSize) == SQLITE_OK); + MOZ_ASSERT(currentSize >= hintSize); + } +#endif + return rc; +} + +/* +** Return the sector-size in bytes for a telemetry_file. +*/ +int +xSectorSize(sqlite3_file *pFile) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xSectorSize(p->pReal); + return rc; +} + +/* +** Return the device characteristic flags supported by a telemetry_file. +*/ +int +xDeviceCharacteristics(sqlite3_file *pFile) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal); + return rc; +} + +/* +** Shared-memory operations. +*/ +int +xShmLock(sqlite3_file *pFile, int ofst, int n, int flags) +{ + telemetry_file *p = (telemetry_file *)pFile; + return p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); +} + +int +xShmMap(sqlite3_file *pFile, int iRegion, int szRegion, int isWrite, void volatile **pp) +{ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp); + return rc; +} + +void +xShmBarrier(sqlite3_file *pFile){ + telemetry_file *p = (telemetry_file *)pFile; + p->pReal->pMethods->xShmBarrier(p->pReal); +} + +int +xShmUnmap(sqlite3_file *pFile, int delFlag){ + telemetry_file *p = (telemetry_file *)pFile; + int rc; + rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag); + return rc; +} + +int +xFetch(sqlite3_file *pFile, sqlite3_int64 iOff, int iAmt, void **pp) +{ + telemetry_file *p = (telemetry_file *)pFile; + MOZ_ASSERT(p->pReal->pMethods->iVersion >= 3); + return p->pReal->pMethods->xFetch(p->pReal, iOff, iAmt, pp); +} + +int +xUnfetch(sqlite3_file *pFile, sqlite3_int64 iOff, void *pResOut) +{ + telemetry_file *p = (telemetry_file *)pFile; + MOZ_ASSERT(p->pReal->pMethods->iVersion >= 3); + return p->pReal->pMethods->xUnfetch(p->pReal, iOff, pResOut); +} + +int +xOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* pFile, + int flags, int *pOutFlags) +{ + IOThreadAutoTimer ioTimer(Telemetry::MOZ_SQLITE_OPEN_MS, + IOInterposeObserver::OpCreateOrOpen); + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + int rc; + telemetry_file *p = (telemetry_file *)pFile; + Histograms *h = nullptr; + // check if the filename is one we are probing for + for(size_t i = 0;i < sizeof(gHistograms)/sizeof(gHistograms[0]);i++) { + h = &gHistograms[i]; + // last probe is the fallback probe + if (!h->name) + break; + if (!zName) + continue; + const char *match = strstr(zName, h->name); + if (!match) + continue; + char c = match[strlen(h->name)]; + // include -wal/-journal too + if (!c || c == '-') + break; + } + p->histograms = h; + + MaybeEstablishQuotaControl(zName, p, flags); + + rc = orig_vfs->xOpen(orig_vfs, zName, p->pReal, flags, pOutFlags); + if( rc != SQLITE_OK ) + return rc; + if( p->pReal->pMethods ){ + sqlite3_io_methods *pNew = new sqlite3_io_methods; + const sqlite3_io_methods *pSub = p->pReal->pMethods; + memset(pNew, 0, sizeof(*pNew)); + // If the io_methods version is higher than the last known one, you should + // update this VFS adding appropriate IO methods for any methods added in + // the version change. + pNew->iVersion = pSub->iVersion; + MOZ_ASSERT(pNew->iVersion <= LAST_KNOWN_IOMETHODS_VERSION); + pNew->xClose = xClose; + pNew->xRead = xRead; + pNew->xWrite = xWrite; + pNew->xTruncate = xTruncate; + pNew->xSync = xSync; + pNew->xFileSize = xFileSize; + pNew->xLock = xLock; + pNew->xUnlock = xUnlock; + pNew->xCheckReservedLock = xCheckReservedLock; + pNew->xFileControl = xFileControl; + pNew->xSectorSize = xSectorSize; + pNew->xDeviceCharacteristics = xDeviceCharacteristics; + if (pNew->iVersion >= 2) { + // Methods added in version 2. + pNew->xShmMap = pSub->xShmMap ? xShmMap : 0; + pNew->xShmLock = pSub->xShmLock ? xShmLock : 0; + pNew->xShmBarrier = pSub->xShmBarrier ? xShmBarrier : 0; + pNew->xShmUnmap = pSub->xShmUnmap ? xShmUnmap : 0; + } + if (pNew->iVersion >= 3) { + // Methods added in version 3. + // SQLite 3.7.17 calls these methods without checking for nullptr first, + // so we always define them. Verify that we're not going to call + // nullptrs, though. + MOZ_ASSERT(pSub->xFetch); + pNew->xFetch = xFetch; + MOZ_ASSERT(pSub->xUnfetch); + pNew->xUnfetch = xUnfetch; + } + pFile->pMethods = pNew; + } + return rc; +} + +int +xDelete(sqlite3_vfs* vfs, const char *zName, int syncDir) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + int rc; + RefPtr quotaObject; + + if (StringEndsWith(nsDependentCString(zName), NS_LITERAL_CSTRING("-wal"))) { + const char *zURIParameterKey = DatabasePathFromWALPath(zName); + MOZ_ASSERT(zURIParameterKey); + + quotaObject = GetQuotaObjectFromNameAndParameters(zName, zURIParameterKey); + } + + rc = orig_vfs->xDelete(orig_vfs, zName, syncDir); + if (rc == SQLITE_OK && quotaObject) { + MOZ_ALWAYS_TRUE(quotaObject->MaybeUpdateSize(0, /* aTruncate */ true)); + } + + return rc; +} + +int +xAccess(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xAccess(orig_vfs, zName, flags, pResOut); +} + +int +xFullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xFullPathname(orig_vfs, zName, nOut, zOut); +} + +void* +xDlOpen(sqlite3_vfs *vfs, const char *zFilename) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xDlOpen(orig_vfs, zFilename); +} + +void +xDlError(sqlite3_vfs *vfs, int nByte, char *zErrMsg) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + orig_vfs->xDlError(orig_vfs, nByte, zErrMsg); +} + +void +(*xDlSym(sqlite3_vfs *vfs, void *pHdle, const char *zSym))(void){ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xDlSym(orig_vfs, pHdle, zSym); +} + +void +xDlClose(sqlite3_vfs *vfs, void *pHandle) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + orig_vfs->xDlClose(orig_vfs, pHandle); +} + +int +xRandomness(sqlite3_vfs *vfs, int nByte, char *zOut) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xRandomness(orig_vfs, nByte, zOut); +} + +int +xSleep(sqlite3_vfs *vfs, int microseconds) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xSleep(orig_vfs, microseconds); +} + +int +xCurrentTime(sqlite3_vfs *vfs, double *prNow) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xCurrentTime(orig_vfs, prNow); +} + +int +xGetLastError(sqlite3_vfs *vfs, int nBuf, char *zBuf) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xGetLastError(orig_vfs, nBuf, zBuf); +} + +int +xCurrentTimeInt64(sqlite3_vfs *vfs, sqlite3_int64 *piNow) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xCurrentTimeInt64(orig_vfs, piNow); +} + +static +int +xSetSystemCall(sqlite3_vfs *vfs, const char *zName, sqlite3_syscall_ptr pFunc) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xSetSystemCall(orig_vfs, zName, pFunc); +} + +static +sqlite3_syscall_ptr +xGetSystemCall(sqlite3_vfs *vfs, const char *zName) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xGetSystemCall(orig_vfs, zName); +} + +static +const char * +xNextSystemCall(sqlite3_vfs *vfs, const char *zName) +{ + sqlite3_vfs *orig_vfs = static_cast(vfs->pAppData); + return orig_vfs->xNextSystemCall(orig_vfs, zName); +} + +} // namespace + +namespace mozilla { +namespace storage { + +sqlite3_vfs* ConstructTelemetryVFS() +{ +#if defined(XP_WIN) +#define EXPECTED_VFS "win32" +#define EXPECTED_VFS_NFS "win32" +#else +#define EXPECTED_VFS "unix" +#define EXPECTED_VFS_NFS "unix-excl" +#endif + + bool expected_vfs; + sqlite3_vfs *vfs; + if (Preferences::GetBool(PREF_NFS_FILESYSTEM)) { + vfs = sqlite3_vfs_find(EXPECTED_VFS_NFS); + expected_vfs = (vfs != nullptr); + } + else { + vfs = sqlite3_vfs_find(nullptr); + expected_vfs = vfs->zName && !strcmp(vfs->zName, EXPECTED_VFS); + } + if (!expected_vfs) { + return nullptr; + } + + sqlite3_vfs *tvfs = new ::sqlite3_vfs; + memset(tvfs, 0, sizeof(::sqlite3_vfs)); + // If the VFS version is higher than the last known one, you should update + // this VFS adding appropriate methods for any methods added in the version + // change. + tvfs->iVersion = vfs->iVersion; + MOZ_ASSERT(vfs->iVersion <= LAST_KNOWN_VFS_VERSION); + tvfs->szOsFile = sizeof(telemetry_file) - sizeof(sqlite3_file) + vfs->szOsFile; + tvfs->mxPathname = vfs->mxPathname; + tvfs->zName = "telemetry-vfs"; + tvfs->pAppData = vfs; + tvfs->xOpen = xOpen; + tvfs->xDelete = xDelete; + tvfs->xAccess = xAccess; + tvfs->xFullPathname = xFullPathname; + tvfs->xDlOpen = xDlOpen; + tvfs->xDlError = xDlError; + tvfs->xDlSym = xDlSym; + tvfs->xDlClose = xDlClose; + tvfs->xRandomness = xRandomness; + tvfs->xSleep = xSleep; + tvfs->xCurrentTime = xCurrentTime; + tvfs->xGetLastError = xGetLastError; + if (tvfs->iVersion >= 2) { + // Methods added in version 2. + tvfs->xCurrentTimeInt64 = xCurrentTimeInt64; + } + if (tvfs->iVersion >= 3) { + // Methods added in version 3. + tvfs->xSetSystemCall = xSetSystemCall; + tvfs->xGetSystemCall = xGetSystemCall; + tvfs->xNextSystemCall = xNextSystemCall; + } + return tvfs; +} + +already_AddRefed +GetQuotaObjectForFile(sqlite3_file *pFile) +{ + MOZ_ASSERT(pFile); + + telemetry_file *p = (telemetry_file *)pFile; + RefPtr result = p->quotaObject; + return result.forget(); +} + +} // namespace storage +} // namespace mozilla diff --git a/storage/moz.build b/storage/moz.build index c155c64257..d1ad9229f1 100644 --- a/storage/moz.build +++ b/storage/moz.build @@ -70,6 +70,7 @@ UNIFIED_SOURCES += [ 'mozStorageStatementRow.cpp', 'SQLCollations.cpp', 'StorageBaseStatementInternal.cpp', + 'TelemetryVFS.cpp', 'VacuumManager.cpp', ] diff --git a/storage/mozStorageConnection.cpp b/storage/mozStorageConnection.cpp index 40a71c28b5..22842ed9e9 100644 --- a/storage/mozStorageConnection.cpp +++ b/storage/mozStorageConnection.cpp @@ -1991,6 +1991,10 @@ Connection::EnableModule(const nsACString& aModuleName) return NS_ERROR_FAILURE; } +// Implemented in TelemetryVFS.cpp +already_AddRefed +GetQuotaObjectForFile(sqlite3_file *pFile); + NS_IMETHODIMP Connection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject, QuotaObject** aJournalQuotaObject) @@ -2011,6 +2015,11 @@ Connection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject, return convertResultCode(srv); } + RefPtr databaseQuotaObject = GetQuotaObjectForFile(file); + if (NS_WARN_IF(!databaseQuotaObject)) { + return NS_ERROR_FAILURE; + } + srv = ::sqlite3_file_control(mDBConn, nullptr, SQLITE_FCNTL_JOURNAL_POINTER, @@ -2019,6 +2028,13 @@ Connection::GetQuotaObjects(QuotaObject** aDatabaseQuotaObject, return convertResultCode(srv); } + RefPtr journalQuotaObject = GetQuotaObjectForFile(file); + if (NS_WARN_IF(!journalQuotaObject)) { + return NS_ERROR_FAILURE; + } + + databaseQuotaObject.forget(aDatabaseQuotaObject); + journalQuotaObject.forget(aJournalQuotaObject); return NS_OK; } diff --git a/storage/mozStorageService.cpp b/storage/mozStorageService.cpp index 56c10a4d04..8c6f65232c 100644 --- a/storage/mozStorageService.cpp +++ b/storage/mozStorageService.cpp @@ -253,6 +253,7 @@ int32_t Service::sDefaultPageSize = PREF_TS_PAGESIZE_DEFAULT; Service::Service() : mMutex("Service::mMutex") +, mSqliteVFS(nullptr) , mRegistrationMutex("Service::mRegistrationMutex") , mConnections() { @@ -263,9 +264,13 @@ Service::~Service() mozilla::UnregisterWeakMemoryReporter(this); mozilla::UnregisterStorageSQLiteDistinguishedAmount(); + int rc = sqlite3_vfs_unregister(mSqliteVFS); + if (rc != SQLITE_OK) + NS_WARNING("Failed to unregister sqlite vfs wrapper."); + // Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but // there is nothing actionable we can do in that case. - int rc = ::sqlite3_shutdown(); + rc = ::sqlite3_shutdown(); if (rc != SQLITE_OK) NS_WARNING("sqlite3 did not shutdown cleanly."); @@ -273,6 +278,8 @@ Service::~Service() NS_ASSERTION(shutdownObserved, "Shutdown was not observed!"); gService = nullptr; + delete mSqliteVFS; + mSqliteVFS = nullptr; } void @@ -365,6 +372,8 @@ Service::shutdown() NS_IF_RELEASE(sXPConnect); } +sqlite3_vfs *ConstructTelemetryVFS(); + #ifdef MOZ_STORAGE_MEMORY namespace { @@ -472,6 +481,15 @@ Service::initialize() if (rc != SQLITE_OK) return convertResultCode(rc); + mSqliteVFS = ConstructTelemetryVFS(); + if (mSqliteVFS) { + rc = sqlite3_vfs_register(mSqliteVFS, 1); + if (rc != SQLITE_OK) + return convertResultCode(rc); + } else { + NS_WARNING("Failed to register telemetry VFS"); + } + // Register for xpcom-shutdown so we can cleanup after ourselves. The // observer service can only be used on the main thread. nsCOMPtr os = mozilla::services::GetObserverService(); diff --git a/storage/mozStorageService.h b/storage/mozStorageService.h index d3c1d74d8b..effd330b16 100644 --- a/storage/mozStorageService.h +++ b/storage/mozStorageService.h @@ -19,6 +19,7 @@ class nsIMemoryReporter; class nsIXPConnect; +struct sqlite3_vfs; namespace mozilla { namespace storage { @@ -135,6 +136,8 @@ private: * synchronizing access to mLocaleCollation. */ Mutex mMutex; + + sqlite3_vfs *mSqliteVFS; /** * Protects mConnections. diff --git a/storage/test/unit/test_telemetry_vfs.js b/storage/test/unit/test_telemetry_vfs.js new file mode 100644 index 0000000000..0822fe3e7e --- /dev/null +++ b/storage/test/unit/test_telemetry_vfs.js @@ -0,0 +1,30 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ + */ + +// Make sure that there are telemetry entries created by sqlite io + +function run_sql(d, sql) { + var stmt = d.createStatement(sql); + stmt.execute(); + stmt.finalize(); +} + +function new_file(name) +{ + var file = dirSvc.get("ProfD", Ci.nsIFile); + file.append(name); + return file; +} +function run_test() +{ + const Telemetry = Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry); + let read_hgram = Telemetry.getHistogramById("MOZ_SQLITE_OTHER_READ_B"); + let old_sum = read_hgram.snapshot().sum; + const file = new_file("telemetry.sqlite"); + var d = getDatabase(file); + run_sql(d, "CREATE TABLE bloat(data varchar)"); + run_sql(d, "DROP TABLE bloat"); + do_check_true(read_hgram.snapshot().sum > old_sum); +} + diff --git a/storage/test/unit/xpcshell.ini b/storage/test/unit/xpcshell.ini index f9075a595d..e93c7d5b98 100644 --- a/storage/test/unit/xpcshell.ini +++ b/storage/test/unit/xpcshell.ini @@ -41,5 +41,6 @@ fail-if = os == "android" [test_storage_value_array.js] [test_unicode.js] [test_vacuum.js] +[test_telemetry_vfs.js] # Bug 676981: test fails consistently on Android # fail-if = os == "android" diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json index deae9e8de8..d4974cf5aa 100644 --- a/toolkit/components/telemetry/Histograms.json +++ b/toolkit/components/telemetry/Histograms.json @@ -3696,6 +3696,265 @@ "n_values": 30, "description": "Flash object instances count on page" }, + "MOZ_SQLITE_OPEN_MS": { + "expires_in_version": "default", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite open() (ms)" + }, + "MOZ_SQLITE_OPEN_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite open() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_TRUNCATE_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite truncate() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_TRUNCATE_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite truncate() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_READ_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_READ_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_PLACES_READ_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_PLACES_READ_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_OPEN_READAHEAD_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on cookie DB open with readahead (ms)" + }, + "MOZ_SQLITE_COOKIES_READ_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_READ_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_READ_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_READ_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite read() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_WRITE_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_WRITE_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_PLACES_WRITE_MS": { + "expires_in_version": "default", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms)" + }, + "MOZ_SQLITE_PLACES_WRITE_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_WRITE_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_WRITE_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_WRITE_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_WRITE_MAIN_THREAD_MS": { + "expires_in_version": "40", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite write() (ms) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_SYNC_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_OTHER_SYNC_MAIN_THREAD_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_PLACES_SYNC_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_PLACES_SYNC_MAIN_THREAD_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_COOKIES_SYNC_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_COOKIES_SYNC_MAIN_THREAD_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_WEBAPPS_SYNC_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_WEBAPPS_SYNC_MAIN_THREAD_MS": { + "expires_in_version": "never", + "kind": "exponential", + "high": 3000, + "n_buckets": 10, + "description": "Time spent on SQLite fsync() (ms)" + }, + "MOZ_SQLITE_OTHER_READ_B": { + "expires_in_version": "default", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite read() (bytes)" + }, + "MOZ_SQLITE_PLACES_READ_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite read() (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_READ_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite read() (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_READ_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite read() (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_PLACES_WRITE_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite write (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_COOKIES_WRITE_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite write (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_WEBAPPS_WRITE_B": { + "expires_in_version": "40", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite write (bytes) *** No longer needed (bug 1156565). Delete histogram and accumulation code! ***" + }, + "MOZ_SQLITE_OTHER_WRITE_B": { + "expires_in_version": "default", + "kind": "linear", + "high": 32768, + "n_buckets": 3, + "description": "SQLite write (bytes)" + }, "MOZ_STORAGE_ASYNC_REQUESTS_MS": { "alert_emails": ["perf-telemetry-alerts@mozilla.com"], "expires_in_version": "40", diff --git a/toolkit/components/telemetry/histogram-whitelists.json b/toolkit/components/telemetry/histogram-whitelists.json index 0aa9061d74..722230ac9d 100644 --- a/toolkit/components/telemetry/histogram-whitelists.json +++ b/toolkit/components/telemetry/histogram-whitelists.json @@ -390,6 +390,43 @@ "MEDIA_WMF_DECODE_ERROR", "MIXED_CONTENT_PAGE_LOAD", "MIXED_CONTENT_UNBLOCK_COUNTER", + "MOZ_SQLITE_COOKIES_OPEN_READAHEAD_MS", + "MOZ_SQLITE_COOKIES_READ_B", + "MOZ_SQLITE_COOKIES_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_READ_MS", + "MOZ_SQLITE_COOKIES_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_SYNC_MS", + "MOZ_SQLITE_COOKIES_WRITE_B", + "MOZ_SQLITE_COOKIES_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_WRITE_MS", + "MOZ_SQLITE_OPEN_MAIN_THREAD_MS", + "MOZ_SQLITE_OPEN_MS", + "MOZ_SQLITE_OTHER_READ_B", + "MOZ_SQLITE_OTHER_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_READ_MS", + "MOZ_SQLITE_OTHER_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_SYNC_MS", + "MOZ_SQLITE_OTHER_WRITE_B", + "MOZ_SQLITE_OTHER_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_WRITE_MS", + "MOZ_SQLITE_PLACES_READ_B", + "MOZ_SQLITE_PLACES_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_READ_MS", + "MOZ_SQLITE_PLACES_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_SYNC_MS", + "MOZ_SQLITE_PLACES_WRITE_B", + "MOZ_SQLITE_PLACES_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_WRITE_MS", + "MOZ_SQLITE_TRUNCATE_MAIN_THREAD_MS", + "MOZ_SQLITE_TRUNCATE_MS", + "MOZ_SQLITE_WEBAPPS_READ_B", + "MOZ_SQLITE_WEBAPPS_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_READ_MS", + "MOZ_SQLITE_WEBAPPS_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_SYNC_MS", + "MOZ_SQLITE_WEBAPPS_WRITE_B", + "MOZ_SQLITE_WEBAPPS_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_WRITE_MS", "NETWORK_CACHE_METADATA_FIRST_READ_SIZE", "NETWORK_CACHE_METADATA_FIRST_READ_TIME_MS", "NETWORK_CACHE_METADATA_SECOND_READ_TIME_MS", @@ -1191,6 +1228,43 @@ "MIXED_CONTENT_HSTS", "MIXED_CONTENT_PAGE_LOAD", "MIXED_CONTENT_UNBLOCK_COUNTER", + "MOZ_SQLITE_COOKIES_OPEN_READAHEAD_MS", + "MOZ_SQLITE_COOKIES_READ_B", + "MOZ_SQLITE_COOKIES_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_READ_MS", + "MOZ_SQLITE_COOKIES_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_SYNC_MS", + "MOZ_SQLITE_COOKIES_WRITE_B", + "MOZ_SQLITE_COOKIES_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_COOKIES_WRITE_MS", + "MOZ_SQLITE_OPEN_MAIN_THREAD_MS", + "MOZ_SQLITE_OPEN_MS", + "MOZ_SQLITE_OTHER_READ_B", + "MOZ_SQLITE_OTHER_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_READ_MS", + "MOZ_SQLITE_OTHER_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_SYNC_MS", + "MOZ_SQLITE_OTHER_WRITE_B", + "MOZ_SQLITE_OTHER_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_OTHER_WRITE_MS", + "MOZ_SQLITE_PLACES_READ_B", + "MOZ_SQLITE_PLACES_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_READ_MS", + "MOZ_SQLITE_PLACES_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_SYNC_MS", + "MOZ_SQLITE_PLACES_WRITE_B", + "MOZ_SQLITE_PLACES_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_PLACES_WRITE_MS", + "MOZ_SQLITE_TRUNCATE_MAIN_THREAD_MS", + "MOZ_SQLITE_TRUNCATE_MS", + "MOZ_SQLITE_WEBAPPS_READ_B", + "MOZ_SQLITE_WEBAPPS_READ_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_READ_MS", + "MOZ_SQLITE_WEBAPPS_SYNC_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_SYNC_MS", + "MOZ_SQLITE_WEBAPPS_WRITE_B", + "MOZ_SQLITE_WEBAPPS_WRITE_MAIN_THREAD_MS", + "MOZ_SQLITE_WEBAPPS_WRITE_MS", "MOZ_STORAGE_ASYNC_REQUESTS_MS", "MOZ_STORAGE_ASYNC_REQUESTS_SUCCESS", "NETWORK_CACHE_METADATA_FIRST_READ_SIZE", @@ -1807,6 +1881,7 @@ "WEAVE_CONFIGURED", "NEWTAB_PAGE_ENABLED", "GRADIENT_DURATION", + "MOZ_SQLITE_OPEN_MS", "SHOULD_TRANSLATION_UI_APPEAR", "NEWTAB_PAGE_LIFE_SPAN", "FX_TOTAL_TOP_VISITS", @@ -1823,6 +1898,7 @@ "FX_NEW_WINDOW_MS", "PDF_VIEWER_TIME_TO_VIEW_MS", "SSL_OCSP_MAY_FETCH", + "MOZ_SQLITE_OTHER_READ_B", "CHECK_JAVA_ENABLED", "TRANSLATION_OPPORTUNITIES", "FX_SESSION_RESTORE_CONTENT_COLLECT_DATA_LONGEST_OP_MS", @@ -1848,9 +1924,11 @@ "FX_SESSION_RESTORE_DOM_STORAGE_SIZE_ESTIMATE_CHARS", "DATA_STORAGE_ENTRIES", "TRANSLATED_PAGES_BY_LANGUAGE", + "MOZ_SQLITE_OTHER_WRITE_B", "LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS", "SSL_CERT_VERIFICATION_ERRORS", "FX_SESSION_RESTORE_NUMBER_OF_WINDOWS_RESTORED", + "MOZ_SQLITE_PLACES_WRITE_MS", "FX_THUMBNAILS_BG_CAPTURE_CANVAS_DRAW_TIME_MS", "FX_SESSION_RESTORE_STARTUP_INIT_SESSION_MS", "FX_SESSION_RESTORE_WRITE_FILE_MS", From 8744911c18f9a3e4391685abcf8fb0abcc40ffd4 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Fri, 14 May 2021 12:25:57 +0000 Subject: [PATCH 2/7] Issue #1772 - Stop using legacy code page conversion for file paths on Linux. OS.File already only supports UTF-8 paths on non-Windows systems, so this change makes our different ways of accessing file paths consistent with each other. This should prevent unexpected crashes in glibc that expect UTF-8. This resolves #1772. --- xpcom/build/XPCOMInit.cpp | 7 - xpcom/io/nsNativeCharsetUtils.cpp | 875 +----------------------------- xpcom/io/nsNativeCharsetUtils.h | 24 +- 3 files changed, 15 insertions(+), 891 deletions(-) diff --git a/xpcom/build/XPCOMInit.cpp b/xpcom/build/XPCOMInit.cpp index 3e2007c1f7..2deb31a006 100644 --- a/xpcom/build/XPCOMInit.cpp +++ b/xpcom/build/XPCOMInit.cpp @@ -575,10 +575,6 @@ NS_InitXPCOM2(nsIServiceManager** aResult, setlocale(LC_ALL, ""); } -#if defined(XP_UNIX) - NS_StartupNativeCharsetUtils(); -#endif - NS_StartupLocalFile(); StartupSpecialSystemDirectory(); @@ -1023,9 +1019,6 @@ ShutdownXPCOM(nsIServiceManager* aServMgr) // Shutdown nsLocalFile string conversion NS_ShutdownLocalFile(); -#ifdef XP_UNIX - NS_ShutdownNativeCharsetUtils(); -#endif // Shutdown xpcom. This will release all loaders and cause others holding // a refcount to the component manager to release it. diff --git a/xpcom/io/nsNativeCharsetUtils.cpp b/xpcom/io/nsNativeCharsetUtils.cpp index 927e8cd591..3ed20d7f04 100644 --- a/xpcom/io/nsNativeCharsetUtils.cpp +++ b/xpcom/io/nsNativeCharsetUtils.cpp @@ -6,864 +6,9 @@ #include "xpcom-private.h" //----------------------------------------------------------------------------- -// XP_UNIX +// Windows //----------------------------------------------------------------------------- -#if defined(XP_UNIX) - -#include // mbtowc, wctomb -#include // setlocale -#include "mozilla/Mutex.h" -#include "nscore.h" -#include "nsAString.h" -#include "nsReadableUtils.h" - -using namespace mozilla; - -// -// choose a conversion library. we used to use mbrtowc/wcrtomb under Linux, -// but that doesn't work for non-BMP characters whether we use '-fshort-wchar' -// or not (see bug 206811 and -// news://news.mozilla.org:119/bajml3$fvr1@ripley.netscape.com). we now use -// iconv for all platforms where nltypes.h and nllanginfo.h are present -// along with iconv. -// -#if defined(HAVE_ICONV) && defined(HAVE_NL_TYPES_H) && defined(HAVE_LANGINFO_CODESET) -#define USE_ICONV 1 -#else -#define USE_STDCONV 1 -#endif - -static void -isolatin1_to_utf16(const char** aInput, uint32_t* aInputLeft, - char16_t** aOutput, uint32_t* aOutputLeft) -{ - while (*aInputLeft && *aOutputLeft) { - **aOutput = (unsigned char)** aInput; - (*aInput)++; - (*aInputLeft)--; - (*aOutput)++; - (*aOutputLeft)--; - } -} - -static void -utf16_to_isolatin1(const char16_t** aInput, uint32_t* aInputLeft, - char** aOutput, uint32_t* aOutputLeft) -{ - while (*aInputLeft && *aOutputLeft) { - **aOutput = (unsigned char)**aInput; - (*aInput)++; - (*aInputLeft)--; - (*aOutput)++; - (*aOutputLeft)--; - } -} - -//----------------------------------------------------------------------------- -// conversion using iconv -//----------------------------------------------------------------------------- -#if defined(USE_ICONV) -#include // CODESET -#include // nl_langinfo -#include // iconv_open, iconv, iconv_close -#include -#include "plstr.h" - -#if defined(HAVE_ICONV_WITH_CONST_INPUT) -#define ICONV_INPUT(x) (x) -#else -#define ICONV_INPUT(x) ((char **)x) -#endif - -// solaris definitely needs this, but we'll enable it by default -// just in case... but we know for sure that iconv(3) in glibc -// doesn't need this. -#if !defined(__GLIBC__) -#define ENABLE_UTF8_FALLBACK_SUPPORT -#endif - -#define INVALID_ICONV_T ((iconv_t)-1) - -static inline size_t -xp_iconv(iconv_t converter, - const char** aInput, size_t* aInputLeft, - char** aOutput, size_t* aOutputLeft) -{ - size_t res, outputAvail = *aOutputLeft; - res = iconv(converter, ICONV_INPUT(aInput), aInputLeft, aOutput, aOutputLeft); - if (res == (size_t)-1) { - // on some platforms (e.g., linux) iconv will fail with - // E2BIG if it cannot convert _all_ of its input. it'll - // still adjust all of the in/out params correctly, so we - // can ignore this error. the assumption is that we will - // be called again to complete the conversion. - if ((errno == E2BIG) && (*aOutputLeft < outputAvail)) { - res = 0; - } - } - return res; -} - -static inline void -xp_iconv_reset(iconv_t converter) -{ - // NOTE: the man pages on Solaris claim that you can pass nullptr - // for all parameter to reset the converter, but beware the - // evil Solaris crash if you go down this route >:-) - - const char* zero_char_in_ptr = nullptr; - char* zero_char_out_ptr = nullptr; - size_t zero_size_in = 0; - size_t zero_size_out = 0; - - xp_iconv(converter, - &zero_char_in_ptr, - &zero_size_in, - &zero_char_out_ptr, - &zero_size_out); -} - -static inline iconv_t -xp_iconv_open(const char** to_list, const char** from_list) -{ - iconv_t res; - const char** from_name; - const char** to_name; - - // try all possible combinations to locate a converter. - to_name = to_list; - while (*to_name) { - if (**to_name) { - from_name = from_list; - while (*from_name) { - if (**from_name) { - res = iconv_open(*to_name, *from_name); - if (res != INVALID_ICONV_T) { - return res; - } - } - from_name++; - } - } - to_name++; - } - - return INVALID_ICONV_T; -} - -/* - * char16_t[] is NOT a UCS-2 array BUT a UTF-16 string. Therefore, we - * have to use UTF-16 with iconv(3) on platforms where it's supported. - * However, the way UTF-16 and UCS-2 are interpreted varies across platforms - * and implementations of iconv(3). On Tru64, it also depends on the environment - * variable. To avoid the trouble arising from byte-swapping - * (bug 208809), we have to try UTF-16LE/BE and UCS-2LE/BE before falling - * back to UTF-16 and UCS-2 and variants. We assume that UTF-16 and UCS-2 - * on systems without UTF-16LE/BE and UCS-2LE/BE have the native endianness, - * which isn't the case of glibc 2.1.x, for which we use 'UNICODELITTLE' - * and 'UNICODEBIG'. It's also not true of Tru64 V4 when the environment - * variable ICONV_BYTEORDER is set to 'big-endian', about which not much - * can be done other than adding a note in the release notes. (bug 206811) - */ -static const char* UTF_16_NAMES[] = { -#if defined(IS_LITTLE_ENDIAN) - "UTF-16LE", -#if defined(__GLIBC__) - "UNICODELITTLE", -#endif - "UCS-2LE", -#else - "UTF-16BE", -#if defined(__GLIBC__) - "UNICODEBIG", -#endif - "UCS-2BE", -#endif - "UTF-16", - "UCS-2", - "UCS2", - "UCS_2", - "ucs-2", - "ucs2", - "ucs_2", - nullptr -}; - -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) -static const char* UTF_8_NAMES[] = { - "UTF-8", - "UTF8", - "UTF_8", - "utf-8", - "utf8", - "utf_8", - nullptr -}; -#endif - -static const char* ISO_8859_1_NAMES[] = { - "ISO-8859-1", -#if !defined(__GLIBC__) - "ISO8859-1", - "ISO88591", - "ISO_8859_1", - "ISO8859_1", - "iso-8859-1", - "iso8859-1", - "iso88591", - "iso_8859_1", - "iso8859_1", -#endif - nullptr -}; - -class nsNativeCharsetConverter -{ -public: - nsNativeCharsetConverter(); - ~nsNativeCharsetConverter(); - - nsresult NativeToUnicode(const char** aInput, uint32_t* aInputLeft, - char16_t** aOutput, uint32_t* aOutputLeft); - nsresult UnicodeToNative(const char16_t** aInput, uint32_t* aInputLeft, - char** aOutput, uint32_t* aOutputLeft); - - static void GlobalInit(); - static void GlobalShutdown(); - static bool IsNativeUTF8(); - -private: - static iconv_t gNativeToUnicode; - static iconv_t gUnicodeToNative; -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - static iconv_t gNativeToUTF8; - static iconv_t gUTF8ToNative; - static iconv_t gUnicodeToUTF8; - static iconv_t gUTF8ToUnicode; -#endif - static Mutex* gLock; - static bool gInitialized; - static bool gIsNativeUTF8; - - static void LazyInit(); - - static void Lock() - { - if (gLock) { - gLock->Lock(); - } - } - static void Unlock() - { - if (gLock) { - gLock->Unlock(); - } - } -}; - -iconv_t nsNativeCharsetConverter::gNativeToUnicode = INVALID_ICONV_T; -iconv_t nsNativeCharsetConverter::gUnicodeToNative = INVALID_ICONV_T; -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) -iconv_t nsNativeCharsetConverter::gNativeToUTF8 = INVALID_ICONV_T; -iconv_t nsNativeCharsetConverter::gUTF8ToNative = INVALID_ICONV_T; -iconv_t nsNativeCharsetConverter::gUnicodeToUTF8 = INVALID_ICONV_T; -iconv_t nsNativeCharsetConverter::gUTF8ToUnicode = INVALID_ICONV_T; -#endif -Mutex* nsNativeCharsetConverter::gLock = nullptr; -bool nsNativeCharsetConverter::gInitialized = false; -bool nsNativeCharsetConverter::gIsNativeUTF8 = false; - -void -nsNativeCharsetConverter::LazyInit() -{ - // LazyInit may be called before NS_StartupNativeCharsetUtils, but - // the setlocale it does has to be called before nl_langinfo. Like in - // NS_StartupNativeCharsetUtils, assume we are called early enough that - // we are the first to care about the locale's charset. - if (!gLock) { - setlocale(LC_CTYPE, ""); - } - const char* blank_list[] = { "", nullptr }; - const char** native_charset_list = blank_list; - const char* native_charset = nl_langinfo(CODESET); - if (!native_charset) { - NS_ERROR("native charset is unknown"); - // fallback to ISO-8859-1 - native_charset_list = ISO_8859_1_NAMES; - } else { - native_charset_list[0] = native_charset; - } - - // Most, if not all, Unixen supporting UTF-8 and nl_langinfo(CODESET) - // return 'UTF-8' (or 'utf-8') - if (!PL_strcasecmp(native_charset, "UTF-8")) { - gIsNativeUTF8 = true; - } - - gNativeToUnicode = xp_iconv_open(UTF_16_NAMES, native_charset_list); - gUnicodeToNative = xp_iconv_open(native_charset_list, UTF_16_NAMES); - -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - if (gNativeToUnicode == INVALID_ICONV_T) { - gNativeToUTF8 = xp_iconv_open(UTF_8_NAMES, native_charset_list); - gUTF8ToUnicode = xp_iconv_open(UTF_16_NAMES, UTF_8_NAMES); - NS_ASSERTION(gNativeToUTF8 != INVALID_ICONV_T, "no native to utf-8 converter"); - NS_ASSERTION(gUTF8ToUnicode != INVALID_ICONV_T, "no utf-8 to utf-16 converter"); - } - if (gUnicodeToNative == INVALID_ICONV_T) { - gUnicodeToUTF8 = xp_iconv_open(UTF_8_NAMES, UTF_16_NAMES); - gUTF8ToNative = xp_iconv_open(native_charset_list, UTF_8_NAMES); - NS_ASSERTION(gUnicodeToUTF8 != INVALID_ICONV_T, "no utf-16 to utf-8 converter"); - NS_ASSERTION(gUTF8ToNative != INVALID_ICONV_T, "no utf-8 to native converter"); - } -#else - NS_ASSERTION(gNativeToUnicode != INVALID_ICONV_T, "no native to utf-16 converter"); - NS_ASSERTION(gUnicodeToNative != INVALID_ICONV_T, "no utf-16 to native converter"); -#endif - - /* - * On Solaris 8 (and newer?), the iconv modules converting to UCS-2 - * prepend a byte order mark unicode character (BOM, u+FEFF) during - * the first use of the iconv converter. The same is the case of - * glibc 2.2.9x and Tru64 V5 (see bug 208809) when 'UTF-16' is used. - * However, we use 'UTF-16LE/BE' in both cases, instead so that we - * should be safe. But just in case... - * - * This dummy conversion gets rid of the BOMs and fixes bug 153562. - */ - char dummy_input[1] = { ' ' }; - char dummy_output[4]; - - if (gNativeToUnicode != INVALID_ICONV_T) { - const char* input = dummy_input; - size_t input_left = sizeof(dummy_input); - char* output = dummy_output; - size_t output_left = sizeof(dummy_output); - - xp_iconv(gNativeToUnicode, &input, &input_left, &output, &output_left); - } -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - if (gUTF8ToUnicode != INVALID_ICONV_T) { - const char* input = dummy_input; - size_t input_left = sizeof(dummy_input); - char* output = dummy_output; - size_t output_left = sizeof(dummy_output); - - xp_iconv(gUTF8ToUnicode, &input, &input_left, &output, &output_left); - } -#endif - - gInitialized = true; -} - -void -nsNativeCharsetConverter::GlobalInit() -{ - gLock = new Mutex("nsNativeCharsetConverter.gLock"); -} - -void -nsNativeCharsetConverter::GlobalShutdown() -{ - delete gLock; - gLock = nullptr; - - if (gNativeToUnicode != INVALID_ICONV_T) { - iconv_close(gNativeToUnicode); - gNativeToUnicode = INVALID_ICONV_T; - } - - if (gUnicodeToNative != INVALID_ICONV_T) { - iconv_close(gUnicodeToNative); - gUnicodeToNative = INVALID_ICONV_T; - } - -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - if (gNativeToUTF8 != INVALID_ICONV_T) { - iconv_close(gNativeToUTF8); - gNativeToUTF8 = INVALID_ICONV_T; - } - if (gUTF8ToNative != INVALID_ICONV_T) { - iconv_close(gUTF8ToNative); - gUTF8ToNative = INVALID_ICONV_T; - } - if (gUnicodeToUTF8 != INVALID_ICONV_T) { - iconv_close(gUnicodeToUTF8); - gUnicodeToUTF8 = INVALID_ICONV_T; - } - if (gUTF8ToUnicode != INVALID_ICONV_T) { - iconv_close(gUTF8ToUnicode); - gUTF8ToUnicode = INVALID_ICONV_T; - } -#endif - - gInitialized = false; -} - -nsNativeCharsetConverter::nsNativeCharsetConverter() -{ - Lock(); - if (!gInitialized) { - LazyInit(); - } -} - -nsNativeCharsetConverter::~nsNativeCharsetConverter() -{ - // reset converters for next time - if (gNativeToUnicode != INVALID_ICONV_T) { - xp_iconv_reset(gNativeToUnicode); - } - if (gUnicodeToNative != INVALID_ICONV_T) { - xp_iconv_reset(gUnicodeToNative); - } -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - if (gNativeToUTF8 != INVALID_ICONV_T) { - xp_iconv_reset(gNativeToUTF8); - } - if (gUTF8ToNative != INVALID_ICONV_T) { - xp_iconv_reset(gUTF8ToNative); - } - if (gUnicodeToUTF8 != INVALID_ICONV_T) { - xp_iconv_reset(gUnicodeToUTF8); - } - if (gUTF8ToUnicode != INVALID_ICONV_T) { - xp_iconv_reset(gUTF8ToUnicode); - } -#endif - Unlock(); -} - -nsresult -nsNativeCharsetConverter::NativeToUnicode(const char** aInput, - uint32_t* aInputLeft, - char16_t** aOutput, - uint32_t* aOutputLeft) -{ - size_t res = 0; - size_t inLeft = (size_t)*aInputLeft; - size_t outLeft = (size_t)*aOutputLeft * 2; - - if (gNativeToUnicode != INVALID_ICONV_T) { - - res = xp_iconv(gNativeToUnicode, aInput, &inLeft, (char**)aOutput, &outLeft); - - *aInputLeft = inLeft; - *aOutputLeft = outLeft / 2; - if (res != (size_t)-1) { - return NS_OK; - } - - NS_WARNING("conversion from native to utf-16 failed"); - - // reset converter - xp_iconv_reset(gNativeToUnicode); - } -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - else if ((gNativeToUTF8 != INVALID_ICONV_T) && - (gUTF8ToUnicode != INVALID_ICONV_T)) { - // convert first to UTF8, then from UTF8 to UCS2 - const char* in = *aInput; - - char ubuf[1024]; - - // we assume we're always called with enough space in |aOutput|, - // so convert many chars at a time... - while (inLeft) { - char* p = ubuf; - size_t n = sizeof(ubuf); - res = xp_iconv(gNativeToUTF8, &in, &inLeft, &p, &n); - if (res == (size_t)-1) { - NS_ERROR("conversion from native to utf-8 failed"); - break; - } - NS_ASSERTION(outLeft > 0, "bad assumption"); - p = ubuf; - n = sizeof(ubuf) - n; - res = xp_iconv(gUTF8ToUnicode, (const char**)&p, &n, - (char**)aOutput, &outLeft); - if (res == (size_t)-1) { - NS_ERROR("conversion from utf-8 to utf-16 failed"); - break; - } - } - - (*aInput) += (*aInputLeft - inLeft); - *aInputLeft = inLeft; - *aOutputLeft = outLeft / 2; - - if (res != (size_t)-1) { - return NS_OK; - } - - // reset converters - xp_iconv_reset(gNativeToUTF8); - xp_iconv_reset(gUTF8ToUnicode); - } -#endif - - // fallback: zero-pad and hope for the best - // XXX This is lame and we have to do better. - isolatin1_to_utf16(aInput, aInputLeft, aOutput, aOutputLeft); - - return NS_OK; -} - -nsresult -nsNativeCharsetConverter::UnicodeToNative(const char16_t** aInput, - uint32_t* aInputLeft, - char** aOutput, - uint32_t* aOutputLeft) -{ - size_t res = 0; - size_t inLeft = (size_t)*aInputLeft * 2; - size_t outLeft = (size_t)*aOutputLeft; - - if (gUnicodeToNative != INVALID_ICONV_T) { - res = xp_iconv(gUnicodeToNative, (const char**)aInput, &inLeft, - aOutput, &outLeft); - - *aInputLeft = inLeft / 2; - *aOutputLeft = outLeft; - if (res != (size_t)-1) { - return NS_OK; - } - - NS_ERROR("iconv failed"); - - // reset converter - xp_iconv_reset(gUnicodeToNative); - } -#if defined(ENABLE_UTF8_FALLBACK_SUPPORT) - else if ((gUnicodeToUTF8 != INVALID_ICONV_T) && - (gUTF8ToNative != INVALID_ICONV_T)) { - const char* in = (const char*)*aInput; - - char ubuf[6]; // max utf-8 char length (really only needs to be 4 bytes) - - // convert one uchar at a time... - while (inLeft && outLeft) { - char* p = ubuf; - size_t n = sizeof(ubuf), one_uchar = sizeof(char16_t); - res = xp_iconv(gUnicodeToUTF8, &in, &one_uchar, &p, &n); - if (res == (size_t)-1) { - NS_ERROR("conversion from utf-16 to utf-8 failed"); - break; - } - p = ubuf; - n = sizeof(ubuf) - n; - res = xp_iconv(gUTF8ToNative, (const char**)&p, &n, aOutput, &outLeft); - if (res == (size_t)-1) { - if (errno == E2BIG) { - // not enough room for last uchar... back up and return. - in -= sizeof(char16_t); - res = 0; - } else { - NS_ERROR("conversion from utf-8 to native failed"); - } - break; - } - inLeft -= sizeof(char16_t); - } - - (*aInput) += (*aInputLeft - inLeft / 2); - *aInputLeft = inLeft / 2; - *aOutputLeft = outLeft; - if (res != (size_t)-1) { - return NS_OK; - } - - // reset converters - xp_iconv_reset(gUnicodeToUTF8); - xp_iconv_reset(gUTF8ToNative); - } -#endif - - // fallback: truncate and hope for the best - // XXX This is lame and we have to do better. - utf16_to_isolatin1(aInput, aInputLeft, aOutput, aOutputLeft); - - return NS_OK; -} - -bool -nsNativeCharsetConverter::IsNativeUTF8() -{ - if (!gInitialized) { - Lock(); - if (!gInitialized) { - LazyInit(); - } - Unlock(); - } - return gIsNativeUTF8; -} - -#endif // USE_ICONV - -//----------------------------------------------------------------------------- -// conversion using mb[r]towc/wc[r]tomb -//----------------------------------------------------------------------------- -#if defined(USE_STDCONV) -#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC) -#include // mbrtowc, wcrtomb -#endif - -class nsNativeCharsetConverter -{ -public: - nsNativeCharsetConverter(); - - nsresult NativeToUnicode(const char** aInput, uint32_t* aInputLeft, - char16_t** aOutput, uint32_t* aOutputLeft); - nsresult UnicodeToNative(const char16_t** aInput, uint32_t* aInputLeft, - char** aOutput, uint32_t* aOutputLeft); - - static void GlobalInit(); - static void GlobalShutdown() { } - static bool IsNativeUTF8(); - -private: - static bool gWCharIsUnicode; - -#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC) - mbstate_t ps; -#endif -}; - -bool nsNativeCharsetConverter::gWCharIsUnicode = false; - -nsNativeCharsetConverter::nsNativeCharsetConverter() -{ -#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC) - memset(&ps, 0, sizeof(ps)); -#endif -} - -void -nsNativeCharsetConverter::GlobalInit() -{ - // verify that wchar_t for the current locale is actually unicode. - // if it is not, then we should avoid calling mbtowc/wctomb and - // just fallback on zero-pad/truncation conversion. - // - // this test cannot be done at build time because the encoding of - // wchar_t may depend on the runtime locale. sad, but true!! - // - // so, if wchar_t is unicode then converting an ASCII character - // to wchar_t should not change its numeric value. we'll just - // check what happens with the ASCII 'a' character. - // - // this test is not perfect... obviously, it could yield false - // positives, but then at least ASCII text would be converted - // properly (or maybe just the 'a' character) -- oh well :( - - char a = 'a'; - unsigned int w = 0; - - int res = mbtowc((wchar_t*)&w, &a, 1); - - gWCharIsUnicode = (res != -1 && w == 'a'); - -#ifdef DEBUG - if (!gWCharIsUnicode) { - NS_WARNING("wchar_t is not unicode (unicode conversion will be lossy)"); - } -#endif -} - -nsresult -nsNativeCharsetConverter::NativeToUnicode(const char** aInput, - uint32_t* aInputLeft, - char16_t** aOutput, - uint32_t* aOutputLeft) -{ - if (gWCharIsUnicode) { - int incr; - - // cannot use wchar_t here since it may have been redefined (e.g., - // via -fshort-wchar). hopefully, sizeof(tmp) is sufficient XP. - unsigned int tmp = 0; - while (*aInputLeft && *aOutputLeft) { -#ifdef HAVE_MBRTOWC - incr = (int)mbrtowc((wchar_t*)&tmp, *aInput, *aInputLeft, &ps); -#else - // XXX is this thread-safe? - incr = (int)mbtowc((wchar_t*)&tmp, *aInput, *aInputLeft); -#endif - if (incr < 0) { - NS_WARNING("mbtowc failed: possible charset mismatch"); - // zero-pad and hope for the best - tmp = (unsigned char)**aInput; - incr = 1; - } - ** aOutput = (char16_t)tmp; - (*aInput) += incr; - (*aInputLeft) -= incr; - (*aOutput)++; - (*aOutputLeft)--; - } - } else { - // wchar_t isn't unicode, so the best we can do is treat the - // input as if it is isolatin1 :( - isolatin1_to_utf16(aInput, aInputLeft, aOutput, aOutputLeft); - } - - return NS_OK; -} - -nsresult -nsNativeCharsetConverter::UnicodeToNative(const char16_t** aInput, - uint32_t* aInputLeft, - char** aOutput, - uint32_t* aOutputLeft) -{ - if (gWCharIsUnicode) { - int incr; - - while (*aInputLeft && *aOutputLeft >= MB_CUR_MAX) { -#ifdef HAVE_WCRTOMB - incr = (int)wcrtomb(*aOutput, (wchar_t)**aInput, &ps); -#else - // XXX is this thread-safe? - incr = (int)wctomb(*aOutput, (wchar_t)**aInput); -#endif - if (incr < 0) { - NS_WARNING("mbtowc failed: possible charset mismatch"); - ** aOutput = (unsigned char)**aInput; // truncate - incr = 1; - } - // most likely we're dead anyways if this assertion should fire - NS_ASSERTION(uint32_t(incr) <= *aOutputLeft, "wrote beyond end of string"); - (*aOutput) += incr; - (*aOutputLeft) -= incr; - (*aInput)++; - (*aInputLeft)--; - } - } else { - // wchar_t isn't unicode, so the best we can do is treat the - // input as if it is isolatin1 :( - utf16_to_isolatin1(aInput, aInputLeft, aOutput, aOutputLeft); - } - - return NS_OK; -} - -// XXX : for now, return false -bool -nsNativeCharsetConverter::IsNativeUTF8() -{ - return false; -} - -#endif // USE_STDCONV - -//----------------------------------------------------------------------------- -// API implementation -//----------------------------------------------------------------------------- - -nsresult -NS_CopyNativeToUnicode(const nsACString& aInput, nsAString& aOutput) -{ - aOutput.Truncate(); - - uint32_t inputLen = aInput.Length(); - - nsACString::const_iterator iter; - aInput.BeginReading(iter); - - // - // OPTIMIZATION: preallocate space for largest possible result; convert - // directly into the result buffer to avoid intermediate buffer copy. - // - // this will generally result in a larger allocation, but that seems - // better than an extra buffer copy. - // - if (!aOutput.SetLength(inputLen, fallible)) { - return NS_ERROR_OUT_OF_MEMORY; - } - nsAString::iterator out_iter; - aOutput.BeginWriting(out_iter); - - char16_t* result = out_iter.get(); - uint32_t resultLeft = inputLen; - - const char* buf = iter.get(); - uint32_t bufLeft = inputLen; - - nsNativeCharsetConverter conv; - nsresult rv = conv.NativeToUnicode(&buf, &bufLeft, &result, &resultLeft); - if (NS_SUCCEEDED(rv)) { - NS_ASSERTION(bufLeft == 0, "did not consume entire input buffer"); - aOutput.SetLength(inputLen - resultLeft); - } - return rv; -} - -nsresult -NS_CopyUnicodeToNative(const nsAString& aInput, nsACString& aOutput) -{ - aOutput.Truncate(); - - nsAString::const_iterator iter, end; - aInput.BeginReading(iter); - aInput.EndReading(end); - - // cannot easily avoid intermediate buffer copy. - char temp[4096]; - - nsNativeCharsetConverter conv; - - const char16_t* buf = iter.get(); - uint32_t bufLeft = Distance(iter, end); - while (bufLeft) { - char* p = temp; - uint32_t tempLeft = sizeof(temp); - - nsresult rv = conv.UnicodeToNative(&buf, &bufLeft, &p, &tempLeft); - if (NS_FAILED(rv)) { - return rv; - } - - if (tempLeft < sizeof(temp)) { - aOutput.Append(temp, sizeof(temp) - tempLeft); - } - } - return NS_OK; -} - -bool -NS_IsNativeUTF8() -{ - return nsNativeCharsetConverter::IsNativeUTF8(); -} - -void -NS_StartupNativeCharsetUtils() -{ - // - // need to initialize the locale or else charset conversion will fail. - // better not delay this in case some other component alters the locale - // settings. - // - // XXX we assume that we are called early enough that we should - // always be the first to care about the locale's charset. - // - setlocale(LC_CTYPE, ""); - - nsNativeCharsetConverter::GlobalInit(); -} - -void -NS_ShutdownNativeCharsetUtils() -{ - nsNativeCharsetConverter::GlobalShutdown(); -} - -//----------------------------------------------------------------------------- -// XP_WIN -//----------------------------------------------------------------------------- -#elif defined(XP_WIN) +#if defined(XP_WIN) #include #include "nsString.h" @@ -980,30 +125,22 @@ NS_ConvertWtoA(const char16_t* aStrInW, int aBufferSizeOut, #else +// Non-windows will always use UTF-8 conversion. + #include "nsReadableUtils.h" nsresult NS_CopyNativeToUnicode(const nsACString& aInput, nsAString& aOutput) { - CopyASCIItoUTF16(aInput, aOutput); + CopyUTF8toUTF16(aInput, aOutput); return NS_OK; } nsresult NS_CopyUnicodeToNative(const nsAString& aInput, nsACString& aOutput) { - LossyCopyUTF16toASCII(aInput, aOutput); + CopyUTF16toUTF8(aInput, aOutput); return NS_OK; } -void -NS_StartupNativeCharsetUtils() -{ -} - -void -NS_ShutdownNativeCharsetUtils() -{ -} - #endif diff --git a/xpcom/io/nsNativeCharsetUtils.h b/xpcom/io/nsNativeCharsetUtils.h index bee533cbef..572995aad5 100644 --- a/xpcom/io/nsNativeCharsetUtils.h +++ b/xpcom/io/nsNativeCharsetUtils.h @@ -14,9 +14,13 @@ * *** THESE ARE NOT GENERAL PURPOSE CONVERTERS *** * * * * NS_CopyNativeToUnicode / NS_CopyUnicodeToNative should only be used * - * for converting *FILENAMES* between native and unicode. They are not * + * for converting *FILENAMES* between bytes and UTF-16. They are not * * designed or tested for general encoding converter use. * * * + * On Windows, these functions convert to and from the system's legacy * + * code page, which cannot represent all of Unicode. Elsewhere, these * + * convert to and from UTF-8. * + * * \*****************************************************************************/ /** @@ -33,25 +37,15 @@ nsresult NS_CopyUnicodeToNative(const nsAString& aInput, nsACString& aOutput); * name in UTF-8 out of nsIFile, we can just use |GetNativeLeafName| rather * than using |GetLeafName| and converting the result to UTF-8 if the file * system encoding is UTF-8. - * On Unix, it depends on the locale and is not known in advance (at the - * compilation time) so that this function needs to be a real function. - * On Windows and other platforms (e.g. OS2), it's never UTF-8. */ -#if defined(XP_UNIX) -bool NS_IsNativeUTF8(); -#else inline bool NS_IsNativeUTF8() { +#ifdef XP_WIN return false; -} +#else + return true; #endif - - -/** - * internal - */ -void NS_StartupNativeCharsetUtils(); -void NS_ShutdownNativeCharsetUtils(); +} #endif // nsNativeCharsetUtils_h__ From 378f01e52602a7aee19aac9d1b3f26443b34e2ab Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Mon, 31 May 2021 18:55:45 -0400 Subject: [PATCH 3/7] Issue #1774 - Add metrics to the editable menulist-button for GTK3 --- toolkit/themes/linux/global/jar.mn | 2 +- toolkit/themes/linux/global/menulist.css | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/toolkit/themes/linux/global/jar.mn b/toolkit/themes/linux/global/jar.mn index ba665adff1..a2f7382bc9 100644 --- a/toolkit/themes/linux/global/jar.mn +++ b/toolkit/themes/linux/global/jar.mn @@ -19,7 +19,7 @@ toolkit.jar: skin/classic/global/inContentUI.css skin/classic/global/listbox.css skin/classic/global/menu.css - skin/classic/global/menulist.css +* skin/classic/global/menulist.css skin/classic/global/netError.css * skin/classic/global/notification.css skin/classic/global/numberbox.css diff --git a/toolkit/themes/linux/global/menulist.css b/toolkit/themes/linux/global/menulist.css index 30e55e11ba..73c1dfd49a 100644 --- a/toolkit/themes/linux/global/menulist.css +++ b/toolkit/themes/linux/global/menulist.css @@ -73,6 +73,10 @@ menulist[editable="true"] { menulist[editable="true"] > .menulist-dropmarker { display: -moz-box; -moz-appearance: menulist-button; +%if MOZ_WIDGET_GTK == 3 + min-width: 2.5em; + min-height: 2.25em; +%endif } html|*.menulist-editable-input { From 859208a201bd3d9c5888e847a3fd8d3effdc93ba Mon Sep 17 00:00:00 2001 From: Moonchild Date: Wed, 2 Jun 2021 09:44:59 +0000 Subject: [PATCH 4/7] [toolkit] Exclude printer spew from troubleshooting information. In almost all cases this is pointless to include, a ton of useless entries, and there's no reason to have people post a full enumeration of installed printers on their system when asked for TS information. If there's a printer specific issue, people can always be asked to post relevant info manually. --- toolkit/modules/Troubleshoot.jsm | 1 + 1 file changed, 1 insertion(+) diff --git a/toolkit/modules/Troubleshoot.jsm b/toolkit/modules/Troubleshoot.jsm index 6ee6cb54e5..8ff6be30ba 100644 --- a/toolkit/modules/Troubleshoot.jsm +++ b/toolkit/modules/Troubleshoot.jsm @@ -96,6 +96,7 @@ const PREFS_WHITELIST = [ const PREFS_BLACKLIST = [ /^network[.]proxy[.]/, /[.]print_to_filename$/, + /^print[.]printer_/, /^print[.]macosx[.]pagesetup/, ]; From 9c3495379fb70afdb5554590a0bcc9747e59033a Mon Sep 17 00:00:00 2001 From: Jan de Mooij Date: Wed, 2 Jun 2021 10:29:47 +0000 Subject: [PATCH 5/7] [js] Fix invalid early return in BaselineFrame::trace. We were not tracing debugger environments for Baseline frames without any local/expression slots. --- js/src/jit/BaselineFrame.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/js/src/jit/BaselineFrame.cpp b/js/src/jit/BaselineFrame.cpp index b8c61300fc..3f00e9b36a 100644 --- a/js/src/jit/BaselineFrame.cpp +++ b/js/src/jit/BaselineFrame.cpp @@ -62,24 +62,24 @@ BaselineFrame::trace(JSTracer* trc, JitFrameIterator& frameIterator) // NB: It is possible that numValueSlots() could be zero, even if nfixed is // nonzero. This is the case if the function has an early stack check. - if (numValueSlots() == 0) - return; + if (numValueSlots > 0) { - MOZ_ASSERT(nfixed <= numValueSlots()); + MOZ_ASSERT(nfixed <= numValueSlots()); - if (nfixed == nlivefixed) { - // All locals are live. - MarkLocals(this, trc, 0, numValueSlots()); - } else { - // Mark operand stack. - MarkLocals(this, trc, nfixed, numValueSlots()); + if (nfixed == nlivefixed) { + // All locals are live. + MarkLocals(this, trc, 0, numValueSlots()); + } else { + // Mark operand stack. + MarkLocals(this, trc, nfixed, numValueSlots()); - // Clear dead block-scoped locals. - while (nfixed > nlivefixed) - unaliasedLocal(--nfixed).setUndefined(); + // Clear dead block-scoped locals. + while (nfixed > nlivefixed) + unaliasedLocal(--nfixed).setUndefined(); - // Mark live locals. - MarkLocals(this, trc, 0, nlivefixed); + // Mark live locals. + MarkLocals(this, trc, 0, nlivefixed); + } } if (script->compartment()->debugEnvs) From d46c25b5e59a12d4677f830796cc80a6f98560ed Mon Sep 17 00:00:00 2001 From: Moonchild Date: Thu, 3 Jun 2021 00:23:41 +0000 Subject: [PATCH 6/7] [js] Fix porting typo Follow-up for 9acd98298ccadd83495fa1615573ebcca6648bae --- js/src/jit/BaselineFrame.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/jit/BaselineFrame.cpp b/js/src/jit/BaselineFrame.cpp index 3f00e9b36a..243fd31293 100644 --- a/js/src/jit/BaselineFrame.cpp +++ b/js/src/jit/BaselineFrame.cpp @@ -62,7 +62,7 @@ BaselineFrame::trace(JSTracer* trc, JitFrameIterator& frameIterator) // NB: It is possible that numValueSlots() could be zero, even if nfixed is // nonzero. This is the case if the function has an early stack check. - if (numValueSlots > 0) { + if (numValueSlots() > 0) { MOZ_ASSERT(nfixed <= numValueSlots()); From 4bcb79066b465e132f9e9a1b3ad97036a0083689 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Sun, 23 May 2021 14:57:13 +0000 Subject: [PATCH 7/7] [Pale-Moon] Issue #1870 - Add try/catch and console logging for failing autocomplete popup. --- application/palemoon/modules/AutoCompletePopup.jsm | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/application/palemoon/modules/AutoCompletePopup.jsm b/application/palemoon/modules/AutoCompletePopup.jsm index 2fe5e70bb5..3d37ceba47 100644 --- a/application/palemoon/modules/AutoCompletePopup.jsm +++ b/application/palemoon/modules/AutoCompletePopup.jsm @@ -188,10 +188,15 @@ this.AutoCompletePopup = { closePopup() { if (this.openedPopup) { - // Note that hidePopup() closes the popup immediately, - // so popuphiding or popuphidden events will be fired - // and handled during this call. - this.openedPopup.hidePopup(); + try { + // Note that hidePopup() closes the popup immediately, + // so popuphiding or popuphidden events will be fired + // and handled during this call. + this.openedPopup.hidePopup(); + } catch(e) { + Cu.reportError(e); + console.log("Debug: ", this.openedPopup); + } } AutoCompleteTreeView.clearResults(); },