Issue #1258 - Part 1: Import mailnews, ldap, and mork from comm-esr52.9.1

mailnews/import/oexpress/nsOEMailbox.cpp

@@ -0,0 +1,673 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsOEMailbox.h"
+
+#include "OEDebugLog.h"
+#include "msgCore.h"
+#include "prprf.h"
+#include "nsMsgLocalFolderHdrs.h"
+#include "nsCRT.h"
+#include "nsNetUtil.h"
+#include "nsIInputStream.h"
+#include "nsIOutputStream.h"
+#include "nsIMsgFolder.h"
+#include "nsIMsgHdr.h"
+#include "nsIMsgPluggableStore.h"
+#include "nsISeekableStream.h"
+#include "nsMsgUtils.h"
+
+class CMbxScanner {
+public:
+  CMbxScanner(nsString& name, nsIFile * mbxFile, nsIMsgFolder *dstFolder);
+  ~CMbxScanner();
+
+  virtual bool    Initialize(void);
+  virtual bool    DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount);
+
+  bool      WasErrorFatal(void) { return m_fatalError;}
+  uint32_t  BytesProcessed(void) { return m_didBytes;}
+
+protected:
+  bool    WriteMailItem(uint32_t flags, uint32_t offset, uint32_t size, uint32_t *pTotalMsgSize = nullptr);
+  virtual void  CleanUp(void);
+
+private:
+  void  ReportWriteError(nsIMsgFolder *folder, bool fatal = true);
+  void  ReportReadError(nsIFile * file, bool fatal = true);
+  bool CopyMbxFileBytes(uint32_t flags, uint32_t numBytes);
+  bool IsFromLineKey(uint8_t *pBuf, uint32_t max);
+
+public:
+  uint32_t      m_msgCount;
+
+protected:
+  uint32_t *    m_pDone;
+  nsString      m_name;
+  nsCOMPtr<nsIFile> m_mbxFile;
+  nsCOMPtr<nsIMsgFolder> m_dstFolder;
+  nsCOMPtr<nsIInputStream> m_mbxFileInputStream;
+  nsCOMPtr<nsIOutputStream> m_dstOutputStream;
+  nsCOMPtr<nsIMsgPluggableStore> m_msgStore;
+  uint8_t *     m_pInBuffer;
+  uint8_t *     m_pOutBuffer;
+  uint32_t      m_bufSz;
+  uint32_t      m_didBytes;
+  bool          m_fatalError;
+  int64_t      m_mbxFileSize;
+  uint32_t      m_mbxOffset;
+
+  static const char *  m_pFromLine;
+
+};
+
+
+class CIndexScanner : public CMbxScanner {
+public:
+  CIndexScanner(nsString& name, nsIFile * idxFile, nsIFile * mbxFile, nsIMsgFolder *dstFolder);
+  ~CIndexScanner();
+
+  virtual bool    Initialize(void);
+  virtual bool    DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount);
+
+protected:
+  virtual void    CleanUp(void);
+
+private:
+  bool            ValidateIdxFile(void);
+  bool            GetMailItem(uint32_t *pFlags, uint32_t *pOffset, uint32_t *pSize);
+
+
+private:
+  nsCOMPtr <nsIFile>   m_idxFile;
+  nsCOMPtr <nsIInputStream> m_idxFileInputStream;
+  uint32_t        m_numMessages;
+  uint32_t        m_idxOffset;
+  uint32_t        m_curItemIndex;
+};
+
+
+bool CImportMailbox::ImportMailbox(uint32_t *pDone, bool *pAbort,
+                                   nsString& name, nsIFile * inFile,
+                                   nsIMsgFolder *outFolder, uint32_t *pCount)
+{
+  bool    done = false;
+  nsresult rv;
+  nsCOMPtr <nsIFile> idxFile = do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv);
+  if (NS_SUCCEEDED(rv))
+    rv  = idxFile->InitWithFile(inFile);
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG0("New file spec failed!\n");
+    return false;
+  }
+
+  if (GetIndexFile(idxFile)) {
+
+    IMPORT_LOG1("Using index file for: %S\n", name.get());
+
+    CIndexScanner *pIdxScanner = new CIndexScanner(name, idxFile, inFile, outFolder);
+    if (pIdxScanner->Initialize()) {
+      if (pIdxScanner->DoWork(pAbort, pDone, pCount)) {
+        done = true;
+      }
+      else {
+        IMPORT_LOG0("CIndexScanner::DoWork() failed\n");
+      }
+    }
+    else {
+      IMPORT_LOG0("CIndexScanner::Initialize() failed\n");
+    }
+
+    delete pIdxScanner;
+  }
+
+  if (done)
+    return done;
+
+    /*
+    something went wrong with the index file, just scan the mailbox
+    file itself.
+  */
+  CMbxScanner *pMbx = new CMbxScanner(name, inFile, outFolder);
+  if (pMbx->Initialize()) {
+    if (pMbx->DoWork(pAbort, pDone, pCount)) {
+      done = true;
+    }
+    else {
+      IMPORT_LOG0("CMbxScanner::DoWork() failed\n");
+    }
+  }
+  else {
+    IMPORT_LOG0("CMbxScanner::Initialize() failed\n");
+  }
+
+  delete pMbx;
+  return done;
+}
+
+
+bool CImportMailbox::GetIndexFile(nsIFile* file)
+{
+  nsCString pLeaf;
+  if (NS_FAILED(file->GetNativeLeafName(pLeaf)))
+    return false;
+  int32_t  len = pLeaf.Length();
+  if (len < 5)
+    return false;
+
+  pLeaf.Replace(len - 3, 3, NS_LITERAL_CSTRING("idx"));
+
+  IMPORT_LOG1("Looking for index leaf name: %s\n", pLeaf);
+
+  nsresult  rv;
+  rv = file->SetNativeLeafName(pLeaf);
+
+  bool    isFile = false;
+  bool    exists = false;
+  if (NS_SUCCEEDED(rv)) rv = file->IsFile(&isFile);
+  if (NS_SUCCEEDED(rv)) rv = file->Exists(&exists);
+
+  return (isFile && exists);
+}
+
+
+const char *CMbxScanner::m_pFromLine = "From - Mon Jan 1 00:00:00 1965\x0D\x0A";
+// let's try a 16K buffer and see how well that works?
+#define  kBufferKB  16
+
+
+CMbxScanner::CMbxScanner(nsString& name, nsIFile* mbxFile,
+                         nsIMsgFolder* dstFolder)
+{
+  m_msgCount = 0;
+  m_name = name;
+  m_mbxFile = mbxFile;
+  m_dstFolder = dstFolder;
+  m_pInBuffer = nullptr;
+  m_pOutBuffer = nullptr;
+  m_bufSz = 0;
+  m_fatalError = false;
+  m_didBytes = 0;
+  m_mbxFileSize = 0;
+  m_mbxOffset = 0;
+}
+
+CMbxScanner::~CMbxScanner()
+{
+  CleanUp();
+}
+
+void CMbxScanner::ReportWriteError(nsIMsgFolder * folder, bool fatal)
+{
+  m_fatalError = fatal;
+}
+
+void CMbxScanner::ReportReadError(nsIFile * file, bool fatal)
+{
+  m_fatalError = fatal;
+}
+
+bool CMbxScanner::Initialize(void)
+{
+  m_bufSz = (kBufferKB * 1024);
+  m_pInBuffer = new uint8_t[m_bufSz];
+  m_pOutBuffer = new uint8_t[m_bufSz];
+  if (!m_pInBuffer || !m_pOutBuffer) {
+    return false;
+  }
+
+  m_mbxFile->GetFileSize(&m_mbxFileSize);
+  // open the mailbox file...
+  if (NS_FAILED(NS_NewLocalFileInputStream(getter_AddRefs(m_mbxFileInputStream), m_mbxFile))) {
+    CleanUp();
+    return false;
+  }
+
+  if (NS_FAILED(m_dstFolder->GetMsgStore(getter_AddRefs(m_msgStore)))) {
+    CleanUp();
+    return false;
+  }
+
+  return true;
+}
+
+
+#define  kMbxHeaderSize    0x0054
+#define kMbxMessageHeaderSz  16
+
+bool CMbxScanner::DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount)
+{
+  m_mbxOffset = kMbxHeaderSize;
+  m_didBytes = kMbxHeaderSize;
+
+  while (!(*pAbort) && ((m_mbxOffset + kMbxMessageHeaderSz) < m_mbxFileSize)) {
+    uint32_t    msgSz;
+    if (!WriteMailItem(0, m_mbxOffset, 0, &msgSz)) {
+      if (!WasErrorFatal())
+        ReportReadError(m_mbxFile);
+      return false;
+    }
+    m_mbxOffset += msgSz;
+    m_didBytes += msgSz;
+    m_msgCount++;
+    if (pDone)
+      *pDone = m_didBytes;
+    if (pCount)
+      *pCount = m_msgCount;
+  }
+
+  CleanUp();
+
+  return true;
+}
+
+
+void CMbxScanner::CleanUp(void)
+{
+  if (m_mbxFileInputStream)
+    m_mbxFileInputStream->Close();
+  if (m_dstOutputStream)
+    m_dstOutputStream->Close();
+
+  delete [] m_pInBuffer;
+  m_pInBuffer = nullptr;
+
+  delete [] m_pOutBuffer;
+  m_pOutBuffer = nullptr;
+}
+
+
+#define  kNumMbxLongsToRead  4
+
+bool CMbxScanner::WriteMailItem(uint32_t flags, uint32_t offset, uint32_t size,
+                                uint32_t *pTotalMsgSize)
+{
+  uint32_t  values[kNumMbxLongsToRead];
+  int32_t    cnt = kNumMbxLongsToRead * sizeof(uint32_t);
+  nsresult  rv;
+  uint32_t    cntRead;
+  int8_t *  pChar = (int8_t *) values;
+
+  nsCOMPtr <nsISeekableStream> seekableStream = do_QueryInterface(m_mbxFileInputStream);
+  rv = seekableStream->Seek(nsISeekableStream::NS_SEEK_SET,  offset);
+
+  if (NS_FAILED(rv)) {
+    IMPORT_LOG1("Mbx seek error: 0x%lx\n", offset);
+    return false;
+  }
+  rv = m_mbxFileInputStream->Read((char *) pChar, cnt, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != cnt)) {
+    IMPORT_LOG1("Mbx read error at: 0x%lx\n", offset);
+    return false;
+  }
+  if (values[0] != 0x7F007F00) {
+    IMPORT_LOG2("Mbx tag field doesn't match: 0x%lx, at offset: 0x%lx\n", values[0], offset);
+    return false;
+  }
+  if (size && (values[2] != size)) {
+    IMPORT_LOG3("Mbx size doesn't match idx, mbx: %ld, idx: %ld, at offset: 0x%lx\n", values[2], size, offset);
+    return false;
+  }
+
+  if (pTotalMsgSize != nullptr)
+    *pTotalMsgSize = values[2];
+
+  nsCOMPtr<nsIMsgDBHdr> msgHdr;
+  bool reusable;
+
+  rv = m_msgStore->GetNewMsgOutputStream(m_dstFolder, getter_AddRefs(msgHdr), &reusable,
+                                         getter_AddRefs(m_dstOutputStream));
+  if (NS_FAILED(rv))
+  {
+    IMPORT_LOG1( "Mbx getting outputstream error: 0x%lx\n", rv);
+    return false;
+  }
+
+  // everything looks kosher...
+  // the actual message text follows and is values[3] bytes long...
+  bool copyOK = CopyMbxFileBytes(flags,  values[3]);
+  if (copyOK)
+    m_msgStore->FinishNewMessage(m_dstOutputStream, msgHdr);
+  else {
+    m_msgStore->DiscardNewMessage(m_dstOutputStream, msgHdr);
+    IMPORT_LOG0( "Mbx CopyMbxFileBytes failed\n");
+  }
+  if (!reusable)
+  {
+    m_dstOutputStream->Close();
+    m_dstOutputStream = nullptr;
+  }
+  return copyOK;
+}
+
+bool CMbxScanner::IsFromLineKey(uint8_t * pBuf, uint32_t max)
+{
+  return (max > 5 && (pBuf[0] == 'F') && (pBuf[1] == 'r') && (pBuf[2] == 'o') && (pBuf[3] == 'm') && (pBuf[4] == ' '));
+}
+
+
+#define IS_ANY_SPACE(_ch) ((_ch == ' ') || (_ch == '\t') || (_ch == 10) || (_ch == 13))
+
+
+bool CMbxScanner::CopyMbxFileBytes(uint32_t flags, uint32_t numBytes)
+{
+  if (!numBytes)
+    return true;
+
+  uint32_t  cnt;
+  uint8_t   last[2] = {0, 0};
+  uint32_t  inIdx = 0;
+  bool      first = true;
+  uint8_t * pIn;
+  uint8_t * pStart;
+  int32_t   fromLen = strlen(m_pFromLine);
+  nsresult  rv;
+  uint32_t   cntRead;
+  uint8_t * pChar;
+
+  while (numBytes) {
+    if (numBytes > (m_bufSz - inIdx))
+      cnt = m_bufSz - inIdx;
+    else
+      cnt = numBytes;
+    // Read some of the message from the file...
+    pChar = m_pInBuffer + inIdx;
+    rv = m_mbxFileInputStream->Read((char *) pChar, (int32_t)cnt, &cntRead);
+    if (NS_FAILED(rv) || (cntRead != (int32_t)cnt)) {
+      ReportReadError(m_mbxFile);
+      return false;
+    }
+    // Keep track of the last 2 bytes of the message for terminating EOL logic
+    if (cnt < 2) {
+      last[0] = last[1];
+      last[1] = m_pInBuffer[cnt - 1];
+    }
+    else {
+      last[0] = m_pInBuffer[cnt - 2];
+      last[1] = m_pInBuffer[cnt - 1];
+    }
+
+    inIdx = 0;
+    // Handle the beginning line, don't duplicate an existing From separator
+    if (first) {
+      // check the first buffer to see if it already starts with a From line
+      // If it does, throw it away and use our own
+      if (IsFromLineKey(m_pInBuffer, cnt)) {
+        // skip past the first line
+        while ((inIdx < cnt) && (m_pInBuffer[inIdx] != nsCRT::CR))
+          inIdx++;
+        while ((inIdx < cnt) && (IS_ANY_SPACE(m_pInBuffer[inIdx])))
+          inIdx++;
+        if (inIdx >= cnt) {
+          // This should not occurr - it means the message starts
+          // with a From separator line that is longer than our
+          // file buffer!  In this bizarre case, just skip this message
+          // since it is probably bogus anyway.
+          return true;
+        }
+
+      }
+      // Begin every message with a From separator
+      rv = m_dstOutputStream->Write(m_pFromLine, fromLen, &cntRead);
+      if (NS_FAILED(rv) || (cntRead != fromLen)) {
+        ReportWriteError(m_dstFolder);
+        return false;
+      }
+      char statusLine[50];
+      uint32_t msgFlags = flags; // need to convert from OE flags to mozilla flags
+      PR_snprintf(statusLine, sizeof(statusLine), X_MOZILLA_STATUS_FORMAT MSG_LINEBREAK, msgFlags & 0xFFFF);
+      rv = m_dstOutputStream->Write(statusLine, strlen(statusLine), &cntRead);
+      if (NS_SUCCEEDED(rv) && cntRead == fromLen)
+      {
+        PR_snprintf(statusLine, sizeof(statusLine), X_MOZILLA_STATUS2_FORMAT MSG_LINEBREAK, msgFlags & 0xFFFF0000);
+        rv = m_dstOutputStream->Write(statusLine, strlen(statusLine), &cntRead);
+      }
+      if (NS_FAILED(rv) || (cntRead != fromLen)) {
+        ReportWriteError(m_dstFolder);
+        return false;
+      }
+      first = false;
+    }
+
+    // Handle generic data, escape any lines that begin with "From "
+    pIn = m_pInBuffer + inIdx;
+    numBytes -= cnt;
+    m_didBytes += cnt;
+    pStart = pIn;
+    cnt -= inIdx;
+    inIdx = 0;
+    while (cnt) {
+      if (*pIn == nsCRT::CR) {
+        // need more in buffer?
+        if ((cnt < 7) && numBytes)
+          break;
+
+        if (cnt > 6) {
+          if ((pIn[1] == nsCRT::LF) && IsFromLineKey(pIn + 2, cnt)) {
+            inIdx += 2;
+            // Match, escape it
+            rv = m_dstOutputStream->Write((const char *)pStart, (int32_t)inIdx, &cntRead);
+            if (NS_SUCCEEDED(rv) && (cntRead == (int32_t)inIdx))
+              rv = m_dstOutputStream->Write(">", 1, &cntRead);
+            if (NS_FAILED(rv) || (cntRead != 1)) {
+              ReportWriteError(m_dstFolder);
+              return false;
+            }
+
+            cnt -= 2;
+            pIn += 2;
+            inIdx = 0;
+            pStart = pIn;
+            continue;
+          }
+        }
+      } // == nsCRT::CR
+
+      cnt--;
+      inIdx++;
+      pIn++;
+    }
+    rv = m_dstOutputStream->Write((const char *)pStart, (int32_t)inIdx, &cntRead);
+    if (NS_FAILED(rv) || (cntRead != (int32_t)inIdx)) {
+      ReportWriteError(m_dstFolder);
+      return false;
+    }
+
+    if (cnt) {
+      inIdx = cnt;
+      memcpy(m_pInBuffer, pIn, cnt);
+    }
+    else
+      inIdx = 0;
+  }
+
+  // I used to check for an eol before writing one but
+  // it turns out that adding a proper EOL before the next
+  // separator never really hurts so better to be safe
+  // and always do it.
+  //  if ((last[0] != nsCRT::CR) || (last[1] != nsCRT::LF)) {
+  rv = m_dstOutputStream->Write("\x0D\x0A", 2, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != 2)) {
+    ReportWriteError(m_dstFolder);
+    return false;
+  }
+  //  } // != nsCRT::CR || != nsCRT::LF
+
+  return true;
+}
+
+CIndexScanner::CIndexScanner(nsString& name, nsIFile * idxFile,
+                             nsIFile * mbxFile, nsIMsgFolder * dstFolder)
+  : CMbxScanner( name, mbxFile, dstFolder)
+{
+  m_idxFile = idxFile;
+  m_curItemIndex = 0;
+  m_idxOffset = 0;
+}
+
+CIndexScanner::~CIndexScanner()
+{
+  CleanUp();
+}
+
+bool CIndexScanner::Initialize(void)
+{
+  if (!CMbxScanner::Initialize())
+    return false;
+
+
+  nsresult   rv = NS_NewLocalFileInputStream(getter_AddRefs(m_idxFileInputStream), m_idxFile);
+  if (NS_FAILED(rv)) {
+    CleanUp();
+    return false;
+  }
+
+  return true;
+}
+
+bool CIndexScanner::ValidateIdxFile(void)
+{
+  int8_t      id[4];
+  int32_t      cnt = 4;
+  nsresult    rv;
+  uint32_t      cntRead;
+  int8_t *    pReadTo;
+
+  pReadTo = id;
+  rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != cnt))
+    return false;
+  if ((id[0] != 'J') || (id[1] != 'M') || (id[2] != 'F') || (id[3] != '9'))
+    return false;
+  cnt = 4;
+  uint32_t    subId;
+  pReadTo = (int8_t *) &subId;
+  rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != cnt))
+    return false;
+  if (subId != 0x00010004) {
+    IMPORT_LOG1("Idx file subid doesn't match: 0x%lx\n", subId);
+    return false;
+  }
+
+  pReadTo = (int8_t *) &m_numMessages;
+  rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != cnt))
+    return false;
+
+  IMPORT_LOG1("Idx file num messages: %ld\n", m_numMessages);
+
+  m_didBytes += 80;
+  m_idxOffset = 80;
+  return true;
+}
+
+/*
+Idx file...
+Header is 80 bytes, JMF9, subId? 0x00010004, numMessages, fileSize, 1, 0x00010010
+Entries start at byte 80
+4 byte numbers
+Flags? maybe
+?? who knows
+index
+start of this entry in the file
+length of this record
+msg offset in mbx
+msg length in mbx
+
+*/
+
+// #define DEBUG_SUBJECT_AND_FLAGS  1
+#define  kNumIdxLongsToRead    7
+
+bool CIndexScanner::GetMailItem(uint32_t *pFlags, uint32_t *pOffset, uint32_t *pSize)
+{
+  uint32_t  values[kNumIdxLongsToRead];
+  int32_t    cnt = kNumIdxLongsToRead * sizeof(uint32_t);
+  int8_t *  pReadTo = (int8_t *) values;
+  uint32_t    cntRead;
+  nsresult  rv;
+
+  nsCOMPtr <nsISeekableStream> seekableStream = do_QueryInterface(m_idxFileInputStream);
+  rv = seekableStream->Seek(nsISeekableStream::NS_SEEK_SET, m_idxOffset);
+  if (NS_FAILED(rv))
+    return false;
+
+  rv = m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead);
+  if (NS_FAILED(rv) || (cntRead != cnt))
+    return false;
+
+  if (values[3] != m_idxOffset) {
+    IMPORT_LOG2("Self pointer invalid: m_idxOffset=0x%lx, self=0x%lx\n", m_idxOffset, values[3]);
+    return false;
+  }
+
+  // So... what do we have here???
+#ifdef DEBUG_SUBJECT_AND_FLAGS
+  IMPORT_LOG2("Number: %ld, msg offset: 0x%lx, ", values[2], values[5]);
+  IMPORT_LOG2("msg length: %ld, Flags: 0x%lx\n", values[6], values[0]);
+  seekableStream->Seek(nsISeekableStream::NS_SEEK_SET, m_idxOffset + 212);
+  uint32_t  subSz = 0;
+  cnt = 4;
+  pReadTo = (int8_t *) &subSz;
+  m_idxFileInputStream->Read((char *) pReadTo, cnt, &cntRead);
+  if ((subSz >= 0) && (subSz < 1024)) {
+    char *pSub = new char[subSz + 1];
+    m_idxFileInputStream->Read(pSub, subSz, &cntRead);
+    pSub[subSz] = 0;
+    IMPORT_LOG1("    Subject: %s\n", pSub);
+    delete [] pSub;
+  }
+#endif
+
+  m_idxOffset += values[4];
+  m_didBytes += values[4];
+
+  *pFlags = values[0];
+  *pOffset = values[5];
+  *pSize = values[6];
+  return true;
+}
+
+#define  kOEDeletedFlag    0x0001
+
+bool CIndexScanner::DoWork(bool *pAbort, uint32_t *pDone, uint32_t *pCount)
+{
+  m_didBytes = 0;
+  if (!ValidateIdxFile())
+    return false;
+
+  bool    failed = false;
+  while ((m_curItemIndex < m_numMessages) && !failed && !(*pAbort)) {
+    uint32_t  flags, offset, size;
+    if (!GetMailItem(&flags, &offset, &size)) {
+      CleanUp();
+      return false;
+    }
+    m_curItemIndex++;
+    if (!(flags & kOEDeletedFlag)) {
+      if (!WriteMailItem(flags, offset, size))
+        failed = true;
+      else {
+        m_msgCount++;
+      }
+    }
+    m_didBytes += size;
+    if (pDone)
+      *pDone = m_didBytes;
+    if (pCount)
+      *pCount = m_msgCount;
+  }
+
+  CleanUp();
+  return !failed;
+}
+
+
+void CIndexScanner::CleanUp(void)
+{
+  CMbxScanner::CleanUp();
+  m_idxFileInputStream->Close();
+}