1) {
+ --prevBoundary;
+ }
}
} else {
const UChar *p=src-1;
@@ -1748,7 +2256,8 @@ Normalizer2Impl::makeFCD(const UChar *src, const UChar *limit,
* The source text does not fulfill the conditions for FCD.
* Decompose and reorder a limited piece of the text.
*/
- if(!decomposeShort(prevBoundary, src, *buffer, errorCode)) {
+ decomposeShort(prevBoundary, src, FALSE, FALSE, *buffer, errorCode);
+ if (U_FAILURE(errorCode)) {
break;
}
prevBoundary=src;
@@ -1792,16 +2301,33 @@ void Normalizer2Impl::makeFCDAndAppend(const UChar *src, const UChar *limit,
}
const UChar *Normalizer2Impl::findPreviousFCDBoundary(const UChar *start, const UChar *p) const {
- while(start0xff) {}
+ while(start
add(firstOrigin);
@@ -1845,35 +2372,47 @@ void CanonIterData::addToStartSet(UChar32 origin, UChar32 decompLead, UErrorCode
}
}
+// C++ class for friend access to private Normalizer2Impl members.
+class InitCanonIterData {
+public:
+ static void doInit(Normalizer2Impl *impl, UErrorCode &errorCode);
+};
+
U_CDECL_BEGIN
-// Call Normalizer2Impl::makeCanonIterDataFromNorm16() for a range of same-norm16 characters.
-// context: the Normalizer2Impl
-static UBool U_CALLCONV
-enumCIDRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) {
- UErrorCode errorCode = U_ZERO_ERROR;
- if (value != 0) {
- Normalizer2Impl *impl = (Normalizer2Impl *)context;
- impl->makeCanonIterDataFromNorm16(
- start, end, (uint16_t)value, *impl->fCanonIterData, errorCode);
- }
- return U_SUCCESS(errorCode);
+// UInitOnce instantiation function for CanonIterData
+static void U_CALLCONV
+initCanonIterData(Normalizer2Impl *impl, UErrorCode &errorCode) {
+ InitCanonIterData::doInit(impl, errorCode);
}
+U_CDECL_END
-
-// UInitOnce instantiation function for CanonIterData
-
-static void U_CALLCONV
-initCanonIterData(Normalizer2Impl *impl, UErrorCode &errorCode) {
+void InitCanonIterData::doInit(Normalizer2Impl *impl, UErrorCode &errorCode) {
U_ASSERT(impl->fCanonIterData == NULL);
impl->fCanonIterData = new CanonIterData(errorCode);
if (impl->fCanonIterData == NULL) {
errorCode=U_MEMORY_ALLOCATION_ERROR;
}
if (U_SUCCESS(errorCode)) {
- utrie2_enum(impl->getNormTrie(), NULL, enumCIDRangeHandler, impl);
- utrie2_freeze(impl->fCanonIterData->trie, UTRIE2_32_VALUE_BITS, &errorCode);
+ UChar32 start = 0, end;
+ uint32_t value;
+ while ((end = ucptrie_getRange(impl->normTrie, start,
+ UCPMAP_RANGE_FIXED_LEAD_SURROGATES, Normalizer2Impl::INERT,
+ nullptr, nullptr, &value)) >= 0) {
+ // Call Normalizer2Impl::makeCanonIterDataFromNorm16() for a range of same-norm16 characters.
+ if (value != Normalizer2Impl::INERT) {
+ impl->makeCanonIterDataFromNorm16(start, end, value, *impl->fCanonIterData, errorCode);
+ }
+ start = end + 1;
+ }
+#ifdef UCPTRIE_DEBUG
+ umutablecptrie_setName(impl->fCanonIterData->mutableTrie, "CanonIterData");
+#endif
+ impl->fCanonIterData->trie = umutablecptrie_buildImmutable(
+ impl->fCanonIterData->mutableTrie, UCPTRIE_TYPE_SMALL, UCPTRIE_VALUE_BITS_32, &errorCode);
+ umutablecptrie_close(impl->fCanonIterData->mutableTrie);
+ impl->fCanonIterData->mutableTrie = nullptr;
}
if (U_FAILURE(errorCode)) {
delete impl->fCanonIterData;
@@ -1881,12 +2420,10 @@ initCanonIterData(Normalizer2Impl *impl, UErrorCode &errorCode) {
}
}
-U_CDECL_END
-
-void Normalizer2Impl::makeCanonIterDataFromNorm16(UChar32 start, UChar32 end, uint16_t norm16,
+void Normalizer2Impl::makeCanonIterDataFromNorm16(UChar32 start, UChar32 end, const uint16_t norm16,
CanonIterData &newData,
UErrorCode &errorCode) const {
- if(norm16==0 || (minYesNo<=norm16 && norm16=minMaybeYes) {
+ if(isMaybeOrNonZeroCC(norm16)) {
// not a segment starter if it occurs in a decomposition or has cc!=0
newValue|=CANON_NOT_SEGMENT_STARTER;
if(norm16 minYesNo) {
// c decomposes, get everything from the variable-length extra data
const uint16_t *mapping=getMapping(norm16_2);
uint16_t firstUnit=*mapping;
@@ -1937,10 +2478,10 @@ void Normalizer2Impl::makeCanonIterDataFromNorm16(UChar32 start, UChar32 end, ui
if(norm16_2>=minNoNo) {
while(itrie, c);
+ return (int32_t)ucptrie_get(fCanonIterData->trie, c);
}
const UnicodeSet &Normalizer2Impl::getCanonStartSet(int32_t n) const {
@@ -1988,7 +2529,7 @@ UBool Normalizer2Impl::getCanonStartSet(UChar32 c, UnicodeSet &set) const {
set.add(value);
}
if((canonValue&CANON_HAS_COMPOSITIONS)!=0) {
- uint16_t norm16=getNorm16(c);
+ uint16_t norm16=getRawNorm16(c);
if(norm16==JAMO_L) {
UChar32 syllable=
(UChar32)(Hangul::HANGUL_BASE+(c-Hangul::JAMO_L_BASE)*Hangul::JAMO_VT_COUNT);
@@ -2017,7 +2558,7 @@ unorm2_swap(const UDataSwapper *ds,
uint8_t *outBytes;
const int32_t *inIndexes;
- int32_t indexes[Normalizer2Impl::IX_MIN_MAYBE_YES+1];
+ int32_t indexes[Normalizer2Impl::IX_TOTAL_SIZE+1];
int32_t i, offset, nextOffset, size;
@@ -2029,12 +2570,13 @@ unorm2_swap(const UDataSwapper *ds,
/* check data format and format version */
pInfo=(const UDataInfo *)((const char *)inData+4);
+ uint8_t formatVersion0=pInfo->formatVersion[0];
if(!(
pInfo->dataFormat[0]==0x4e && /* dataFormat="Nrm2" */
pInfo->dataFormat[1]==0x72 &&
pInfo->dataFormat[2]==0x6d &&
pInfo->dataFormat[3]==0x32 &&
- (pInfo->formatVersion[0]==1 || pInfo->formatVersion[0]==2)
+ (1<=formatVersion0 && formatVersion0<=4)
)) {
udata_printError(ds, "unorm2_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as Normalizer2 data\n",
pInfo->dataFormat[0], pInfo->dataFormat[1],
@@ -2048,10 +2590,18 @@ unorm2_swap(const UDataSwapper *ds,
outBytes=(uint8_t *)outData+headerSize;
inIndexes=(const int32_t *)inBytes;
+ int32_t minIndexesLength;
+ if(formatVersion0==1) {
+ minIndexesLength=Normalizer2Impl::IX_MIN_MAYBE_YES+1;
+ } else if(formatVersion0==2) {
+ minIndexesLength=Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY+1;
+ } else {
+ minIndexesLength=Normalizer2Impl::IX_MIN_LCCC_CP+1;
+ }
if(length>=0) {
length-=headerSize;
- if(length<(int32_t)sizeof(indexes)) {
+ if(lengthswapArray32(ds, inBytes, nextOffset-offset, outBytes, pErrorCode);
offset=nextOffset;
- /* swap the UTrie2 */
+ /* swap the trie */
nextOffset=indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET];
- utrie2_swap(ds, inBytes+offset, nextOffset-offset, outBytes+offset, pErrorCode);
+ utrie_swapAnyVersion(ds, inBytes+offset, nextOffset-offset, outBytes+offset, pErrorCode);
offset=nextOffset;
/* swap the uint16_t extraData[] */
diff --git a/intl/icu/source/common/normalizer2impl.h b/intl/icu/source/common/normalizer2impl.h
index a6bf967979..2e6aff3088 100644
--- a/intl/icu/source/common/normalizer2impl.h
+++ b/intl/icu/source/common/normalizer2impl.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: normalizer2impl.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -24,17 +24,30 @@
#if !UCONFIG_NO_NORMALIZATION
#include "unicode/normalizer2.h"
+#include "unicode/ucptrie.h"
#include "unicode/unistr.h"
#include "unicode/unorm.h"
+#include "unicode/utf.h"
#include "unicode/utf16.h"
#include "mutex.h"
+#include "udataswp.h"
#include "uset_imp.h"
-#include "utrie2.h"
+
+// When the nfc.nrm data is *not* hardcoded into the common library
+// (with this constant set to 0),
+// then it needs to be built into the data package:
+// Add nfc.nrm to icu4c/source/data/Makefile.in DAT_FILES_SHORT
+#define NORM2_HARDCODE_NFC_DATA 1
U_NAMESPACE_BEGIN
struct CanonIterData;
+class ByteSink;
+class Edits;
+class InitCanonIterData;
+class LcccContext;
+
class U_COMMON_API Hangul {
public:
/* Korean Hangul and Jamo constants */
@@ -63,9 +76,9 @@ public:
return HANGUL_BASE<=c && c=MIN_NORMAL_MAYBE_YES) {
- return (uint8_t)norm16;
+ return getCCFromNormalYesOrMaybe(norm16);
}
if(norm16> OFFSET_SHIFT);
+ }
static uint8_t getCCFromYesOrMaybe(uint16_t norm16) {
- return norm16>=MIN_NORMAL_MAYBE_YES ? (uint8_t)norm16 : 0;
+ return norm16>=MIN_NORMAL_MAYBE_YES ? getCCFromNormalYesOrMaybe(norm16) : 0;
+ }
+ uint8_t getCCFromYesOrMaybeCP(UChar32 c) const {
+ if (c < minCompNoMaybeCP) { return 0; }
+ return getCCFromYesOrMaybe(getNorm16(c));
}
/**
@@ -272,10 +307,8 @@ public:
* @return The lccc(c) in bits 15..8 and tccc(c) in bits 7..0.
*/
uint16_t getFCD16(UChar32 c) const {
- if(c<0) {
+ if(c1) for quick FCC boundary-after tests.
+ DELTA_TCCC_0=0,
+ DELTA_TCCC_1=2,
+ DELTA_TCCC_GT_1=4,
+ DELTA_TCCC_MASK=6,
+ DELTA_SHIFT=3,
- enum {
- MIN_YES_YES_WITH_CC=0xff01,
- JAMO_VT=0xff00,
- MIN_NORMAL_MAYBE_YES=0xfe00,
- JAMO_L=1,
MAX_DELTA=0x40
};
@@ -394,21 +431,32 @@ public:
IX_MIN_COMP_NO_MAYBE_CP,
// Norm16 value thresholds for quick check combinations and types of extra data.
- IX_MIN_YES_NO, // Mappings & compositions in [minYesNo..minYesNoMappingsOnly[.
+
+ /** Mappings & compositions in [minYesNo..minYesNoMappingsOnly[. */
+ IX_MIN_YES_NO,
+ /** Mappings are comp-normalized. */
IX_MIN_NO_NO,
IX_LIMIT_NO_NO,
IX_MIN_MAYBE_YES,
- IX_MIN_YES_NO_MAPPINGS_ONLY, // Mappings only in [minYesNoMappingsOnly..minNoNo[.
+ /** Mappings only in [minYesNoMappingsOnly..minNoNo[. */
+ IX_MIN_YES_NO_MAPPINGS_ONLY,
+ /** Mappings are not comp-normalized but have a comp boundary before. */
+ IX_MIN_NO_NO_COMP_BOUNDARY_BEFORE,
+ /** Mappings do not have a comp boundary before. */
+ IX_MIN_NO_NO_COMP_NO_MAYBE_CC,
+ /** Mappings to the empty string. */
+ IX_MIN_NO_NO_EMPTY,
- IX_RESERVED15,
+ IX_MIN_LCCC_CP,
+ IX_RESERVED19,
IX_COUNT
};
enum {
MAPPING_HAS_CCC_LCCC_WORD=0x80,
MAPPING_HAS_RAW_MAPPING=0x40,
- MAPPING_NO_COMP_BOUNDARY_AFTER=0x20,
+ // unused bit 0x20,
MAPPING_LENGTH_MASK=0x1f
};
@@ -457,6 +505,12 @@ public:
UnicodeString &safeMiddle,
ReorderingBuffer &buffer,
UErrorCode &errorCode) const;
+
+ /** sink==nullptr: isNormalized() */
+ UBool composeUTF8(uint32_t options, UBool onlyContiguous,
+ const uint8_t *src, const uint8_t *limit,
+ ByteSink *sink, icu::Edits *edits, UErrorCode &errorCode) const;
+
const UChar *makeFCD(const UChar *src, const UChar *limit,
ReorderingBuffer *buffer, UErrorCode &errorCode) const;
void makeFCDAndAppend(const UChar *src, const UChar *limit,
@@ -465,27 +519,42 @@ public:
ReorderingBuffer &buffer,
UErrorCode &errorCode) const;
- UBool hasDecompBoundary(UChar32 c, UBool before) const;
+ UBool hasDecompBoundaryBefore(UChar32 c) const;
+ UBool norm16HasDecompBoundaryBefore(uint16_t norm16) const;
+ UBool hasDecompBoundaryAfter(UChar32 c) const;
+ UBool norm16HasDecompBoundaryAfter(uint16_t norm16) const;
UBool isDecompInert(UChar32 c) const { return isDecompYesAndZeroCC(getNorm16(c)); }
UBool hasCompBoundaryBefore(UChar32 c) const {
- return c=minMaybeYes; }
- static UBool isInert(uint16_t norm16) { return norm16==0; }
- static UBool isJamoL(uint16_t norm16) { return norm16==1; }
+ static UBool isInert(uint16_t norm16) { return norm16==INERT; }
+ static UBool isJamoL(uint16_t norm16) { return norm16==JAMO_L; }
static UBool isJamoVT(uint16_t norm16) { return norm16==JAMO_VT; }
- UBool isHangul(uint16_t norm16) const { return norm16==minYesNo; }
+ uint16_t hangulLVT() const { return minYesNoMappingsOnly|HAS_COMP_BOUNDARY_AFTER; }
+ UBool isHangulLV(uint16_t norm16) const { return norm16==minYesNo; }
+ UBool isHangulLVT(uint16_t norm16) const {
+ return norm16==hangulLVT();
+ }
UBool isCompYesAndZeroCC(uint16_t norm16) const { return norm16=MIN_YES_YES_WITH_CC || norm16=MIN_YES_YES_WITH_CC ? (uint8_t)norm16 : 0;
+ // return norm16>=MIN_YES_YES_WITH_CC ? getCCFromNormalYesOrMaybe(norm16) : 0;
// }
uint8_t getCCFromNoNo(uint16_t norm16) const {
const uint16_t *mapping=getMapping(norm16);
@@ -525,30 +594,47 @@ private:
}
}
// requires that the [cpStart..cpLimit[ character passes isCompYesAndZeroCC()
- uint8_t getTrailCCFromCompYesAndZeroCC(const UChar *cpStart, const UChar *cpLimit) const;
+ uint8_t getTrailCCFromCompYesAndZeroCC(uint16_t norm16) const {
+ if(norm16<=minYesNo) {
+ return 0; // yesYes and Hangul LV have ccc=tccc=0
+ } else {
+ // For Hangul LVT we harmlessly fetch a firstUnit with tccc=0 here.
+ return (uint8_t)(*getMapping(norm16)>>8); // tccc from yesNo
+ }
+ }
+ uint8_t getPreviousTrailCC(const UChar *start, const UChar *p) const;
+ uint8_t getPreviousTrailCC(const uint8_t *start, const uint8_t *p) const;
// Requires algorithmic-NoNo.
UChar32 mapAlgorithmic(UChar32 c, uint16_t norm16) const {
- return c+norm16-(minMaybeYes-MAX_DELTA-1);
+ return c+(norm16>>DELTA_SHIFT)-centerNoNoDelta;
+ }
+ UChar32 getAlgorithmicDelta(uint16_t norm16) const {
+ return (norm16>>DELTA_SHIFT)-centerNoNoDelta;
}
// Requires minYesNo>OFFSET_SHIFT); }
const uint16_t *getCompositionsListForDecompYes(uint16_t norm16) const {
- if(norm16==0 || MIN_NORMAL_MAYBE_YES<=norm16) {
+ if(norm16>OFFSET_SHIFT);
+ }
/**
* @param c code point must have compositions
* @return compositions list pointer
@@ -563,46 +649,78 @@ private:
UChar32 minNeedDataCP,
ReorderingBuffer *buffer,
UErrorCode &errorCode) const;
- UBool decomposeShort(const UChar *src, const UChar *limit,
- ReorderingBuffer &buffer, UErrorCode &errorCode) const;
+ const UChar *decomposeShort(const UChar *src, const UChar *limit,
+ UBool stopAtCompBoundary, UBool onlyContiguous,
+ ReorderingBuffer &buffer, UErrorCode &errorCode) const;
UBool decompose(UChar32 c, uint16_t norm16,
ReorderingBuffer &buffer, UErrorCode &errorCode) const;
+ const uint8_t *decomposeShort(const uint8_t *src, const uint8_t *limit,
+ UBool stopAtCompBoundary, UBool onlyContiguous,
+ ReorderingBuffer &buffer, UErrorCode &errorCode) const;
+
static int32_t combine(const uint16_t *list, UChar32 trail);
void addComposites(const uint16_t *list, UnicodeSet &set) const;
void recompose(ReorderingBuffer &buffer, int32_t recomposeStartIndex,
UBool onlyContiguous) const;
- UBool hasCompBoundaryBefore(UChar32 c, uint16_t norm16) const;
- const UChar *findPreviousCompBoundary(const UChar *start, const UChar *p) const;
- const UChar *findNextCompBoundary(const UChar *p, const UChar *limit) const;
+ UBool hasCompBoundaryBefore(UChar32 c, uint16_t norm16) const {
+ return ctrie, start, end, (uint32_t)rowIndex, TRUE, pErrorCode);
diff --git a/intl/icu/source/common/propsvec.h b/intl/icu/source/common/propsvec.h
index b34e4ee8ff..39080615ea 100644
--- a/intl/icu/source/common/propsvec.h
+++ b/intl/icu/source/common/propsvec.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: propsvec.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/punycode.cpp b/intl/icu/source/common/punycode.cpp
index 1d2ccb342e..90fe1ec3c8 100644
--- a/intl/icu/source/common/punycode.cpp
+++ b/intl/icu/source/common/punycode.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: punycode.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/punycode.h b/intl/icu/source/common/punycode.h
index ff23eb0c37..5d8a243175 100644
--- a/intl/icu/source/common/punycode.h
+++ b/intl/icu/source/common/punycode.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: punycode.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/putil.cpp b/intl/icu/source/common/putil.cpp
index 03f6548689..769309b422 100644
--- a/intl/icu/source/common/putil.cpp
+++ b/intl/icu/source/common/putil.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -43,8 +43,24 @@
// Must be before any other #includes.
#include "uposixdefs.h"
-/* include ICU headers */
-#include "unicode/utypes.h"
+// First, the platform type. Need this for U_PLATFORM.
+#include "unicode/platform.h"
+
+#if U_PLATFORM == U_PF_MINGW && defined __STRICT_ANSI__
+/* tzset isn't defined in strict ANSI on MinGW. */
+#undef __STRICT_ANSI__
+#endif
+
+/*
+ * Cygwin with GCC requires inclusion of time.h after the above disabling strict asci mode statement.
+ */
+#include
+
+#if !U_PLATFORM_USES_ONLY_WIN32_API
+#include
+#endif
+
+/* include the rest of the ICU headers */
#include "unicode/putil.h"
#include "unicode/ustring.h"
#include "putilimp.h"
@@ -76,14 +92,28 @@
* Should Cygwin be included as well (U_PLATFORM_HAS_WIN32_API)
* to use native APIs as much as possible?
*/
+#ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
+#endif
# define VC_EXTRALEAN
# define NOUSER
# define NOSERVICE
# define NOIME
# define NOMCX
# include
+# include "unicode/uloc.h"
# include "wintz.h"
+#if U_PLATFORM_HAS_WINUWP_API
+typedef PVOID LPMSG; // TODO: figure out how to get rid of this typedef
+#include
+#include
+#include
+#include
+
+using namespace ABI::Windows::Foundation;
+using namespace Microsoft::WRL;
+using namespace Microsoft::WRL::Wrappers;
+#endif
#elif U_PLATFORM == U_PF_OS400
# include
# include /* error code structure */
@@ -104,20 +134,6 @@
# include
#endif
-#if (U_PF_MINGW <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN) && defined(__STRICT_ANSI__)
-/* tzset isn't defined in strict ANSI on Cygwin and MinGW. */
-#undef __STRICT_ANSI__
-#endif
-
-/*
- * Cygwin with GCC requires inclusion of time.h after the above disabling strict asci mode statement.
- */
-#include
-
-#if !U_PLATFORM_USES_ONLY_WIN32_API
-#include
-#endif
-
/*
* Only include langinfo.h if we have a way to get the codeset. If we later
* depend on more feature, we can test on U_HAVE_NL_LANGINFO.
@@ -516,6 +532,28 @@ uprv_fmin(double x, double y)
return (x > y ? y : x);
}
+U_CAPI UBool U_EXPORT2
+uprv_add32_overflow(int32_t a, int32_t b, int32_t* res) {
+ // NOTE: Some compilers (GCC, Clang) have primitives available, like __builtin_add_overflow.
+ // This function could be optimized by calling one of those primitives.
+ auto a64 = static_cast(a);
+ auto b64 = static_cast(b);
+ int64_t res64 = a64 + b64;
+ *res = static_cast(res64);
+ return res64 != *res;
+}
+
+U_CAPI UBool U_EXPORT2
+uprv_mul32_overflow(int32_t a, int32_t b, int32_t* res) {
+ // NOTE: Some compilers (GCC, Clang) have primitives available, like __builtin_mul_overflow.
+ // This function could be optimized by calling one of those primitives.
+ auto a64 = static_cast(a);
+ auto b64 = static_cast(b);
+ int64_t res64 = a64 * b64;
+ *res = static_cast(res64);
+ return res64 != *res;
+}
+
/**
* Truncates the given double.
* trunc(3.3) = 3.0, trunc (-3.3) = -3.0
@@ -651,13 +689,23 @@ uprv_timezone()
/* Note that U_TZNAME does *not* have to be tzname, but if it is,
some platforms need to have it declared here. */
-#if defined(U_TZNAME) && (U_PLATFORM == U_PF_IRIX || U_PLATFORM_IS_DARWIN_BASED || (U_PLATFORM == U_PF_CYGWIN && !U_PLATFORM_USES_ONLY_WIN32_API))
+#if defined(U_TZNAME) && (U_PLATFORM == U_PF_IRIX || U_PLATFORM_IS_DARWIN_BASED)
/* RS6000 and others reject char **tzname. */
extern U_IMPORT char *U_TZNAME[];
#endif
#if !UCONFIG_NO_FILE_IO && ((U_PLATFORM_IS_DARWIN_BASED && (U_PLATFORM != U_PF_IPHONE || defined(U_TIMEZONE))) || U_PLATFORM_IS_LINUX_BASED || U_PLATFORM == U_PF_BSD || U_PLATFORM == U_PF_SOLARIS)
/* These platforms are likely to use Olson timezone IDs. */
+/* common targets of the symbolic link at TZDEFAULT are:
+ * "/usr/share/zoneinfo/" default, older Linux distros, macOS to 10.12
+ * "../usr/share/zoneinfo/" newer Linux distros: Red Hat Enterprise Linux 7, Ubuntu 16, SuSe Linux 12
+ * "/usr/share/lib/zoneinfo/" Solaris
+ * "../usr/share/lib/zoneinfo/" Solaris
+ * "/var/db/timezone/zoneinfo/" macOS 10.13
+ * To avoid checking lots of paths, just check that the target path
+ * before the ends with "/zoneinfo/", and the is valid.
+ */
+
#define CHECK_LOCALTIME_LINK 1
#if U_PLATFORM_IS_DARWIN_BASED
#include
@@ -665,12 +713,12 @@ extern U_IMPORT char *U_TZNAME[];
#elif U_PLATFORM == U_PF_SOLARIS
#define TZDEFAULT "/etc/localtime"
#define TZZONEINFO "/usr/share/lib/zoneinfo/"
-#define TZZONEINFO2 "../usr/share/lib/zoneinfo/"
#define TZ_ENV_CHECK "localtime"
#else
#define TZDEFAULT "/etc/localtime"
#define TZZONEINFO "/usr/share/zoneinfo/"
#endif
+#define TZZONEINFOTAIL "/zoneinfo/"
#if U_HAVE_DIRENT_H
#define TZFILE_SKIP "posixrules" /* tz file to skip when searching. */
/* Some Linux distributions have 'localtime' in /usr/share/zoneinfo
@@ -922,30 +970,30 @@ static CharString *gSearchTZFileResult = NULL;
* This function is not thread safe - it uses a global, gSearchTZFileResult, to hold its results.
*/
static char* searchForTZFile(const char* path, DefaultTZInfo* tzInfo) {
- DIR* dirp = opendir(path);
- DIR* subDirp = NULL;
+ DIR* dirp = NULL;
struct dirent* dirEntry = NULL;
-
char* result = NULL;
+ UErrorCode status = U_ZERO_ERROR;
+
+ /* Save the current path */
+ CharString curpath(path, -1, status);
+ if (U_FAILURE(status)) {
+ goto cleanupAndReturn;
+ }
+
+ dirp = opendir(path);
if (dirp == NULL) {
- return result;
+ goto cleanupAndReturn;
}
if (gSearchTZFileResult == NULL) {
gSearchTZFileResult = new CharString;
if (gSearchTZFileResult == NULL) {
- return NULL;
+ goto cleanupAndReturn;
}
ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup);
}
- /* Save the current path */
- UErrorCode status = U_ZERO_ERROR;
- CharString curpath(path, -1, status);
- if (U_FAILURE(status)) {
- return NULL;
- }
-
/* Check each entry in the directory. */
while((dirEntry = readdir(dirp)) != NULL) {
const char* dirName = dirEntry->d_name;
@@ -954,15 +1002,16 @@ static char* searchForTZFile(const char* path, DefaultTZInfo* tzInfo) {
CharString newpath(curpath, status);
newpath.append(dirName, -1, status);
if (U_FAILURE(status)) {
- return NULL;
+ break;
}
+ DIR* subDirp = NULL;
if ((subDirp = opendir(newpath.data())) != NULL) {
/* If this new path is a directory, make a recursive call with the newpath. */
closedir(subDirp);
newpath.append('/', status);
if (U_FAILURE(status)) {
- return NULL;
+ break;
}
result = searchForTZFile(newpath.data(), tzInfo);
/*
@@ -986,7 +1035,7 @@ static char* searchForTZFile(const char* path, DefaultTZInfo* tzInfo) {
gSearchTZFileResult->clear();
gSearchTZFileResult->append(zoneid, -1, status);
if (U_FAILURE(status)) {
- return NULL;
+ break;
}
result = gSearchTZFileResult->data();
/* Get out after the first one found. */
@@ -995,7 +1044,11 @@ static char* searchForTZFile(const char* path, DefaultTZInfo* tzInfo) {
}
}
}
- closedir(dirp);
+
+ cleanupAndReturn:
+ if (dirp) {
+ closedir(dirp);
+ }
return result;
}
#endif
@@ -1011,6 +1064,7 @@ uprv_tzname_clear_cache()
U_CAPI const char* U_EXPORT2
uprv_tzname(int n)
{
+ (void)n; // Avoid unreferenced parameter warning.
const char *tzid = NULL;
#if U_PLATFORM_USES_ONLY_WIN32_API
tzid = uprv_detectWindowsTimeZone();
@@ -1018,6 +1072,15 @@ uprv_tzname(int n)
if (tzid != NULL) {
return tzid;
}
+
+#ifndef U_TZNAME
+ // The return value is free'd in timezone.cpp on Windows because
+ // the other code path returns a pointer to a heap location.
+ // If we don't have a name already, then tzname wouldn't be any
+ // better, so just fall back.
+ return uprv_strdup("Etc/UTC");
+#endif // !U_TZNAME
+
#else
/*#if U_PLATFORM_IS_DARWIN_BASED
@@ -1059,24 +1122,15 @@ uprv_tzname(int n)
*/
int32_t ret = (int32_t)readlink(TZDEFAULT, gTimeZoneBuffer, sizeof(gTimeZoneBuffer)-1);
if (0 < ret) {
- int32_t tzZoneInfoLen = uprv_strlen(TZZONEINFO);
+ int32_t tzZoneInfoTailLen = uprv_strlen(TZZONEINFOTAIL);
gTimeZoneBuffer[ret] = 0;
- if (uprv_strncmp(gTimeZoneBuffer, TZZONEINFO, tzZoneInfoLen) == 0
- && isValidOlsonID(gTimeZoneBuffer + tzZoneInfoLen))
+ char * tzZoneInfoTailPtr = uprv_strstr(gTimeZoneBuffer, TZZONEINFOTAIL);
+
+ if (tzZoneInfoTailPtr != NULL
+ && isValidOlsonID(tzZoneInfoTailPtr + tzZoneInfoTailLen))
{
- return (gTimeZoneBufferPtr = gTimeZoneBuffer + tzZoneInfoLen);
+ return (gTimeZoneBufferPtr = tzZoneInfoTailPtr + tzZoneInfoTailLen);
}
-#if U_PLATFORM == U_PF_SOLARIS
- else
- {
- tzZoneInfoLen = uprv_strlen(TZZONEINFO2);
- if (uprv_strncmp(gTimeZoneBuffer, TZZONEINFO2, tzZoneInfoLen) == 0
- && isValidOlsonID(gTimeZoneBuffer + tzZoneInfoLen))
- {
- return (gTimeZoneBufferPtr = gTimeZoneBuffer + tzZoneInfoLen);
- }
- }
-#endif
} else {
#if defined(SEARCH_TZFILE)
DefaultTZInfo* tzInfo = (DefaultTZInfo*)uprv_malloc(sizeof(DefaultTZInfo));
@@ -1162,7 +1216,8 @@ UInitOnce gTimeZoneFilesInitOnce = U_INITONCE_INITIALIZER;
static CharString *gTimeZoneFilesDirectory = NULL;
#if U_POSIX_LOCALE || U_PLATFORM_USES_ONLY_WIN32_API
- static char *gCorrectedPOSIXLocale = NULL; /* Heap allocated */
+ static const char *gCorrectedPOSIXLocale = NULL; /* Sometimes heap allocated */
+ static bool gCorrectedPOSIXLocaleHeapAllocated = false;
#endif
static UBool U_CALLCONV putil_cleanup(void)
@@ -1183,9 +1238,10 @@ static UBool U_CALLCONV putil_cleanup(void)
#endif
#if U_POSIX_LOCALE || U_PLATFORM_USES_ONLY_WIN32_API
- if (gCorrectedPOSIXLocale) {
- uprv_free(gCorrectedPOSIXLocale);
+ if (gCorrectedPOSIXLocale && gCorrectedPOSIXLocaleHeapAllocated) {
+ uprv_free(const_cast(gCorrectedPOSIXLocale));
gCorrectedPOSIXLocale = NULL;
+ gCorrectedPOSIXLocaleHeapAllocated = false;
}
#endif
return TRUE;
@@ -1219,7 +1275,7 @@ u_setDataDirectory(const char *directory) {
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
{
char *p;
- while(p = uprv_strchr(newDataDir, U_FILE_ALT_SEP_CHAR)) {
+ while((p = uprv_strchr(newDataDir, U_FILE_ALT_SEP_CHAR)) != NULL) {
*p = U_FILE_SEP_CHAR;
}
}
@@ -1269,6 +1325,43 @@ uprv_pathIsAbsolute(const char *path)
# endif
#endif
+#if U_PLATFORM_HAS_WINUWP_API != 0
+// Helper function to get the ICU Data Directory under the Windows directory location.
+static BOOL U_CALLCONV getIcuDataDirectoryUnderWindowsDirectory(char* directoryBuffer, UINT bufferLength)
+{
+#if defined(ICU_DATA_DIR_WINDOWS)
+ wchar_t windowsPath[MAX_PATH];
+ char windowsPathUtf8[MAX_PATH];
+
+ UINT length = GetSystemWindowsDirectoryW(windowsPath, UPRV_LENGTHOF(windowsPath));
+ if ((length > 0) && (length < (UPRV_LENGTHOF(windowsPath) - 1))) {
+ // Convert UTF-16 to a UTF-8 string.
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t windowsPathUtf8Len = 0;
+ u_strToUTF8(windowsPathUtf8, static_cast(UPRV_LENGTHOF(windowsPathUtf8)),
+ &windowsPathUtf8Len, reinterpret_cast(windowsPath), -1, &status);
+
+ if (U_SUCCESS(status) && (status != U_STRING_NOT_TERMINATED_WARNING) &&
+ (windowsPathUtf8Len < (UPRV_LENGTHOF(windowsPathUtf8) - 1))) {
+ // Ensure it always has a separator, so we can append the ICU data path.
+ if (windowsPathUtf8[windowsPathUtf8Len - 1] != U_FILE_SEP_CHAR) {
+ windowsPathUtf8[windowsPathUtf8Len++] = U_FILE_SEP_CHAR;
+ windowsPathUtf8[windowsPathUtf8Len] = '\0';
+ }
+ // Check if the concatenated string will fit.
+ if ((windowsPathUtf8Len + UPRV_LENGTHOF(ICU_DATA_DIR_WINDOWS)) < bufferLength) {
+ uprv_strcpy(directoryBuffer, windowsPathUtf8);
+ uprv_strcat(directoryBuffer, ICU_DATA_DIR_WINDOWS);
+ return TRUE;
+ }
+ }
+ }
+#endif
+
+ return FALSE;
+}
+#endif
+
static void U_CALLCONV dataDirectoryInitFn() {
/* If we already have the directory, then return immediately. Will happen if user called
* u_setDataDirectory().
@@ -1297,7 +1390,9 @@ static void U_CALLCONV dataDirectoryInitFn() {
*/
# if !defined(ICU_NO_USER_DATA_OVERRIDE) && !UCONFIG_NO_FILE_IO
/* First try to get the environment variable */
- path=getenv("ICU_DATA");
+# if U_PLATFORM_HAS_WINUWP_API == 0 // Windows UWP does not support getenv
+ path=getenv("ICU_DATA");
+# endif
# endif
/* ICU_DATA_DIR may be set as a compile option.
@@ -1326,9 +1421,21 @@ static void U_CALLCONV dataDirectoryInitFn() {
}
#endif
+#if U_PLATFORM_HAS_WINUWP_API != 0 && defined(ICU_DATA_DIR_WINDOWS)
+ char datadir_path_buffer[MAX_PATH];
+ if (getIcuDataDirectoryUnderWindowsDirectory(datadir_path_buffer, UPRV_LENGTHOF(datadir_path_buffer))) {
+ path = datadir_path_buffer;
+ }
+#endif
+
if(path==NULL) {
/* It looks really bad, set it to something. */
+#if U_PLATFORM_HAS_WIN32_API
+ // Windows UWP will require icudtl.dat file in same directory as icuuc.dll
+ path = ".\\";
+#else
path = "";
+#endif
}
u_setDataDirectory(path);
@@ -1349,7 +1456,7 @@ static void setTimeZoneFilesDir(const char *path, UErrorCode &status) {
gTimeZoneFilesDirectory->append(path, status);
#if (U_FILE_SEP_CHAR != U_FILE_ALT_SEP_CHAR)
char *p = gTimeZoneFilesDirectory->data();
- while (p = uprv_strchr(p, U_FILE_ALT_SEP_CHAR)) {
+ while ((p = uprv_strchr(p, U_FILE_ALT_SEP_CHAR)) != NULL) {
*p = U_FILE_SEP_CHAR;
}
#endif
@@ -1366,15 +1473,30 @@ static void U_CALLCONV TimeZoneDataDirInitFn(UErrorCode &status) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
- const char *dir = getenv("ICU_TIMEZONE_FILES_DIR");
+
+ const char *dir = "";
+
+#if U_PLATFORM_HAS_WINUWP_API != 0
+ // The UWP version does not support the environment variable setting, but can possibly pick them up from the Windows directory.
+ char datadir_path_buffer[MAX_PATH];
+ if (getIcuDataDirectoryUnderWindowsDirectory(datadir_path_buffer, UPRV_LENGTHOF(datadir_path_buffer))) {
+ dir = datadir_path_buffer;
+ }
+#else
+ dir = getenv("ICU_TIMEZONE_FILES_DIR");
+#endif // U_PLATFORM_HAS_WINUWP_API
+
#if defined(U_TIMEZONE_FILES_DIR)
if (dir == NULL) {
+ // Build time configuration setting.
dir = TO_STRING(U_TIMEZONE_FILES_DIR);
}
#endif
+
if (dir == NULL) {
dir = "";
}
+
setTimeZoneFilesDir(dir, status);
}
@@ -1546,7 +1668,8 @@ The leftmost codepage (.xxx) wins.
/* Note that we scan the *uncorrected* ID. */
if ((p = uprv_strrchr(posixID, '@')) != NULL) {
if (correctedPOSIXLocale == NULL) {
- correctedPOSIXLocale = static_cast(uprv_malloc(uprv_strlen(posixID)+1));
+ /* new locale can be 1 char longer than old one if @ -> __ */
+ correctedPOSIXLocale = static_cast(uprv_malloc(uprv_strlen(posixID)+2));
/* Exit on memory allocation error. */
if (correctedPOSIXLocale == NULL) {
return NULL;
@@ -1563,7 +1686,7 @@ The leftmost codepage (.xxx) wins.
}
if (uprv_strchr(correctedPOSIXLocale,'_') == NULL) {
- uprv_strcat(correctedPOSIXLocale, "__"); /* aa@b -> aa__b */
+ uprv_strcat(correctedPOSIXLocale, "__"); /* aa@b -> aa__b (note this can make the new locale 1 char longer) */
}
else {
uprv_strcat(correctedPOSIXLocale, "_"); /* aa_CC@b -> aa_CC_b */
@@ -1603,6 +1726,7 @@ The leftmost codepage (.xxx) wins.
if (gCorrectedPOSIXLocale == NULL) {
gCorrectedPOSIXLocale = correctedPOSIXLocale;
+ gCorrectedPOSIXLocaleHeapAllocated = true;
ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup);
correctedPOSIXLocale = NULL;
}
@@ -1616,27 +1740,71 @@ The leftmost codepage (.xxx) wins.
#elif U_PLATFORM_USES_ONLY_WIN32_API
#define POSIX_LOCALE_CAPACITY 64
UErrorCode status = U_ZERO_ERROR;
- char *correctedPOSIXLocale = 0;
+ char *correctedPOSIXLocale = nullptr;
- if (gCorrectedPOSIXLocale != NULL) {
+ // If we have already figured this out just use the cached value
+ if (gCorrectedPOSIXLocale != nullptr) {
return gCorrectedPOSIXLocale;
}
- LCID id = GetThreadLocale();
- correctedPOSIXLocale = static_cast(uprv_malloc(POSIX_LOCALE_CAPACITY + 1));
- if (correctedPOSIXLocale) {
- int32_t posixLen = uprv_convertToPosix(id, correctedPOSIXLocale, POSIX_LOCALE_CAPACITY, &status);
- if (U_SUCCESS(status)) {
- *(correctedPOSIXLocale + posixLen) = 0;
- gCorrectedPOSIXLocale = correctedPOSIXLocale;
- ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup);
- } else {
- uprv_free(correctedPOSIXLocale);
+ // No cached value, need to determine the current value
+ static WCHAR windowsLocale[LOCALE_NAME_MAX_LENGTH] = {};
+ int length = GetLocaleInfoEx(LOCALE_NAME_USER_DEFAULT, LOCALE_SNAME, windowsLocale, LOCALE_NAME_MAX_LENGTH);
+
+ // Now we should have a Windows locale name that needs converted to the POSIX style.
+ if (length > 0) // If length is 0, then the GetLocaleInfoEx failed.
+ {
+ // First we need to go from UTF-16 to char (and also convert from _ to - while we're at it.)
+ char modifiedWindowsLocale[LOCALE_NAME_MAX_LENGTH] = {};
+
+ int32_t i;
+ for (i = 0; i < UPRV_LENGTHOF(modifiedWindowsLocale); i++)
+ {
+ if (windowsLocale[i] == '_')
+ {
+ modifiedWindowsLocale[i] = '-';
+ }
+ else
+ {
+ modifiedWindowsLocale[i] = static_cast(windowsLocale[i]);
+ }
+
+ if (modifiedWindowsLocale[i] == '\0')
+ {
+ break;
+ }
+ }
+
+ if (i >= UPRV_LENGTHOF(modifiedWindowsLocale))
+ {
+ // Ran out of room, can't really happen, maybe we'll be lucky about a matching
+ // locale when tags are dropped
+ modifiedWindowsLocale[UPRV_LENGTHOF(modifiedWindowsLocale) - 1] = '\0';
+ }
+
+ // Now normalize the resulting name
+ correctedPOSIXLocale = static_cast(uprv_malloc(POSIX_LOCALE_CAPACITY + 1));
+ /* TODO: Should we just exit on memory allocation failure? */
+ if (correctedPOSIXLocale)
+ {
+ int32_t posixLen = uloc_canonicalize(modifiedWindowsLocale, correctedPOSIXLocale, POSIX_LOCALE_CAPACITY, &status);
+ if (U_SUCCESS(status))
+ {
+ *(correctedPOSIXLocale + posixLen) = 0;
+ gCorrectedPOSIXLocale = correctedPOSIXLocale;
+ gCorrectedPOSIXLocaleHeapAllocated = true;
+ ucln_common_registerCleanup(UCLN_COMMON_PUTIL, putil_cleanup);
+ }
+ else
+ {
+ uprv_free(correctedPOSIXLocale);
+ }
}
}
- if (gCorrectedPOSIXLocale == NULL) {
- return "en_US";
+ // If unable to find a locale we can agree upon, use en-US by default
+ if (gCorrectedPOSIXLocale == nullptr) {
+ gCorrectedPOSIXLocale = "en_US";
}
return gCorrectedPOSIXLocale;
@@ -1923,8 +2091,34 @@ int_getDefaultCodepage()
#elif U_PLATFORM_USES_ONLY_WIN32_API
static char codepage[64];
- sprintf(codepage, "windows-%d", GetACP());
- return codepage;
+ DWORD codepageNumber = 0;
+
+#if U_PLATFORM_HAS_WINUWP_API > 0
+ // UWP doesn't have a direct API to get the default ACP as Microsoft would rather
+ // have folks use Unicode than a "system" code page, however this is the same
+ // codepage as the system default locale codepage. (FWIW, the system locale is
+ // ONLY used for codepage, it should never be used for anything else)
+ GetLocaleInfoEx(LOCALE_NAME_SYSTEM_DEFAULT, LOCALE_IDEFAULTANSICODEPAGE | LOCALE_RETURN_NUMBER,
+ (LPWSTR)&codepageNumber, sizeof(codepageNumber) / sizeof(WCHAR));
+#else
+ // Win32 apps can call GetACP
+ codepageNumber = GetACP();
+#endif
+ // Special case for UTF-8
+ if (codepageNumber == 65001)
+ {
+ return "UTF-8";
+ }
+ // Windows codepages can look like windows-1252, so format the found number
+ // the numbers are eclectic, however all valid system code pages, besides UTF-8
+ // are between 3 and 19999
+ if (codepageNumber > 0 && codepageNumber < 20000)
+ {
+ sprintf(codepage, "windows-%ld", codepageNumber);
+ return codepage;
+ }
+ // If the codepage number call failed then return UTF-8
+ return "UTF-8";
#elif U_POSIX_LOCALE
static char codesetName[100];
@@ -2108,19 +2302,16 @@ u_getVersion(UVersionInfo versionArray) {
* icucfg.h dependent code
*/
-#if U_ENABLE_DYLOAD
-
-#if HAVE_DLOPEN && !U_PLATFORM_USES_ONLY_WIN32_API
+#if U_ENABLE_DYLOAD && HAVE_DLOPEN && !U_PLATFORM_USES_ONLY_WIN32_API
#if HAVE_DLFCN_H
-
#ifdef __MVS__
#ifndef __SUSV3
#define __SUSV3 1
#endif
#endif
#include
-#endif
+#endif /* HAVE_DLFCN_H */
U_INTERNAL void * U_EXPORT2
uprv_dl_open(const char *libName, UErrorCode *status) {
@@ -2160,38 +2351,10 @@ uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
return uret.fp;
}
-#else
+#elif U_ENABLE_DYLOAD && U_PLATFORM_USES_ONLY_WIN32_API && !U_PLATFORM_HAS_WINUWP_API
-/* null (nonexistent) implementation. */
-
-U_INTERNAL void * U_EXPORT2
-uprv_dl_open(const char *libName, UErrorCode *status) {
- if(U_FAILURE(*status)) return NULL;
- *status = U_UNSUPPORTED_ERROR;
- return NULL;
-}
-
-U_INTERNAL void U_EXPORT2
-uprv_dl_close(void *lib, UErrorCode *status) {
- if(U_FAILURE(*status)) return;
- *status = U_UNSUPPORTED_ERROR;
- return;
-}
-
-
-U_INTERNAL UVoidFunction* U_EXPORT2
-uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
- if(U_SUCCESS(*status)) {
- *status = U_UNSUPPORTED_ERROR;
- }
- return (UVoidFunction*)NULL;
-}
-
-
-
-#endif
-
-#elif U_PLATFORM_USES_ONLY_WIN32_API
+/* Windows API implementation. */
+// Note: UWP does not expose/allow these APIs, so the UWP version gets the null implementation. */
U_INTERNAL void * U_EXPORT2
uprv_dl_open(const char *libName, UErrorCode *status) {
@@ -2218,7 +2381,6 @@ uprv_dl_close(void *lib, UErrorCode *status) {
return;
}
-
U_INTERNAL UVoidFunction* U_EXPORT2
uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
HMODULE handle = (HMODULE)lib;
@@ -2240,10 +2402,9 @@ uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
return addr;
}
-
#else
-/* No dynamic loading set. */
+/* No dynamic loading, null (nonexistent) implementation. */
U_INTERNAL void * U_EXPORT2
uprv_dl_open(const char *libName, UErrorCode *status) {
@@ -2261,7 +2422,6 @@ uprv_dl_close(void *lib, UErrorCode *status) {
return;
}
-
U_INTERNAL UVoidFunction* U_EXPORT2
uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
(void)lib;
@@ -2272,7 +2432,7 @@ uprv_dlsym_func(void *lib, const char* sym, UErrorCode *status) {
return (UVoidFunction*)NULL;
}
-#endif /* U_ENABLE_DYLOAD */
+#endif
/*
* Hey, Emacs, please set the following:
diff --git a/intl/icu/source/common/putilimp.h b/intl/icu/source/common/putilimp.h
index 05f739103d..d2cc37cde3 100644
--- a/intl/icu/source/common/putilimp.h
+++ b/intl/icu/source/common/putilimp.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -72,22 +72,13 @@
typedef size_t uintptr_t;
#endif
-/**
- * \def U_HAVE_MSVC_2003_OR_EARLIER
- * Flag for workaround of MSVC 2003 optimization bugs
- * @internal
- */
-#if !defined(U_HAVE_MSVC_2003_OR_EARLIER) && defined(_MSC_VER) && (_MSC_VER < 1400)
-#define U_HAVE_MSVC_2003_OR_EARLIER
-#endif
-
/*===========================================================================*/
/** @{ Information about POSIX support */
/*===========================================================================*/
#ifdef U_HAVE_NL_LANGINFO_CODESET
/* Use the predefined value. */
-#elif U_PLATFORM_HAS_WIN32_API || U_PLATFORM == U_PF_ANDROID || U_PLATFORM == U_PF_QNX
+#elif U_PLATFORM_USES_ONLY_WIN32_API || U_PLATFORM == U_PF_ANDROID || U_PLATFORM == U_PF_QNX
# define U_HAVE_NL_LANGINFO_CODESET 0
#else
# define U_HAVE_NL_LANGINFO_CODESET 1
@@ -103,10 +94,13 @@ typedef size_t uintptr_t;
# define U_NL_LANGINFO_CODESET CODESET
#endif
-#ifdef U_TZSET
+#if defined(U_TZSET) || defined(U_HAVE_TZSET)
/* Use the predefined value. */
#elif U_PLATFORM_USES_ONLY_WIN32_API
+ // UWP doesn't support tzset or environment variables for tz
+#if U_PLATFORM_HAS_WINUWP_API == 0
# define U_TZSET _tzset
+#endif
#elif U_PLATFORM == U_PF_OS400
/* not defined */
#else
@@ -117,15 +111,15 @@ typedef size_t uintptr_t;
/* Use the predefined value. */
#elif U_PLATFORM == U_PF_ANDROID
# define U_TIMEZONE timezone
+#elif defined(__UCLIBC__)
+ // uClibc does not have __timezone or _timezone.
+#elif defined(_NEWLIB_VERSION)
+# define U_TIMEZONE _timezone
+#elif defined(__GLIBC__)
+ // glibc
+# define U_TIMEZONE __timezone
#elif U_PLATFORM_IS_LINUX_BASED
-# if defined(__UCLIBC__)
- /* uClibc does not have __timezone or _timezone. */
-# elif defined(_NEWLIB_VERSION)
-# define U_TIMEZONE _timezone
-# elif defined(__GLIBC__)
- /* glibc */
-# define U_TIMEZONE __timezone
-# endif
+ // not defined
#elif U_PLATFORM_USES_ONLY_WIN32_API
# define U_TIMEZONE _timezone
#elif U_PLATFORM == U_PF_BSD && !defined(__NetBSD__)
@@ -138,10 +132,13 @@ typedef size_t uintptr_t;
# define U_TIMEZONE timezone
#endif
-#ifdef U_TZNAME
+#if defined(U_TZNAME) || defined(U_HAVE_TZNAME)
/* Use the predefined value. */
#elif U_PLATFORM_USES_ONLY_WIN32_API
+ /* not usable on all windows platforms */
+#if U_PLATFORM_HAS_WINUWP_API == 0
# define U_TZNAME _tzname
+#endif
#elif U_PLATFORM == U_PF_OS400
/* not defined */
#else
@@ -207,33 +204,21 @@ typedef size_t uintptr_t;
/**
* \def U_HAVE_STD_ATOMICS
- * Defines whether the standard C++11 is available.
- * ICU will use this when avialable,
- * otherwise will fall back to compiler or platform specific alternatives.
+ * Defines whether to use the standard C++11 functions
+ * If false, ICU will fall back to compiler or platform specific alternatives.
+ * Note: support for these fall back options for atomics will be removed in a future version
+ * of ICU, and the use of C++ 11 atomics will be required.
* @internal
*/
#ifdef U_HAVE_STD_ATOMICS
/* Use the predefined value. */
-#elif U_CPLUSPLUS_VERSION < 11
- /* Not C++11, disable use of atomics */
-# define U_HAVE_STD_ATOMICS 0
-#elif __clang__ && __clang_major__==3 && __clang_minor__<=1
- /* Clang 3.1, has atomic variable initializer bug. */
-# define U_HAVE_STD_ATOMICS 0
-#else
- /* U_HAVE_ATOMIC is typically set by an autoconf test of #include */
- /* Can be set manually, or left undefined, on platforms without autoconf. */
-# if defined(U_HAVE_ATOMIC) && U_HAVE_ATOMIC
-# define U_HAVE_STD_ATOMICS 1
-# else
-# define U_HAVE_STD_ATOMICS 0
-# endif
+#else
+# define U_HAVE_STD_ATOMICS 1
#endif
-
/**
* \def U_HAVE_CLANG_ATOMICS
- * Defines whether Clang c11 style built-in atomics are avaialable.
+ * Defines whether Clang c11 style built-in atomics are available.
* These are used in preference to gcc atomics when both are available.
*/
#ifdef U_HAVE_CLANG_ATOMICS
@@ -271,7 +256,7 @@ typedef size_t uintptr_t;
/**
* Platform utilities isolates the platform dependencies of the
- * libarary. For each platform which this code is ported to, these
+ * library. For each platform which this code is ported to, these
* functions may have to be re-implemented.
*/
@@ -394,6 +379,32 @@ U_INTERNAL double U_EXPORT2 uprv_log(double d);
*/
U_INTERNAL double U_EXPORT2 uprv_round(double x);
+/**
+ * Adds the signed integers a and b, storing the result in res.
+ * Checks for signed integer overflow.
+ * Similar to the GCC/Clang extension __builtin_add_overflow
+ *
+ * @param a The first operand.
+ * @param b The second operand.
+ * @param res a + b
+ * @return true if overflow occurred; false if no overflow occurred.
+ * @internal
+ */
+U_INTERNAL UBool U_EXPORT2 uprv_add32_overflow(int32_t a, int32_t b, int32_t* res);
+
+/**
+ * Multiplies the signed integers a and b, storing the result in res.
+ * Checks for signed integer overflow.
+ * Similar to the GCC/Clang extension __builtin_mul_overflow
+ *
+ * @param a The first multiplicand.
+ * @param b The second multiplicand.
+ * @param res a * b
+ * @return true if overflow occurred; false if no overflow occurred.
+ * @internal
+ */
+U_INTERNAL UBool U_EXPORT2 uprv_mul32_overflow(int32_t a, int32_t b, int32_t* res);
+
#if 0
/**
* Returns the number of digits after the decimal point in a double number x.
@@ -419,7 +430,7 @@ U_INTERNAL const char* U_EXPORT2 uprv_getDefaultCodepage(void);
/**
* Please use uloc_getDefault() instead.
- * Return the default locale ID string by querying ths system, or
+ * Return the default locale ID string by querying the system, or
* zero if one cannot be found.
* This function can call setlocale() on Unix platforms. Please read the
* platform documentation on setlocale() before calling this function.
@@ -563,6 +574,49 @@ U_INTERNAL void * U_EXPORT2 uprv_maximumPtr(void *base);
# endif
#endif
+
+#ifdef __cplusplus
+/**
+ * Pin a buffer capacity such that doing pointer arithmetic
+ * on the destination pointer and capacity cannot overflow.
+ *
+ * The pinned capacity must fulfill the following conditions (for positive capacities):
+ * - dest + capacity is a valid pointer according to the machine arcitecture (AS/400, 64-bit, etc.)
+ * - (dest + capacity) >= dest
+ * - The size (in bytes) of T[capacity] does not exceed 0x7fffffff
+ *
+ * @param dest the destination buffer pointer.
+ * @param capacity the requested buffer capacity, in units of type T.
+ * @return the pinned capacity.
+ * @internal
+ */
+template
+inline int32_t pinCapacity(T *dest, int32_t capacity) {
+ if (capacity <= 0) { return capacity; }
+
+ uintptr_t destInt = (uintptr_t)dest;
+ uintptr_t maxInt;
+
+# if U_PLATFORM == U_PF_OS390 && !defined(_LP64)
+ // We have 31-bit pointers.
+ maxInt = 0x7fffffff;
+# elif U_PLATFORM == U_PF_OS400
+ maxInt = (uintptr_t)uprv_maximumPtr((void *)dest);
+# else
+ maxInt = destInt + 0x7fffffffu;
+ if (maxInt < destInt) {
+ // Less than 2GB to the end of the address space.
+ // Pin to that to prevent address overflow.
+ maxInt = (uintptr_t)-1;
+ }
+# endif
+
+ uintptr_t maxBytes = maxInt - destInt; // max. 2GB
+ int32_t maxCapacity = (int32_t)(maxBytes / sizeof(T));
+ return capacity <= maxCapacity ? capacity : maxCapacity;
+}
+#endif // __cplusplus
+
/* Dynamic Library Functions */
typedef void (UVoidFunction)(void);
diff --git a/intl/icu/source/common/rbbi.cpp b/intl/icu/source/common/rbbi.cpp
index 2680bf216c..cb3766506f 100644
--- a/intl/icu/source/common/rbbi.cpp
+++ b/intl/icu/source/common/rbbi.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
***************************************************************************
@@ -7,7 +7,7 @@
***************************************************************************
*/
//
-// file: rbbi.c Contains the implementation of the rule based break iterator
+// file: rbbi.cpp Contains the implementation of the rule based break iterator
// runtime engine and the API implementation for
// class RuleBasedBreakIterator
//
@@ -18,21 +18,24 @@
#if !UCONFIG_NO_BREAK_ITERATION
+#include
+
#include "unicode/rbbi.h"
#include "unicode/schriter.h"
#include "unicode/uchriter.h"
-#include "unicode/udata.h"
#include "unicode/uclean.h"
-#include "rbbidata.h"
-#include "rbbirb.h"
+#include "unicode/udata.h"
+
+#include "brkeng.h"
+#include "ucln_cmn.h"
#include "cmemory.h"
#include "cstring.h"
-#include "umutex.h"
-#include "ucln_cmn.h"
-#include "brkeng.h"
-
+#include "rbbidata.h"
+#include "rbbi_cache.h"
+#include "rbbirb.h"
#include "uassert.h"
-#include "uvector.h"
+#include "umutex.h"
+#include "uvectr32.h"
// if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included.
#if U_LOCAL_SERVICE_HOOK
@@ -40,16 +43,16 @@
#endif
#ifdef RBBI_DEBUG
-static UBool fTrace = FALSE;
+static UBool gTrace = FALSE;
#endif
U_NAMESPACE_BEGIN
// The state number of the starting state
-#define START_STATE 1
+constexpr int32_t START_STATE = 1;
// The state-transition value indicating "stop"
-#define STOP_STATE 0
+constexpr int32_t STOP_STATE = 0;
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
@@ -64,8 +67,9 @@ UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
* tables object that is passed in as a parameter.
*/
RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
+ : fSCharIter(UnicodeString())
{
- init();
+ init(status);
fData = new RBBIDataWrapper(data, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
@@ -80,8 +84,10 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode
//
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
uint32_t ruleLength,
- UErrorCode &status) {
- init();
+ UErrorCode &status)
+ : fSCharIter(UnicodeString())
+{
+ init(status);
if (U_FAILURE(status)) {
return;
}
@@ -94,13 +100,13 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
status = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
- fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status);
+ fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status);
if (U_FAILURE(status)) {return;}
if(fData == 0) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
-}
+}
//-------------------------------------------------------------------------------
@@ -110,8 +116,9 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
//
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
+ : fSCharIter(UnicodeString())
{
- init();
+ init(status);
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == 0) {
@@ -130,8 +137,9 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &sta
RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules,
UParseError &parseError,
UErrorCode &status)
+ : fSCharIter(UnicodeString())
{
- init();
+ init(status);
if (U_FAILURE(status)) {return;}
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
@@ -152,8 +160,11 @@ RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules,
// Used when creating a RuleBasedBreakIterator from a set
// of rules.
//-------------------------------------------------------------------------------
-RuleBasedBreakIterator::RuleBasedBreakIterator() {
- init();
+RuleBasedBreakIterator::RuleBasedBreakIterator()
+ : fSCharIter(UnicodeString())
+{
+ UErrorCode status = U_ZERO_ERROR;
+ init(status);
}
@@ -164,9 +175,11 @@ RuleBasedBreakIterator::RuleBasedBreakIterator() {
//
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
-: BreakIterator(other)
+: BreakIterator(other),
+ fSCharIter(UnicodeString())
{
- this->init();
+ UErrorCode status = U_ZERO_ERROR;
+ this->init(status);
*this = other;
}
@@ -175,34 +188,29 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& oth
* Destructor
*/
RuleBasedBreakIterator::~RuleBasedBreakIterator() {
- if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
+ if (fCharIter != &fSCharIter) {
// fCharIter was adopted from the outside.
delete fCharIter;
}
fCharIter = NULL;
- delete fSCharIter;
- fCharIter = NULL;
- delete fDCharIter;
- fDCharIter = NULL;
-
- utext_close(fText);
+
+ utext_close(&fText);
if (fData != NULL) {
fData->removeReference();
fData = NULL;
}
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- fCachedBreakPositions = NULL;
- }
- if (fLanguageBreakEngines) {
- delete fLanguageBreakEngines;
- fLanguageBreakEngines = NULL;
- }
- if (fUnhandledBreakEngine) {
- delete fUnhandledBreakEngine;
- fUnhandledBreakEngine = NULL;
- }
+ delete fBreakCache;
+ fBreakCache = NULL;
+
+ delete fDictionaryCache;
+ fDictionaryCache = NULL;
+
+ delete fLanguageBreakEngines;
+ fLanguageBreakEngines = NULL;
+
+ delete fUnhandledBreakEngine;
+ fUnhandledBreakEngine = NULL;
}
/**
@@ -214,27 +222,31 @@ RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
if (this == &that) {
return *this;
}
- reset(); // Delete break cache information
- fBreakType = that.fBreakType;
+ BreakIterator::operator=(that);
+
if (fLanguageBreakEngines != NULL) {
delete fLanguageBreakEngines;
fLanguageBreakEngines = NULL; // Just rebuild for now
}
// TODO: clone fLanguageBreakEngines from "that"
UErrorCode status = U_ZERO_ERROR;
- fText = utext_clone(fText, that.fText, FALSE, TRUE, &status);
+ utext_clone(&fText, &that.fText, FALSE, TRUE, &status);
- if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
+ if (fCharIter != &fSCharIter) {
delete fCharIter;
}
- fCharIter = NULL;
+ fCharIter = &fSCharIter;
- if (that.fCharIter != NULL ) {
+ if (that.fCharIter != NULL && that.fCharIter != &that.fSCharIter) {
// This is a little bit tricky - it will intially appear that
// this->fCharIter is adopted, even if that->fCharIter was
// not adopted. That's ok.
fCharIter = that.fCharIter->clone();
}
+ fSCharIter = that.fSCharIter;
+ if (fCharIter == NULL) {
+ fCharIter = &fSCharIter;
+ }
if (fData != NULL) {
fData->removeReference();
@@ -244,6 +256,17 @@ RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
fData = that.fData->addReference();
}
+ fPosition = that.fPosition;
+ fRuleStatusIndex = that.fRuleStatusIndex;
+ fDone = that.fDone;
+
+ // TODO: both the dictionary and the main cache need to be copied.
+ // Current position could be within a dictionary range. Trying to continue
+ // the iteration without the caches present would go to the rules, with
+ // the assumption that the current position is on a rule boundary.
+ fBreakCache->reset(fPosition, fRuleStatusIndex);
+ fDictionaryCache->reset();
+
return *this;
}
@@ -255,33 +278,40 @@ RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
// Initializes all fields, leaving the object in a consistent state.
//
//-----------------------------------------------------------------------------
-void RuleBasedBreakIterator::init() {
- UErrorCode status = U_ZERO_ERROR;
- fText = utext_openUChars(NULL, NULL, 0, &status);
+void RuleBasedBreakIterator::init(UErrorCode &status) {
fCharIter = NULL;
- fSCharIter = NULL;
- fDCharIter = NULL;
fData = NULL;
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
+ fPosition = 0;
+ fRuleStatusIndex = 0;
+ fDone = false;
fDictionaryCharCount = 0;
- fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable
- // dictionary behavior for Break Iterators that are
- // built from rules. Even better would be the ability to
- // declare the type in the rules.
+ fLanguageBreakEngines = NULL;
+ fUnhandledBreakEngine = NULL;
+ fBreakCache = NULL;
+ fDictionaryCache = NULL;
- fCachedBreakPositions = NULL;
- fLanguageBreakEngines = NULL;
- fUnhandledBreakEngine = NULL;
- fNumCachedBreakPositions = 0;
- fPositionInCache = 0;
+ // Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER.
+ // fText = UTEXT_INITIALIZER;
+ static const UText initializedUText = UTEXT_INITIALIZER;
+ uprv_memcpy(&fText, &initializedUText, sizeof(UText));
+
+ if (U_FAILURE(status)) {
+ return;
+ }
+
+ utext_openUChars(&fText, NULL, 0, &status);
+ fDictionaryCache = new DictionaryCache(this, status);
+ fBreakCache = new BreakCache(this, status);
+ if (U_SUCCESS(status) && (fDictionaryCache == NULL || fBreakCache == NULL)) {
+ status = U_MEMORY_ALLOCATION_ERROR;
+ }
#ifdef RBBI_DEBUG
static UBool debugInitDone = FALSE;
if (debugInitDone == FALSE) {
char *debugEnv = getenv("U_RBBIDEBUG");
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
- fTrace = TRUE;
+ gTrace = TRUE;
}
debugInitDone = TRUE;
}
@@ -311,16 +341,28 @@ RuleBasedBreakIterator::operator==(const BreakIterator& that) const {
if (typeid(*this) != typeid(that)) {
return FALSE;
}
+ if (this == &that) {
+ return TRUE;
+ }
+
+ // The base class BreakIterator carries no state that participates in equality,
+ // and does not implement an equality function that would otherwise be
+ // checked at this point.
const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that;
- if (!utext_equals(fText, that2.fText)) {
+ if (!utext_equals(&fText, &that2.fText)) {
// The two break iterators are operating on different text,
- // or have a different interation position.
+ // or have a different iteration position.
+ // Note that fText's position is always the same as the break iterator's position.
return FALSE;
};
- // TODO: need a check for when in a dictionary region at different offsets.
+ if (!(fPosition == that2.fPosition &&
+ fRuleStatusIndex == that2.fRuleStatusIndex &&
+ fDone == that2.fDone)) {
+ return FALSE;
+ }
if (that2.fData == fData ||
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
@@ -348,8 +390,9 @@ void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
- reset();
- fText = utext_clone(fText, ut, FALSE, TRUE, &status);
+ fBreakCache->reset();
+ fDictionaryCache->reset();
+ utext_clone(&fText, ut, FALSE, TRUE, &status);
// Set up a dummy CharacterIterator to be returned if anyone
// calls getText(). With input from UText, there is no reasonable
@@ -357,58 +400,30 @@ void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) {
// Return one over an empty string instead - this is the closest
// we can come to signaling a failure.
// (GetText() is obsolete, this failure is sort of OK)
- if (fDCharIter == NULL) {
- static const UChar c = 0;
- fDCharIter = new UCharCharacterIterator(&c, 0);
- if (fDCharIter == NULL) {
- status = U_MEMORY_ALLOCATION_ERROR;
- return;
- }
- }
+ fSCharIter.setText(UnicodeString());
- if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
+ if (fCharIter != &fSCharIter) {
// existing fCharIter was adopted from the outside. Delete it now.
delete fCharIter;
}
- fCharIter = fDCharIter;
+ fCharIter = &fSCharIter;
this->first();
}
UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const {
- UText *result = utext_clone(fillIn, fText, FALSE, TRUE, &status);
+ UText *result = utext_clone(fillIn, &fText, FALSE, TRUE, &status);
return result;
}
-
-/**
- * Returns the description used to create this iterator
- */
-const UnicodeString&
-RuleBasedBreakIterator::getRules() const {
- if (fData != NULL) {
- return fData->getRuleSourceString();
- } else {
- static const UnicodeString *s;
- if (s == NULL) {
- // TODO: something more elegant here.
- // perhaps API should return the string by value.
- // Note: thread unsafe init & leak are semi-ok, better than
- // what was before. Sould be cleaned up, though.
- s = new UnicodeString;
- }
- return *s;
- }
-}
-
//=======================================================================
// BreakIterator overrides
//=======================================================================
/**
- * Return a CharacterIterator over the text being analyzed.
+ * Return a CharacterIterator over the text being analyzed.
*/
CharacterIterator&
RuleBasedBreakIterator::getText() const {
@@ -422,21 +437,22 @@ RuleBasedBreakIterator::getText() const {
*/
void
RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
- // If we are holding a CharacterIterator adopted from a
+ // If we are holding a CharacterIterator adopted from a
// previous call to this function, delete it now.
- if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
+ if (fCharIter != &fSCharIter) {
delete fCharIter;
}
fCharIter = newText;
UErrorCode status = U_ZERO_ERROR;
- reset();
- if (newText==NULL || newText->startIndex() != 0) {
+ fBreakCache->reset();
+ fDictionaryCache->reset();
+ if (newText==NULL || newText->startIndex() != 0) {
// startIndex !=0 wants to be an error, but there's no way to report it.
// Make the iterator text be an empty string.
- fText = utext_openUChars(fText, NULL, 0, &status);
+ utext_openUChars(&fText, NULL, 0, &status);
} else {
- fText = utext_openCharacterIterator(fText, newText, &status);
+ utext_openCharacterIterator(&fText, newText, &status);
}
this->first();
}
@@ -449,24 +465,21 @@ RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
void
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
UErrorCode status = U_ZERO_ERROR;
- reset();
- fText = utext_openConstUnicodeString(fText, &newText, &status);
+ fBreakCache->reset();
+ fDictionaryCache->reset();
+ utext_openConstUnicodeString(&fText, &newText, &status);
- // Set up a character iterator on the string.
+ // Set up a character iterator on the string.
// Needed in case someone calls getText().
// Can not, unfortunately, do this lazily on the (probably never)
// call to getText(), because getText is const.
- if (fSCharIter == NULL) {
- fSCharIter = new StringCharacterIterator(newText);
- } else {
- fSCharIter->setText(newText);
- }
+ fSCharIter.setText(newText);
- if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) {
+ if (fCharIter != &fSCharIter) {
// old fCharIter was adopted from the outside. Delete it.
delete fCharIter;
}
- fCharIter = fSCharIter;
+ fCharIter = &fSCharIter;
this->first();
}
@@ -486,14 +499,14 @@ RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, U
status = U_ILLEGAL_ARGUMENT_ERROR;
return *this;
}
- int64_t pos = utext_getNativeIndex(fText);
+ int64_t pos = utext_getNativeIndex(&fText);
// Shallow read-only clone of the new UText into the existing input UText
- fText = utext_clone(fText, input, FALSE, TRUE, &status);
+ utext_clone(&fText, input, FALSE, TRUE, &status);
if (U_FAILURE(status)) {
return *this;
}
- utext_setNativeIndex(fText, pos);
- if (utext_getNativeIndex(fText) != pos) {
+ utext_setNativeIndex(&fText, pos);
+ if (utext_getNativeIndex(&fText) != pos) {
// Sanity check. The new input utext is supposed to have the exact same
// contents as the old. If we can't set to the same position, it doesn't.
// The contents underlying the old utext might be invalid at this point,
@@ -509,13 +522,12 @@ RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, U
* @return The new iterator position, which is zero.
*/
int32_t RuleBasedBreakIterator::first(void) {
- reset();
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- //if (fText == NULL)
- // return BreakIterator::DONE;
-
- utext_setNativeIndex(fText, 0);
+ UErrorCode status = U_ZERO_ERROR;
+ if (!fBreakCache->seek(0)) {
+ fBreakCache->populateNear(0, status);
+ }
+ fBreakCache->current();
+ U_ASSERT(fPosition == 0);
return 0;
}
@@ -524,17 +536,12 @@ int32_t RuleBasedBreakIterator::first(void) {
* @return The text's past-the-end offset.
*/
int32_t RuleBasedBreakIterator::last(void) {
- reset();
- if (fText == NULL) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- fLastStatusIndexValid = FALSE;
- int32_t pos = (int32_t)utext_nativeLength(fText);
- utext_setNativeIndex(fText, pos);
- return pos;
+ int32_t endPos = (int32_t)utext_nativeLength(&fText);
+ UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
+ (void)endShouldBeBoundary;
+ U_ASSERT(endShouldBeBoundary);
+ U_ASSERT(fPosition == endPos);
+ return endPos;
}
/**
@@ -547,14 +554,17 @@ int32_t RuleBasedBreakIterator::last(void) {
* the current one.
*/
int32_t RuleBasedBreakIterator::next(int32_t n) {
- int32_t result = current();
- while (n > 0) {
- result = next();
- --n;
- }
- while (n < 0) {
- result = previous();
- ++n;
+ int32_t result = 0;
+ if (n > 0) {
+ for (; n > 0 && result != UBRK_DONE; --n) {
+ result = next();
+ }
+ } else if (n < 0) {
+ for (; n < 0 && result != UBRK_DONE; ++n) {
+ result = previous();
+ }
+ } else {
+ result = current();
}
return result;
}
@@ -564,396 +574,120 @@ int32_t RuleBasedBreakIterator::next(int32_t n) {
* @return The position of the first boundary after this one.
*/
int32_t RuleBasedBreakIterator::next(void) {
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step forward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache < fNumCachedBreakPositions - 1) {
- ++fPositionInCache;
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- int32_t startPos = current();
- fDictionaryCharCount = 0;
- int32_t result = handleNext(fData->fForwardTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(startPos, result, FALSE);
- }
- return result;
+ fBreakCache->next();
+ return fDone ? UBRK_DONE : fPosition;
}
/**
- * Advances the iterator backwards, to the last boundary preceding this one.
- * @return The position of the last boundary position preceding this one.
+ * Move the iterator backwards, to the boundary preceding the current one.
+ *
+ * Starts from the current position within fText.
+ * Starting position need not be on a boundary.
+ *
+ * @return The position of the boundary position immediately preceding the starting position.
*/
int32_t RuleBasedBreakIterator::previous(void) {
- int32_t result;
- int32_t startPos;
-
- // if we have cached break positions and we're still in the range
- // covered by them, just move one step backward in the cache
- if (fCachedBreakPositions != NULL) {
- if (fPositionInCache > 0) {
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- // if we're already sitting at the beginning of the text, return DONE
- if (fText == NULL || (startPos = current()) == 0) {
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- return BreakIterator::DONE;
- }
-
- if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) {
- result = handlePrevious(fData->fReverseTable);
- if (fDictionaryCharCount > 0) {
- result = checkDictionary(result, startPos, TRUE);
- }
- return result;
- }
-
- // old rule syntax
- // set things up. handlePrevious() will back us up to some valid
- // break position before the current position (we back our internal
- // iterator up one step to prevent handlePrevious() from returning
- // the current position), but not necessarily the last one before
- // where we started
-
- int32_t start = current();
-
- (void)UTEXT_PREVIOUS32(fText);
- int32_t lastResult = handlePrevious(fData->fReverseTable);
- if (lastResult == UBRK_DONE) {
- lastResult = 0;
- utext_setNativeIndex(fText, 0);
- }
- result = lastResult;
- int32_t lastTag = 0;
- UBool breakTagValid = FALSE;
-
- // iterate forward from the known break position until we pass our
- // starting point. The last break position before the starting
- // point is our return value
-
- for (;;) {
- result = next();
- if (result == BreakIterator::DONE || result >= start) {
- break;
- }
- lastResult = result;
- lastTag = fLastRuleStatusIndex;
- breakTagValid = TRUE;
- }
-
- // fLastBreakTag wants to have the value for section of text preceding
- // the result position that we are to return (in lastResult.) If
- // the backwards rules overshot and the above loop had to do two or more
- // next()s to move up to the desired return position, we will have a valid
- // tag value. But, if handlePrevious() took us to exactly the correct result position,
- // we wont have a tag value for that position, which is only set by handleNext().
-
- // Set the current iteration position to be the last break position
- // before where we started, and then return that value.
- utext_setNativeIndex(fText, lastResult);
- fLastRuleStatusIndex = lastTag; // for use by getRuleStatus()
- fLastStatusIndexValid = breakTagValid;
-
- // No need to check the dictionary; it will have been handled by
- // next()
-
- return lastResult;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->previous(status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
- * @offset The position from which to begin searching for a break position.
+ * @param startPos The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
*/
-int32_t RuleBasedBreakIterator::following(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
+int32_t RuleBasedBreakIterator::following(int32_t startPos) {
+ // if the supplied position is before the beginning, return the
// text's starting offset
- if (fText == NULL || offset >= utext_nativeLength(fText)) {
- last();
- return next();
- }
- else if (offset < 0) {
+ if (startPos < 0) {
return first();
}
// Move requested offset to a code point start. It might be on a trail surrogate,
- // or on a trail byte if the input is UTF-8.
- utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
+ // or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text.
+ utext_setNativeIndex(&fText, startPos);
+ startPos = (int32_t)utext_getNativeIndex(&fText);
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- // TODO: could use binary search
- // TODO: what if offset is outside range, but break is not?
- if (fCachedBreakPositions != NULL) {
- if (offset >= fCachedBreakPositions[0]
- && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- // We are guaranteed not to leave the array due to range test above
- while (offset >= fCachedBreakPositions[fPositionInCache]) {
- ++fPositionInCache;
- }
- int32_t pos = fCachedBreakPositions[fPositionInCache];
- utext_setNativeIndex(fText, pos);
- return pos;
- }
- else {
- reset();
- }
- }
-
- // Set our internal iteration position (temporarily)
- // to the position passed in. If this is the _beginning_ position,
- // then we can just use next() to get our return value
-
- int32_t result = 0;
-
- if (fData->fSafeRevTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- // move forward one codepoint to prepare for moving back to a
- // safe point.
- // this handles offset being between a supplementary character
- // TODO: is this still needed, with move to code point boundary handled above?
- (void)UTEXT_NEXT32(fText);
- // handlePrevious will move most of the time to < 1 boundary away
- handlePrevious(fData->fSafeRevTable);
- int32_t result = next();
- while (result <= offset) {
- result = next();
- }
- return result;
- }
- if (fData->fSafeFwdTable != NULL) {
- // backup plan if forward safe table is not available
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_PREVIOUS32(fText);
- // handle next will give result >= offset
- handleNext(fData->fSafeFwdTable);
- // previous will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = previous();
- while (oldresult > offset) {
- int32_t result = previous();
- if (result <= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = next();
- if (result <= offset) {
- return next();
- }
- return result;
- }
- // otherwise, we have to sync up first. Use handlePrevious() to back
- // up to a known break position before the specified position (if
- // we can determine that the specified position is a break position,
- // we don't back up at all). This may or may not be the last break
- // position at or before our starting position. Advance forward
- // from here until we've passed the starting position. The position
- // we stop on will be the first break position after the specified one.
- // old rule syntax
-
- utext_setNativeIndex(fText, offset);
- if (offset==0 ||
- (offset==1 && utext_getNativeIndex(fText)==0)) {
- return next();
- }
- result = previous();
-
- while (result != BreakIterator::DONE && result <= offset) {
- result = next();
- }
-
- return result;
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->following(startPos, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
- * @offset The position to begin searching for a break from.
+ * @param offset The position to begin searching for a break from.
* @return The position of the last boundary before the starting position.
*/
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
- // if the offset passed in is already past the end of the text,
- // just return DONE; if it's before the beginning, return the
- // text's starting offset
- if (fText == NULL || offset > utext_nativeLength(fText)) {
+ if (offset > utext_nativeLength(&fText)) {
return last();
}
- else if (offset < 0) {
- return first();
- }
// Move requested offset to a code point start. It might be on a trail surrogate,
// or on a trail byte if the input is UTF-8.
- utext_setNativeIndex(fText, offset);
- offset = (int32_t)utext_getNativeIndex(fText);
- // if we have cached break positions and offset is in the range
- // covered by them, use them
- if (fCachedBreakPositions != NULL) {
- // TODO: binary search?
- // TODO: What if offset is outside range, but break is not?
- if (offset > fCachedBreakPositions[0]
- && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
- fPositionInCache = 0;
- while (fPositionInCache < fNumCachedBreakPositions
- && offset > fCachedBreakPositions[fPositionInCache])
- ++fPositionInCache;
- --fPositionInCache;
- // If we're at the beginning of the cache, need to reevaluate the
- // rule status
- if (fPositionInCache <= 0) {
- fLastStatusIndexValid = FALSE;
- }
- utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]);
- return fCachedBreakPositions[fPositionInCache];
- }
- else {
- reset();
- }
- }
+ utext_setNativeIndex(&fText, offset);
+ int32_t adjustedOffset = static_cast(utext_getNativeIndex(&fText));
- // if we start by updating the current iteration position to the
- // position specified by the caller, we can just use previous()
- // to carry out this operation
-
- if (fData->fSafeFwdTable != NULL) {
- // new rule syntax
- utext_setNativeIndex(fText, offset);
- int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (newOffset != offset) {
- // Will come here if specified offset was not a code point boundary AND
- // the underlying implmentation is using UText, which snaps any non-code-point-boundary
- // indices to the containing code point.
- // For breakitereator::preceding only, these non-code-point indices need to be moved
- // up to refer to the following codepoint.
- (void)UTEXT_NEXT32(fText);
- offset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- }
-
- // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair,
- // rather than adjusting the position unconditionally?
- // (Change would interact with safe rules.)
- // TODO: change RBBI behavior for off-boundary indices to match that of UText?
- // affects only preceding(), seems cleaner, but is slightly different.
- (void)UTEXT_PREVIOUS32(fText);
- handleNext(fData->fSafeFwdTable);
- int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- while (result >= offset) {
- result = previous();
- }
- return result;
- }
- if (fData->fSafeRevTable != NULL) {
- // backup plan if forward safe table is not available
- // TODO: check whether this path can be discarded
- // It's probably OK to say that rules must supply both safe tables
- // if they use safe tables at all. We have certainly never described
- // to anyone how to work with just one safe table.
- utext_setNativeIndex(fText, offset);
- (void)UTEXT_NEXT32(fText);
-
- // handle previous will give result <= offset
- handlePrevious(fData->fSafeRevTable);
-
- // next will give result 0 or 1 boundary away from offset,
- // most of the time
- // we have to
- int32_t oldresult = next();
- while (oldresult < offset) {
- int32_t result = next();
- if (result >= offset) {
- return oldresult;
- }
- oldresult = result;
- }
- int32_t result = previous();
- if (result >= offset) {
- return previous();
- }
- return result;
- }
-
- // old rule syntax
- utext_setNativeIndex(fText, offset);
- return previous();
+ UErrorCode status = U_ZERO_ERROR;
+ fBreakCache->preceding(adjustedOffset, status);
+ return fDone ? UBRK_DONE : fPosition;
}
/**
* Returns true if the specfied position is a boundary position. As a side
* effect, leaves the iterator pointing to the first boundary position at
* or after "offset".
+ *
* @param offset the offset to check.
* @return True if "offset" is a boundary position.
*/
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
- // the beginning index of the iterator is always a boundary position by definition
- if (offset == 0) {
- first(); // For side effects on current position, tag values.
- return TRUE;
- }
-
- if (offset == (int32_t)utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
- return TRUE;
- }
-
// out-of-range indexes are never boundary positions
if (offset < 0) {
first(); // For side effects on current position, tag values.
return FALSE;
}
- if (offset > utext_nativeLength(fText)) {
- last(); // For side effects on current position, tag values.
- return FALSE;
+ // Adjust offset to be on a code point boundary and not beyond the end of the text.
+ // Note that isBoundary() is always false for offsets that are not on code point boundaries.
+ // But we still need the side effect of leaving iteration at the following boundary.
+
+ utext_setNativeIndex(&fText, offset);
+ int32_t adjustedOffset = static_cast(utext_getNativeIndex(&fText));
+
+ bool result = false;
+ UErrorCode status = U_ZERO_ERROR;
+ if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) {
+ result = (fBreakCache->current() == offset);
}
- // otherwise, we can use following() on the position before the specified
- // one and return true if the position we get back is the one the user
- // specified
- utext_previous32From(fText, offset);
- int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- UBool result = following(backOne) == offset;
+ if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) {
+ // Original offset is beyond the end of the text. Return FALSE, it's not a boundary,
+ // but the iteration position remains set to the end of the text, which is a boundary.
+ return FALSE;
+ }
+ if (!result) {
+ // Not on a boundary. isBoundary() must leave iterator on the following boundary.
+ // Cache->seek(), above, left us on the preceding boundary, so advance one.
+ next();
+ }
return result;
}
+
/**
* Returns the current iteration position.
* @return The current iteration position.
*/
int32_t RuleBasedBreakIterator::current(void) const {
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- return pos;
+ return fPosition;
}
-
+
+
//=======================================================================
// implementation
//=======================================================================
@@ -1020,43 +754,43 @@ struct LookAheadResults {
//-----------------------------------------------------------------------------------
//
-// handleNext(stateTable)
-// This method is the actual implementation of the rbbi next() method.
-// This method initializes the state machine to state 1
-// and advances through the text character by character until we reach the end
-// of the text or the state machine transitions to state 0. We update our return
-// value every time the state machine passes through an accepting state.
+// handleNext()
+// Run the state machine to find a boundary
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handleNext() {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
-
+
RBBIStateTableRow *row;
UChar32 c;
LookAheadResults lookAheadMatches;
int32_t result = 0;
int32_t initialPosition = 0;
+ const RBBIStateTable *statetable = fData->fForwardTable;
const char *tableData = statetable->fTableData;
uint32_t tableRowLen = statetable->fRowLen;
-
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Next pos char state category");
}
#endif
- // No matter what, handleNext alway correctly sets the break tag value.
- fLastStatusIndexValid = TRUE;
- fLastRuleStatusIndex = 0;
+ // handleNext alway sets the break tag value.
+ // Set the default for it.
+ fRuleStatusIndex = 0;
+
+ fDictionaryCharCount = 0;
// if we're already at the end of the text, return DONE.
- initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ initialPosition = fPosition;
+ UTEXT_SETNATIVEINDEX(&fText, initialPosition);
result = initialPosition;
- c = UTEXT_NEXT32(fText);
- if (fData == NULL || c==U_SENTINEL) {
- return BreakIterator::DONE;
+ c = UTEXT_NEXT32(&fText);
+ if (c==U_SENTINEL) {
+ fDone = TRUE;
+ return UBRK_DONE;
}
// Set the initial state for the state machine
@@ -1064,8 +798,8 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
row = (RBBIStateTableRow *)
//(statetable->fTableData + (statetable->fRowLen * state));
(tableData + tableRowLen * state);
-
-
+
+
mode = RBBI_RUN;
if (statetable->fFlags & RBBI_BOF_REQUIRED) {
category = 2;
@@ -1079,7 +813,7 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
if (c == U_SENTINEL) {
// Reached end of input string.
if (mode == RBBI_END) {
- // We have already run the loop one last time with the
+ // We have already run the loop one last time with the
// character set to the psueudo {eof} value. Now it is time
// to unconditionally bail out.
break;
@@ -1100,10 +834,10 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
// Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
// not the size of the character going in, which is a UChar32.
//
- UTRIE_GET16(&fData->fTrie, c, category);
+ category = UTRIE2_GET16(fData->fTrie, c);
// Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
+ // Counter is only used by dictionary based iteration.
// Chars that need to be handled by a dictionary have a flag bit set
// in their category values.
//
@@ -1115,8 +849,8 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
}
#ifdef RBBI_DEBUG
- if (fTrace) {
- RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText));
+ if (gTrace) {
+ RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
} else {
@@ -1129,9 +863,7 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
// State Transition - move machine to its next state
//
- // Note: fNextState is defined as uint16_t[2], but we are casting
- // a generated RBBI table to RBBIStateTableRow and some tables
- // actually have more than 2 categories.
+ // fNextState is a variable-length array.
U_ASSERT(categoryfHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RBBIStateTableRow *)
@@ -1142,25 +874,25 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
if (row->fAccepting == -1) {
// Match found, common case.
if (mode != RBBI_START) {
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
}
- fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
+ fRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values.
}
int16_t completedRule = row->fAccepting;
if (completedRule > 0) {
- // Lookahead match is completed.
+ // Lookahead match is completed.
int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule);
if (lookaheadResult >= 0) {
- fLastRuleStatusIndex = row->fTagIdx;
- UTEXT_SETNATIVEINDEX(fText, lookaheadResult);
+ fRuleStatusIndex = row->fTagIdx;
+ fPosition = lookaheadResult;
return lookaheadResult;
}
}
int16_t rule = row->fLookAhead;
if (rule != 0) {
// At the position of a '/' in a look-ahead match. Record it.
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
lookAheadMatches.setPosition(rule, pos);
}
@@ -1170,20 +902,18 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
// longer match is possible, no matter what characters follow.
break;
}
-
- // Advance to the next character.
+
+ // Advance to the next character.
// If this is a beginning-of-input loop iteration, don't advance
// the input position. The next iteration will be processing the
// first real input character.
if (mode == RBBI_RUN) {
- c = UTEXT_NEXT32(fText);
+ c = UTEXT_NEXT32(&fText);
} else {
if (mode == RBBI_START) {
mode = RBBI_RUN;
}
}
-
-
}
// The state machine is done. Check whether it found a match...
@@ -1192,15 +922,16 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
// (This really indicates a defect in the break rules. They should always match
// at least one character.)
if (result == initialPosition) {
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- UTEXT_NEXT32(fText);
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
+ utext_setNativeIndex(&fText, initialPosition);
+ utext_next32(&fText);
+ result = (int32_t)utext_getNativeIndex(&fText);
+ fRuleStatusIndex = 0;
}
// Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
+ fPosition = result;
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
@@ -1208,112 +939,57 @@ int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) {
}
-
//-----------------------------------------------------------------------------------
//
-// handlePrevious()
+// handleSafePrevious()
//
-// Iterate backwards, according to the logic of the reverse rules.
-// This version handles the exact style backwards rules.
-//
-// The logic of this function is very similar to handleNext(), above.
+// Iterate backwards using the safe reverse rules.
+// The logic of this function is similar to handleNext(), but simpler
+// because the safe table does not require as many options.
//
//-----------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) {
+int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
int32_t state;
uint16_t category = 0;
- RBBIRunMode mode;
RBBIStateTableRow *row;
UChar32 c;
- LookAheadResults lookAheadMatches;
int32_t result = 0;
- int32_t initialPosition = 0;
+ const RBBIStateTable *stateTable = fData->fReverseTable;
+ UTEXT_SETNATIVEINDEX(&fText, fromPosition);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPuts("Handle Previous pos char state category");
}
#endif
- // handlePrevious() never gets the rule status.
- // Flag the status as invalid; if the user ever asks for status, we will need
- // to back up, then re-find the break position using handleNext(), which does
- // get the status value.
- fLastStatusIndexValid = FALSE;
- fLastRuleStatusIndex = 0;
-
// if we're already at the start of the text, return DONE.
- if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) {
+ if (fData == NULL || UTEXT_GETNATIVEINDEX(&fText)==0) {
return BreakIterator::DONE;
}
- // Set up the starting char.
- initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- result = initialPosition;
- c = UTEXT_PREVIOUS32(fText);
-
// Set the initial state for the state machine
+ c = UTEXT_PREVIOUS32(&fText);
state = START_STATE;
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
- category = 3;
- mode = RBBI_RUN;
- if (statetable->fFlags & RBBI_BOF_REQUIRED) {
- category = 2;
- mode = RBBI_START;
- }
-
+ (stateTable->fTableData + (stateTable->fRowLen * state));
// loop until we reach the start of the text or transition to state 0
//
- for (;;) {
- if (c == U_SENTINEL) {
- // Reached end of input string.
- if (mode == RBBI_END) {
- // We have already run the loop one last time with the
- // character set to the psueudo {eof} value. Now it is time
- // to unconditionally bail out.
- if (result == initialPosition) {
- // Ran off start, no match found.
- // move one index one (towards the start, since we are doing a previous())
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- (void)UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check.
- }
- break;
- }
- // Run the loop one last time with the fake end-of-input character category.
- mode = RBBI_END;
- category = 1;
- }
+ for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) {
+ // look up the current character's character category, which tells us
+ // which column in the state table to look at.
+ // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
+ // not the size of the character going in, which is a UChar32.
//
- // Get the char category. An incoming category of 1 or 2 means that
- // we are preset for doing the beginning or end of input, and
- // that we shouldn't get a category from an actual text input character.
- //
- if (mode == RBBI_RUN) {
- // look up the current character's character category, which tells us
- // which column in the state table to look at.
- // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned,
- // not the size of the character going in, which is a UChar32.
- //
- UTRIE_GET16(&fData->fTrie, c, category);
-
- // Check the dictionary bit in the character's category.
- // Counter is only used by dictionary based iterators (subclasses).
- // Chars that need to be handled by a dictionary have a flag bit set
- // in their category values.
- //
- if ((category & 0x4000) != 0) {
- fDictionaryCharCount++;
- // And off the dictionary flag bit.
- category &= ~0x4000;
- }
- }
+ // And off the dictionary flag bit. For reverse iteration it is not used.
+ category = UTRIE2_GET16(fData->fTrie, c);
+ category &= ~0x4000;
#ifdef RBBI_DEBUG
- if (fTrace) {
- RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText));
+ if (gTrace) {
+ RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
} else {
@@ -1325,92 +1001,29 @@ int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable)
// State Transition - move machine to its next state
//
-
- // Note: fNextState is defined as uint16_t[2], but we are casting
- // a generated RBBI table to RBBIStateTableRow and some tables
- // actually have more than 2 categories.
+ // fNextState is a variable-length array.
U_ASSERT(categoryfHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RBBIStateTableRow *)
- (statetable->fTableData + (statetable->fRowLen * state));
-
- if (row->fAccepting == -1) {
- // Match found, common case.
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- }
-
- int16_t completedRule = row->fAccepting;
- if (completedRule > 0) {
- // Lookahead match is completed.
- int32_t lookaheadResult = lookAheadMatches.getPosition(completedRule);
- if (lookaheadResult >= 0) {
- UTEXT_SETNATIVEINDEX(fText, lookaheadResult);
- return lookaheadResult;
- }
- }
- int16_t rule = row->fLookAhead;
- if (rule != 0) {
- // At the position of a '/' in a look-ahead match. Record it.
- int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- lookAheadMatches.setPosition(rule, pos);
- }
+ (stateTable->fTableData + (stateTable->fRowLen * state));
if (state == STOP_STATE) {
// This is the normal exit from the lookup state machine.
- // We have advanced through the string until it is certain that no
- // longer match is possible, no matter what characters follow.
+ // Transistion to state zero means we have found a safe point.
break;
}
-
- // Move (backwards) to the next character to process.
- // If this is a beginning-of-input loop iteration, don't advance
- // the input position. The next iteration will be processing the
- // first real input character.
- if (mode == RBBI_RUN) {
- c = UTEXT_PREVIOUS32(fText);
- } else {
- if (mode == RBBI_START) {
- mode = RBBI_RUN;
- }
- }
}
// The state machine is done. Check whether it found a match...
-
- // If the iterator failed to advance in the match engine, force it ahead by one.
- // (This really indicates a defect in the break rules. They should always match
- // at least one character.)
- if (result == initialPosition) {
- UTEXT_SETNATIVEINDEX(fText, initialPosition);
- UTEXT_PREVIOUS32(fText);
- result = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- }
-
- // Leave the iterator at our result position.
- UTEXT_SETNATIVEINDEX(fText, result);
+ result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
#ifdef RBBI_DEBUG
- if (fTrace) {
+ if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
return result;
}
-
-void
-RuleBasedBreakIterator::reset()
-{
- if (fCachedBreakPositions) {
- uprv_free(fCachedBreakPositions);
- }
- fCachedBreakPositions = NULL;
- fNumCachedBreakPositions = 0;
- fDictionaryCharCount = 0;
- fPositionInCache = 0;
-}
-
-
-
//-------------------------------------------------------------------------------
//
// getRuleStatus() Return the break rule tag associated with the current
@@ -1418,64 +1031,27 @@ RuleBasedBreakIterator::reset()
// position by iterating forwards, the value will have been
// cached by the handleNext() function.
//
-// If no cached status value is available, the status is
-// found by doing a previous() followed by a next(), which
-// leaves the iterator where it started, and computes the
-// status while doing the next().
-//
//-------------------------------------------------------------------------------
-void RuleBasedBreakIterator::makeRuleStatusValid() {
- if (fLastStatusIndexValid == FALSE) {
- // No cached status is available.
- if (fText == NULL || current() == 0) {
- // At start of text, or there is no text. Status is always zero.
- fLastRuleStatusIndex = 0;
- fLastStatusIndexValid = TRUE;
- } else {
- // Not at start of text. Find status the tedious way.
- int32_t pa = current();
- previous();
- if (fNumCachedBreakPositions > 0) {
- reset(); // Blow off the dictionary cache
- }
- int32_t pb = next();
- if (pa != pb) {
- // note: the if (pa != pb) test is here only to eliminate warnings for
- // unused local variables on gcc. Logically, it isn't needed.
- U_ASSERT(pa == pb);
- }
- }
- }
- U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx);
-}
-
int32_t RuleBasedBreakIterator::getRuleStatus() const {
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
// fLastRuleStatusIndex indexes to the start of the appropriate status record
// (the number of status values.)
// This function returns the last (largest) of the array of status values.
- int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex];
int32_t tagVal = fData->fRuleStatusTable[idx];
return tagVal;
}
-
-
int32_t RuleBasedBreakIterator::getRuleStatusVec(
- int32_t *fillInVec, int32_t capacity, UErrorCode &status)
-{
+ int32_t *fillInVec, int32_t capacity, UErrorCode &status) {
if (U_FAILURE(status)) {
return 0;
}
- RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this;
- nonConstThis->makeRuleStatusValid();
- int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex];
+ int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex];
int32_t numValsToCopy = numVals;
if (numVals > capacity) {
status = U_BUFFER_OVERFLOW_ERROR;
@@ -1483,7 +1059,7 @@ int32_t RuleBasedBreakIterator::getRuleStatusVec(
}
int i;
for (i=0; ifRuleStatusTable[fLastRuleStatusIndex + i + 1];
+ fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1];
}
return numVals;
}
@@ -1531,205 +1107,25 @@ BreakIterator * RuleBasedBreakIterator::createBufferClone(void * /*stackBuffer*
return (RuleBasedBreakIterator *)clonedBI;
}
-
-//-------------------------------------------------------------------------------
-//
-// isDictionaryChar Return true if the category lookup for this char
-// indicates that it is in the set of dictionary lookup
-// chars.
-//
-// This function is intended for use by dictionary based
-// break iterators.
-//
-//-------------------------------------------------------------------------------
-/*UBool RuleBasedBreakIterator::isDictionaryChar(UChar32 c) {
- if (fData == NULL) {
- return FALSE;
- }
- uint16_t category;
- UTRIE_GET16(&fData->fTrie, c, category);
- return (category & 0x4000) != 0;
-}*/
-
-
-//-------------------------------------------------------------------------------
-//
-// checkDictionary This function handles all processing of characters in
-// the "dictionary" set. It will determine the appropriate
-// course of action, and possibly set up a cache in the
-// process.
-//
-//-------------------------------------------------------------------------------
-int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos,
- int32_t endPos,
- UBool reverse) {
- // Reset the old break cache first.
- reset();
-
- // note: code segment below assumes that dictionary chars are in the
- // startPos-endPos range
- // value returned should be next character in sequence
- if ((endPos - startPos) <= 1) {
- return (reverse ? startPos : endPos);
- }
-
- // Starting from the starting point, scan towards the proposed result,
- // looking for the first dictionary character (which may be the one
- // we're on, if we're starting in the middle of a range).
- utext_setNativeIndex(fText, reverse ? endPos : startPos);
- if (reverse) {
- UTEXT_PREVIOUS32(fText);
- }
-
- int32_t rangeStart = startPos;
- int32_t rangeEnd = endPos;
-
- uint16_t category;
- int32_t current;
- UErrorCode status = U_ZERO_ERROR;
- UStack breaks(status);
- int32_t foundBreakCount = 0;
- UChar32 c = utext_current32(fText);
-
- UTRIE_GET16(&fData->fTrie, c, category);
-
- // Is the character we're starting on a dictionary character? If so, we
- // need to back up to include the entire run; otherwise the results of
- // the break algorithm will differ depending on where we start. Since
- // the result is cached and there is typically a non-dictionary break
- // within a small number of words, there should be little performance impact.
- if (category & 0x4000) {
- if (reverse) {
- do {
- utext_next32(fText); // TODO: recast to work directly with postincrement.
- c = utext_current32(fText);
- UTRIE_GET16(&fData->fTrie, c, category);
- } while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText);
- if (c == U_SENTINEL) {
- // c = fText->last32();
- // TODO: why was this if needed?
- c = UTEXT_PREVIOUS32(fText);
- }
- else {
- c = UTEXT_PREVIOUS32(fText);
- }
- }
- else {
- do {
- c = UTEXT_PREVIOUS32(fText);
- UTRIE_GET16(&fData->fTrie, c, category);
- }
- while (c != U_SENTINEL && (category & 0x4000));
- // Back up to the last dictionary character
- if (c == U_SENTINEL) {
- // c = fText->first32();
- c = utext_current32(fText);
- }
- else {
- utext_next32(fText);
- c = utext_current32(fText);
- }
- rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);;
- }
- UTRIE_GET16(&fData->fTrie, c, category);
- }
-
- // Loop through the text, looking for ranges of dictionary characters.
- // For each span, find the appropriate break engine, and ask it to find
- // any breaks within the span.
- // Note: we always do this in the forward direction, so that the break
- // cache is built in the right order.
- if (reverse) {
- utext_setNativeIndex(fText, rangeStart);
- c = utext_current32(fText);
- UTRIE_GET16(&fData->fTrie, c, category);
- }
- while(U_SUCCESS(status)) {
- while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) {
- utext_next32(fText); // TODO: tweak for post-increment operation
- c = utext_current32(fText);
- UTRIE_GET16(&fData->fTrie, c, category);
- }
- if (current >= rangeEnd) {
- break;
- }
-
- // We now have a dictionary character. Get the appropriate language object
- // to deal with it.
- const LanguageBreakEngine *lbe = getLanguageBreakEngine(c);
-
- // Ask the language object if there are any breaks. It will leave the text
- // pointer on the other side of its range, ready to search for the next one.
- if (lbe != NULL) {
- foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks);
- }
-
- // Reload the loop variables for the next go-round
- c = utext_current32(fText);
- UTRIE_GET16(&fData->fTrie, c, category);
- }
-
- // If we found breaks, build a new break cache. The first and last entries must
- // be the original starting and ending position.
- if (foundBreakCount > 0) {
- U_ASSERT(foundBreakCount == breaks.size());
- int32_t totalBreaks = foundBreakCount;
- if (startPos < breaks.elementAti(0)) {
- totalBreaks += 1;
- }
- if (endPos > breaks.peeki()) {
- totalBreaks += 1;
- }
- fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t));
- if (fCachedBreakPositions != NULL) {
- int32_t out = 0;
- fNumCachedBreakPositions = totalBreaks;
- if (startPos < breaks.elementAti(0)) {
- fCachedBreakPositions[out++] = startPos;
- }
- for (int32_t i = 0; i < foundBreakCount; ++i) {
- fCachedBreakPositions[out++] = breaks.elementAti(i);
- }
- if (endPos > fCachedBreakPositions[out-1]) {
- fCachedBreakPositions[out] = endPos;
- }
- // If there are breaks, then by definition, we are replacing the original
- // proposed break by one of the breaks we found. Use following() and
- // preceding() to do the work. They should never recurse in this case.
- if (reverse) {
- return preceding(endPos);
- }
- else {
- return following(startPos);
- }
- }
- // If the allocation failed, just fall through to the "no breaks found" case.
- }
-
- // If we get here, there were no language-based breaks. Set the text pointer
- // to the original proposed break.
- utext_setNativeIndex(fText, reverse ? startPos : endPos);
- return (reverse ? startPos : endPos);
-}
-
U_NAMESPACE_END
-static icu::UStack *gLanguageBreakFactories = NULL;
+static icu::UStack *gLanguageBreakFactories = nullptr;
+static const icu::UnicodeString *gEmptyString = nullptr;
static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER;
+static icu::UInitOnce gRBBIInitOnce = U_INITONCE_INITIALIZER;
/**
- * Release all static memory held by breakiterator.
+ * Release all static memory held by breakiterator.
*/
U_CDECL_BEGIN
-static UBool U_CALLCONV breakiterator_cleanup_dict(void) {
- if (gLanguageBreakFactories) {
- delete gLanguageBreakFactories;
- gLanguageBreakFactories = NULL;
- }
+static UBool U_CALLCONV rbbi_cleanup(void) {
+ delete gLanguageBreakFactories;
+ gLanguageBreakFactories = nullptr;
+ delete gEmptyString;
+ gEmptyString = nullptr;
gLanguageBreakFactoriesInitOnce.reset();
+ gRBBIInitOnce.reset();
return TRUE;
}
U_CDECL_END
@@ -1741,6 +1137,11 @@ static void U_CALLCONV _deleteFactory(void *obj) {
U_CDECL_END
U_NAMESPACE_BEGIN
+static void U_CALLCONV rbbiInit() {
+ gEmptyString = new UnicodeString();
+ ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
+}
+
static void U_CALLCONV initLanguageFactories() {
UErrorCode status = U_ZERO_ERROR;
U_ASSERT(gLanguageBreakFactories == NULL);
@@ -1755,23 +1156,23 @@ static void U_CALLCONV initLanguageFactories() {
}
#endif
}
- ucln_common_registerCleanup(UCLN_COMMON_BREAKITERATOR_DICT, breakiterator_cleanup_dict);
+ ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
}
static const LanguageBreakEngine*
-getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType)
+getLanguageBreakEngineFromFactory(UChar32 c)
{
umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories);
if (gLanguageBreakFactories == NULL) {
return NULL;
}
-
+
int32_t i = gLanguageBreakFactories->size();
const LanguageBreakEngine *lbe = NULL;
while (--i >= 0) {
LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i));
- lbe = factory->getEngineFor(c, breakType);
+ lbe = factory->getEngineFor(c);
if (lbe != NULL) {
break;
}
@@ -1790,7 +1191,7 @@ const LanguageBreakEngine *
RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
const LanguageBreakEngine *lbe = NULL;
UErrorCode status = U_ZERO_ERROR;
-
+
if (fLanguageBreakEngines == NULL) {
fLanguageBreakEngines = new UStack(status);
if (fLanguageBreakEngines == NULL || U_FAILURE(status)) {
@@ -1799,19 +1200,19 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
return NULL;
}
}
-
+
int32_t i = fLanguageBreakEngines->size();
while (--i >= 0) {
lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i));
- if (lbe->handles(c, fBreakType)) {
+ if (lbe->handles(c)) {
return lbe;
}
}
-
+
// No existing dictionary took the character. See if a factory wants to
// give us a new LanguageBreakEngine for this character.
- lbe = getLanguageBreakEngineFromFactory(c, fBreakType);
-
+ lbe = getLanguageBreakEngineFromFactory(c);
+
// If we got one, use it and push it on our stack.
if (lbe != NULL) {
fLanguageBreakEngines->push((void *)lbe, status);
@@ -1819,13 +1220,14 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
// return it even if the push fails.
return lbe;
}
-
+
// No engine is forthcoming for this character. Add it to the
// reject set. Create the reject break engine if needed.
if (fUnhandledBreakEngine == NULL) {
fUnhandledBreakEngine = new UnhandledEngine(status);
if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) {
status = U_MEMORY_ALLOCATION_ERROR;
+ return nullptr;
}
// Put it last so that scripts for which we have an engine get tried
// first.
@@ -1837,23 +1239,34 @@ RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
return NULL;
}
}
-
+
// Tell the reject engine about the character; at its discretion, it may
// add more than just the one character.
- fUnhandledBreakEngine->handleCharacter(c, fBreakType);
-
+ fUnhandledBreakEngine->handleCharacter(c);
+
return fUnhandledBreakEngine;
}
+void RuleBasedBreakIterator::dumpCache() {
+ fBreakCache->dumpCache();
+}
+void RuleBasedBreakIterator::dumpTables() {
+ fData->printData();
+}
-/*int32_t RuleBasedBreakIterator::getBreakType() const {
- return fBreakType;
-}*/
+/**
+ * Returns the description used to create this iterator
+ */
-void RuleBasedBreakIterator::setBreakType(int32_t type) {
- fBreakType = type;
- reset();
+const UnicodeString&
+RuleBasedBreakIterator::getRules() const {
+ if (fData != NULL) {
+ return fData->getRuleSourceString();
+ } else {
+ umtx_initOnce(gRBBIInitOnce, &rbbiInit);
+ return *gEmptyString;
+ }
}
U_NAMESPACE_END
diff --git a/intl/icu/source/common/rbbi_cache.cpp b/intl/icu/source/common/rbbi_cache.cpp
new file mode 100644
index 0000000000..519c610498
--- /dev/null
+++ b/intl/icu/source/common/rbbi_cache.cpp
@@ -0,0 +1,658 @@
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// file: rbbi_cache.cpp
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+#include "unicode/ubrk.h"
+#include "unicode/rbbi.h"
+
+#include "rbbi_cache.h"
+
+#include "brkeng.h"
+#include "cmemory.h"
+#include "rbbidata.h"
+#include "rbbirb.h"
+#include "uassert.h"
+#include "uvectr32.h"
+
+U_NAMESPACE_BEGIN
+
+/*
+ * DictionaryCache implementation
+ */
+
+RuleBasedBreakIterator::DictionaryCache::DictionaryCache(RuleBasedBreakIterator *bi, UErrorCode &status) :
+ fBI(bi), fBreaks(status), fPositionInCache(-1),
+ fStart(0), fLimit(0), fFirstRuleStatusIndex(0), fOtherRuleStatusIndex(0) {
+}
+
+RuleBasedBreakIterator::DictionaryCache::~DictionaryCache() {
+}
+
+void RuleBasedBreakIterator::DictionaryCache::reset() {
+ fPositionInCache = -1;
+ fStart = 0;
+ fLimit = 0;
+ fFirstRuleStatusIndex = 0;
+ fOtherRuleStatusIndex = 0;
+ fBreaks.removeAllElements();
+}
+
+UBool RuleBasedBreakIterator::DictionaryCache::following(int32_t fromPos, int32_t *result, int32_t *statusIndex) {
+ if (fromPos >= fLimit || fromPos < fStart) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ // Sequential iteration, move from previous boundary to the following
+
+ int32_t r = 0;
+ if (fPositionInCache >= 0 && fPositionInCache < fBreaks.size() && fBreaks.elementAti(fPositionInCache) == fromPos) {
+ ++fPositionInCache;
+ if (fPositionInCache >= fBreaks.size()) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+ r = fBreaks.elementAti(fPositionInCache);
+ U_ASSERT(r > fromPos);
+ *result = r;
+ *statusIndex = fOtherRuleStatusIndex;
+ return TRUE;
+ }
+
+ // Random indexing. Linear search for the boundary following the given position.
+
+ for (fPositionInCache = 0; fPositionInCache < fBreaks.size(); ++fPositionInCache) {
+ r= fBreaks.elementAti(fPositionInCache);
+ if (r > fromPos) {
+ *result = r;
+ *statusIndex = fOtherRuleStatusIndex;
+ return TRUE;
+ }
+ }
+ U_ASSERT(FALSE);
+ fPositionInCache = -1;
+ return FALSE;
+}
+
+
+UBool RuleBasedBreakIterator::DictionaryCache::preceding(int32_t fromPos, int32_t *result, int32_t *statusIndex) {
+ if (fromPos <= fStart || fromPos > fLimit) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ if (fromPos == fLimit) {
+ fPositionInCache = fBreaks.size() - 1;
+ if (fPositionInCache >= 0) {
+ U_ASSERT(fBreaks.elementAti(fPositionInCache) == fromPos);
+ }
+ }
+
+ int32_t r;
+ if (fPositionInCache > 0 && fPositionInCache < fBreaks.size() && fBreaks.elementAti(fPositionInCache) == fromPos) {
+ --fPositionInCache;
+ r = fBreaks.elementAti(fPositionInCache);
+ U_ASSERT(r < fromPos);
+ *result = r;
+ *statusIndex = ( r== fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return TRUE;
+ }
+
+ if (fPositionInCache == 0) {
+ fPositionInCache = -1;
+ return FALSE;
+ }
+
+ for (fPositionInCache = fBreaks.size()-1; fPositionInCache >= 0; --fPositionInCache) {
+ r = fBreaks.elementAti(fPositionInCache);
+ if (r < fromPos) {
+ *result = r;
+ *statusIndex = ( r == fStart) ? fFirstRuleStatusIndex : fOtherRuleStatusIndex;
+ return TRUE;
+ }
+ }
+ U_ASSERT(FALSE);
+ fPositionInCache = -1;
+ return FALSE;
+}
+
+void RuleBasedBreakIterator::DictionaryCache::populateDictionary(int32_t startPos, int32_t endPos,
+ int32_t firstRuleStatus, int32_t otherRuleStatus) {
+ if ((endPos - startPos) <= 1) {
+ return;
+ }
+
+ reset();
+ fFirstRuleStatusIndex = firstRuleStatus;
+ fOtherRuleStatusIndex = otherRuleStatus;
+
+ int32_t rangeStart = startPos;
+ int32_t rangeEnd = endPos;
+
+ uint16_t category;
+ int32_t current;
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t foundBreakCount = 0;
+ UText *text = &fBI->fText;
+
+ // Loop through the text, looking for ranges of dictionary characters.
+ // For each span, find the appropriate break engine, and ask it to find
+ // any breaks within the span.
+
+ utext_setNativeIndex(text, rangeStart);
+ UChar32 c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+
+ while(U_SUCCESS(status)) {
+ while((current = (int32_t)UTEXT_GETNATIVEINDEX(text)) < rangeEnd && (category & 0x4000) == 0) {
+ utext_next32(text); // TODO: cleaner loop structure.
+ c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+ }
+ if (current >= rangeEnd) {
+ break;
+ }
+
+ // We now have a dictionary character. Get the appropriate language object
+ // to deal with it.
+ const LanguageBreakEngine *lbe = fBI->getLanguageBreakEngine(c);
+
+ // Ask the language object if there are any breaks. It will add them to the cache and
+ // leave the text pointer on the other side of its range, ready to search for the next one.
+ if (lbe != NULL) {
+ foundBreakCount += lbe->findBreaks(text, rangeStart, rangeEnd, fBreaks);
+ }
+
+ // Reload the loop variables for the next go-round
+ c = utext_current32(text);
+ category = UTRIE2_GET16(fBI->fData->fTrie, c);
+ }
+
+ // If we found breaks, ensure that the first and last entries are
+ // the original starting and ending position. And initialize the
+ // cache iteration position to the first entry.
+
+ // printf("foundBreakCount = %d\n", foundBreakCount);
+ if (foundBreakCount > 0) {
+ U_ASSERT(foundBreakCount == fBreaks.size());
+ if (startPos < fBreaks.elementAti(0)) {
+ // The dictionary did not place a boundary at the start of the segment of text.
+ // Add one now. This should not commonly happen, but it would be easy for interactions
+ // of the rules for dictionary segments and the break engine implementations to
+ // inadvertently cause it. Cover it here, just in case.
+ fBreaks.insertElementAt(startPos, 0, status);
+ }
+ if (endPos > fBreaks.peeki()) {
+ fBreaks.push(endPos, status);
+ }
+ fPositionInCache = 0;
+ // Note: Dictionary matching may extend beyond the original limit.
+ fStart = fBreaks.elementAti(0);
+ fLimit = fBreaks.peeki();
+ } else {
+ // there were no language-based breaks, even though the segment contained
+ // dictionary characters. Subsequent attempts to fetch boundaries from the dictionary cache
+ // for this range will fail, and the calling code will fall back to the rule based boundaries.
+ }
+}
+
+
+/*
+ * BreakCache implemetation
+ */
+
+RuleBasedBreakIterator::BreakCache::BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status) :
+ fBI(bi), fSideBuffer(status) {
+ reset();
+}
+
+
+RuleBasedBreakIterator::BreakCache::~BreakCache() {
+}
+
+
+void RuleBasedBreakIterator::BreakCache::reset(int32_t pos, int32_t ruleStatus) {
+ fStartBufIdx = 0;
+ fEndBufIdx = 0;
+ fTextIdx = pos;
+ fBufIdx = 0;
+ fBoundaries[0] = pos;
+ fStatuses[0] = (uint16_t)ruleStatus;
+}
+
+
+int32_t RuleBasedBreakIterator::BreakCache::current() {
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ fBI->fDone = FALSE;
+ return fTextIdx;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::following(int32_t startPos, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos, status)) {
+ // startPos is in the cache. Do a next() from that position.
+ // TODO: an awkward set of interactions with bi->fDone
+ // seek() does not clear it; it can't because of interactions with populateNear().
+ // next() does not clear it in the fast-path case, where everything matters. Maybe it should.
+ // So clear it here, for the case where seek() succeeded on an iterator that had previously run off the end.
+ fBI->fDone = false;
+ next();
+ }
+ return;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::preceding(int32_t startPos, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ if (startPos == fTextIdx || seek(startPos) || populateNear(startPos, status)) {
+ if (startPos == fTextIdx) {
+ previous(status);
+ } else {
+ // seek() leaves the BreakCache positioned at the preceding boundary
+ // if the requested position is between two bounaries.
+ // current() pushes the BreakCache position out to the BreakIterator itself.
+ U_ASSERT(startPos > fTextIdx);
+ current();
+ }
+ }
+ return;
+}
+
+
+/*
+ * Out-of-line code for BreakCache::next().
+ * Cache does not already contain the boundary
+ */
+void RuleBasedBreakIterator::BreakCache::nextOL() {
+ fBI->fDone = !populateFollowing();
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ return;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::previous(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ int32_t initialBufIdx = fBufIdx;
+ if (fBufIdx == fStartBufIdx) {
+ // At start of cache. Prepend to it.
+ populatePreceding(status);
+ } else {
+ // Cache already holds the next boundary
+ fBufIdx = modChunkSize(fBufIdx - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ }
+ fBI->fDone = (fBufIdx == initialBufIdx);
+ fBI->fPosition = fTextIdx;
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ return;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::seek(int32_t pos) {
+ if (pos < fBoundaries[fStartBufIdx] || pos > fBoundaries[fEndBufIdx]) {
+ return FALSE;
+ }
+ if (pos == fBoundaries[fStartBufIdx]) {
+ // Common case: seek(0), from BreakIterator::first()
+ fBufIdx = fStartBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return TRUE;
+ }
+ if (pos == fBoundaries[fEndBufIdx]) {
+ fBufIdx = fEndBufIdx;
+ fTextIdx = fBoundaries[fBufIdx];
+ return TRUE;
+ }
+
+ int32_t min = fStartBufIdx;
+ int32_t max = fEndBufIdx;
+ while (min != max) {
+ int32_t probe = (min + max + (min>max ? CACHE_SIZE : 0)) / 2;
+ probe = modChunkSize(probe);
+ if (fBoundaries[probe] > pos) {
+ max = probe;
+ } else {
+ min = modChunkSize(probe + 1);
+ }
+ }
+ U_ASSERT(fBoundaries[max] > pos);
+ fBufIdx = modChunkSize(max - 1);
+ fTextIdx = fBoundaries[fBufIdx];
+ U_ASSERT(fTextIdx <= pos);
+ return TRUE;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::populateNear(int32_t position, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return FALSE;
+ }
+ U_ASSERT(position < fBoundaries[fStartBufIdx] || position > fBoundaries[fEndBufIdx]);
+
+ // Find a boundary somewhere in the vicinity of the requested position.
+ // Depending on the safe rules and the text data, it could be either before, at, or after
+ // the requested position.
+
+
+ // If the requested position is not near already cached positions, clear the existing cache,
+ // find a near-by boundary and begin new cache contents there.
+
+ if ((position < fBoundaries[fStartBufIdx] - 15) || position > (fBoundaries[fEndBufIdx] + 15)) {
+ int32_t aBoundary = 0;
+ int32_t ruleStatusIndex = 0;
+ if (position > 20) {
+ int32_t backupPos = fBI->handleSafePrevious(position);
+
+ if (backupPos > 0) {
+ // Advance to the boundary following the backup position.
+ // There is a complication: the safe reverse rules identify pairs of code points
+ // that are safe. If advancing from the safe point moves forwards by less than
+ // two code points, we need to advance one more time to ensure that the boundary
+ // is good, including a correct rules status value.
+ //
+ fBI->fPosition = backupPos;
+ aBoundary = fBI->handleNext();
+ if (aBoundary <= backupPos + 4) {
+ // +4 is a quick test for possibly having advanced only one codepoint.
+ // Four being the length of the longest potential code point, a supplementary in UTF-8
+ utext_setNativeIndex(&fBI->fText, aBoundary);
+ if (backupPos == utext_getPreviousNativeIndex(&fBI->fText)) {
+ // The initial handleNext() only advanced by a single code point. Go again.
+ aBoundary = fBI->handleNext(); // Safe rules identify safe pairs.
+ }
+ }
+ ruleStatusIndex = fBI->fRuleStatusIndex;
+ }
+ }
+ reset(aBoundary, ruleStatusIndex); // Reset cache to hold aBoundary as a single starting point.
+ }
+
+ // Fill in boundaries between existing cache content and the new requested position.
+
+ if (fBoundaries[fEndBufIdx] < position) {
+ // The last position in the cache precedes the requested position.
+ // Add following position(s) to the cache.
+ while (fBoundaries[fEndBufIdx] < position) {
+ if (!populateFollowing()) {
+ U_ASSERT(false);
+ return false;
+ }
+ }
+ fBufIdx = fEndBufIdx; // Set iterator position to the end of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populateFollowing may add extra boundaries.
+ while (fTextIdx > position) { // Move backwards to a position at or preceding the requested pos.
+ previous(status);
+ }
+ return true;
+ }
+
+ if (fBoundaries[fStartBufIdx] > position) {
+ // The first position in the cache is beyond the requested position.
+ // back up more until we get a boundary <= the requested position.
+ while (fBoundaries[fStartBufIdx] > position) {
+ populatePreceding(status);
+ }
+ fBufIdx = fStartBufIdx; // Set iterator position to the start of the buffer.
+ fTextIdx = fBoundaries[fBufIdx]; // Required because populatePreceding may add extra boundaries.
+ while (fTextIdx < position) { // Move forwards to a position at or following the requested pos.
+ next();
+ }
+ if (fTextIdx > position) {
+ // If position is not itself a boundary, the next() loop above will overshoot.
+ // Back up one, leaving cache position at the boundary preceding the requested position.
+ previous(status);
+ }
+ return true;
+ }
+
+ U_ASSERT(fTextIdx == position);
+ return true;
+}
+
+
+
+UBool RuleBasedBreakIterator::BreakCache::populateFollowing() {
+ int32_t fromPosition = fBoundaries[fEndBufIdx];
+ int32_t fromRuleStatusIdx = fStatuses[fEndBufIdx];
+ int32_t pos = 0;
+ int32_t ruleStatusIdx = 0;
+
+ if (fBI->fDictionaryCache->following(fromPosition, &pos, &ruleStatusIdx)) {
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+ return TRUE;
+ }
+
+ fBI->fPosition = fromPosition;
+ pos = fBI->handleNext();
+ if (pos == UBRK_DONE) {
+ return FALSE;
+ }
+
+ ruleStatusIdx = fBI->fRuleStatusIndex;
+ if (fBI->fDictionaryCharCount > 0) {
+ // The text segment obtained from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ fBI->fDictionaryCache->populateDictionary(fromPosition, pos, fromRuleStatusIdx, ruleStatusIdx);
+ if (fBI->fDictionaryCache->following(fromPosition, &pos, &ruleStatusIdx)) {
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+ return TRUE;
+ // TODO: may want to move a sizable chunk of dictionary cache to break cache at this point.
+ // But be careful with interactions with populateNear().
+ }
+ }
+
+ // Rule based segment did not include dictionary characters.
+ // Or, it did contain dictionary chars, but the dictionary segmenter didn't handle them,
+ // meaning that we didn't take the return, above.
+ // Add its end point to the cache.
+ addFollowing(pos, ruleStatusIdx, UpdateCachePosition);
+
+ // Add several non-dictionary boundaries at this point, to optimize straight forward iteration.
+ // (subsequent calls to BreakIterator::next() will take the fast path, getting cached results.
+ //
+ for (int count=0; count<6; ++count) {
+ pos = fBI->handleNext();
+ if (pos == UBRK_DONE || fBI->fDictionaryCharCount > 0) {
+ break;
+ }
+ addFollowing(pos, fBI->fRuleStatusIndex, RetainCachePosition);
+ }
+
+ return TRUE;
+}
+
+
+UBool RuleBasedBreakIterator::BreakCache::populatePreceding(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return FALSE;
+ }
+
+ int32_t fromPosition = fBoundaries[fStartBufIdx];
+ if (fromPosition == 0) {
+ return FALSE;
+ }
+
+ int32_t position = 0;
+ int32_t positionStatusIdx = 0;
+
+ if (fBI->fDictionaryCache->preceding(fromPosition, &position, &positionStatusIdx)) {
+ addPreceding(position, positionStatusIdx, UpdateCachePosition);
+ return TRUE;
+ }
+
+ int32_t backupPosition = fromPosition;
+
+ // Find a boundary somewhere preceding the first already-cached boundary
+ do {
+ backupPosition = backupPosition - 30;
+ if (backupPosition <= 0) {
+ backupPosition = 0;
+ } else {
+ backupPosition = fBI->handleSafePrevious(backupPosition);
+ }
+ if (backupPosition == UBRK_DONE || backupPosition == 0) {
+ position = 0;
+ positionStatusIdx = 0;
+ } else {
+ // Advance to the boundary following the backup position.
+ // There is a complication: the safe reverse rules identify pairs of code points
+ // that are safe. If advancing from the safe point moves forwards by less than
+ // two code points, we need to advance one more time to ensure that the boundary
+ // is good, including a correct rules status value.
+ //
+ fBI->fPosition = backupPosition;
+ position = fBI->handleNext();
+ if (position <= backupPosition + 4) {
+ // +4 is a quick test for possibly having advanced only one codepoint.
+ // Four being the length of the longest potential code point, a supplementary in UTF-8
+ utext_setNativeIndex(&fBI->fText, position);
+ if (backupPosition == utext_getPreviousNativeIndex(&fBI->fText)) {
+ // The initial handleNext() only advanced by a single code point. Go again.
+ position = fBI->handleNext(); // Safe rules identify safe pairs.
+ }
+ };
+ positionStatusIdx = fBI->fRuleStatusIndex;
+ }
+ } while (position >= fromPosition);
+
+ // Find boundaries between the one we just located and the first already-cached boundary
+ // Put them in a side buffer, because we don't yet know where they will fall in the circular cache buffer..
+
+ fSideBuffer.removeAllElements();
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+
+ do {
+ int32_t prevPosition = fBI->fPosition = position;
+ int32_t prevStatusIdx = positionStatusIdx;
+ position = fBI->handleNext();
+ positionStatusIdx = fBI->fRuleStatusIndex;
+ if (position == UBRK_DONE) {
+ break;
+ }
+
+ UBool segmentHandledByDictionary = FALSE;
+ if (fBI->fDictionaryCharCount != 0) {
+ // Segment from the rules includes dictionary characters.
+ // Subdivide it, with subdivided results going into the dictionary cache.
+ int32_t dictSegEndPosition = position;
+ fBI->fDictionaryCache->populateDictionary(prevPosition, dictSegEndPosition, prevStatusIdx, positionStatusIdx);
+ while (fBI->fDictionaryCache->following(prevPosition, &position, &positionStatusIdx)) {
+ segmentHandledByDictionary = true;
+ U_ASSERT(position > prevPosition);
+ if (position >= fromPosition) {
+ break;
+ }
+ U_ASSERT(position <= dictSegEndPosition);
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+ prevPosition = position;
+ }
+ U_ASSERT(position==dictSegEndPosition || position>=fromPosition);
+ }
+
+ if (!segmentHandledByDictionary && position < fromPosition) {
+ fSideBuffer.addElement(position, status);
+ fSideBuffer.addElement(positionStatusIdx, status);
+ }
+ } while (position < fromPosition);
+
+ // Move boundaries from the side buffer to the main circular buffer.
+ UBool success = FALSE;
+ if (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.popi();
+ position = fSideBuffer.popi();
+ addPreceding(position, positionStatusIdx, UpdateCachePosition);
+ success = TRUE;
+ }
+
+ while (!fSideBuffer.isEmpty()) {
+ positionStatusIdx = fSideBuffer.popi();
+ position = fSideBuffer.popi();
+ if (!addPreceding(position, positionStatusIdx, RetainCachePosition)) {
+ // No space in circular buffer to hold a new preceding result while
+ // also retaining the current cache (iteration) position.
+ // Bailing out is safe; the cache will refill again if needed.
+ break;
+ }
+ }
+
+ return success;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::addFollowing(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update) {
+ U_ASSERT(position > fBoundaries[fEndBufIdx]);
+ U_ASSERT(ruleStatusIdx <= UINT16_MAX);
+ int32_t nextIdx = modChunkSize(fEndBufIdx + 1);
+ if (nextIdx == fStartBufIdx) {
+ fStartBufIdx = modChunkSize(fStartBufIdx + 6); // TODO: experiment. Probably revert to 1.
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = static_cast(ruleStatusIdx);
+ fEndBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ // Set current position to the newly added boundary.
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ } else {
+ // Retaining the original cache position.
+ // Check if the added boundary wraps around the buffer, and would over-write the original position.
+ // It's the responsibility of callers of this function to not add too many.
+ U_ASSERT(nextIdx != fBufIdx);
+ }
+}
+
+bool RuleBasedBreakIterator::BreakCache::addPreceding(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update) {
+ U_ASSERT(position < fBoundaries[fStartBufIdx]);
+ U_ASSERT(ruleStatusIdx <= UINT16_MAX);
+ int32_t nextIdx = modChunkSize(fStartBufIdx - 1);
+ if (nextIdx == fEndBufIdx) {
+ if (fBufIdx == fEndBufIdx && update == RetainCachePosition) {
+ // Failure. The insertion of the new boundary would claim the buffer position that is the
+ // current iteration position. And we also want to retain the current iteration position.
+ // (The buffer is already completely full of entries that precede the iteration position.)
+ return false;
+ }
+ fEndBufIdx = modChunkSize(fEndBufIdx - 1);
+ }
+ fBoundaries[nextIdx] = position;
+ fStatuses[nextIdx] = static_cast(ruleStatusIdx);
+ fStartBufIdx = nextIdx;
+ if (update == UpdateCachePosition) {
+ fBufIdx = nextIdx;
+ fTextIdx = position;
+ }
+ return true;
+}
+
+
+void RuleBasedBreakIterator::BreakCache::dumpCache() {
+#ifdef RBBI_DEBUG
+ RBBIDebugPrintf("fTextIdx:%d fBufIdx:%d\n", fTextIdx, fBufIdx);
+ for (int32_t i=fStartBufIdx; ; i=modChunkSize(i+1)) {
+ RBBIDebugPrintf("%d %d\n", i, fBoundaries[i]);
+ if (i == fEndBufIdx) {
+ break;
+ }
+ }
+#endif
+}
+
+U_NAMESPACE_END
+
+#endif // #if !UCONFIG_NO_BREAK_ITERATION
diff --git a/intl/icu/source/common/rbbi_cache.h b/intl/icu/source/common/rbbi_cache.h
new file mode 100644
index 0000000000..b4338c3749
--- /dev/null
+++ b/intl/icu/source/common/rbbi_cache.h
@@ -0,0 +1,203 @@
+// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// file: rbbi_cache.h
+//
+#ifndef RBBI_CACHE_H
+#define RBBI_CACHE_H
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+#include "unicode/rbbi.h"
+#include "unicode/uobject.h"
+
+#include "uvectr32.h"
+
+U_NAMESPACE_BEGIN
+
+/* DictionaryCache stores the boundaries obtained from a run of dictionary characters.
+ * Dictionary boundaries are moved first to this cache, then from here
+ * to the main BreakCache, where they may inter-leave with non-dictionary
+ * boundaries. The public BreakIterator API always fetches directly
+ * from the main BreakCache, not from here.
+ *
+ * In common situations, the number of boundaries in a single dictionary run
+ * should be quite small, it will be terminated by punctuation, spaces,
+ * or any other non-dictionary characters. The main BreakCache may end
+ * up with boundaries from multiple dictionary based runs.
+ *
+ * The boundaries are stored in a simple ArrayList (vector), with the
+ * assumption that they will be accessed sequentially.
+ */
+class RuleBasedBreakIterator::DictionaryCache: public UMemory {
+ public:
+ DictionaryCache(RuleBasedBreakIterator *bi, UErrorCode &status);
+ ~DictionaryCache();
+
+ void reset();
+
+ UBool following(int32_t fromPos, int32_t *pos, int32_t *statusIndex);
+ UBool preceding(int32_t fromPos, int32_t *pos, int32_t *statusIndex);
+
+ /**
+ * Populate the cache with the dictionary based boundaries within a region of text.
+ * @param startPos The start position of a range of text
+ * @param endPos The end position of a range of text
+ * @param firstRuleStatus The rule status index that applies to the break at startPos
+ * @param otherRuleStatus The rule status index that applies to boundaries other than startPos
+ * @internal
+ */
+ void populateDictionary(int32_t startPos, int32_t endPos,
+ int32_t firstRuleStatus, int32_t otherRuleStatus);
+
+
+
+ RuleBasedBreakIterator *fBI;
+
+ UVector32 fBreaks; // A vector containing the boundaries.
+ int32_t fPositionInCache; // Index in fBreaks of last boundary returned by following()
+ // or preceding(). Optimizes sequential access.
+ int32_t fStart; // Text position of first boundary in cache.
+ int32_t fLimit; // Last boundary in cache. Which is the limit of the
+ // text segment being handled by the dictionary.
+ int32_t fFirstRuleStatusIndex; // Rule status info for first boundary.
+ int32_t fOtherRuleStatusIndex; // Rule status info for 2nd through last boundaries.
+};
+
+
+/*
+ * class BreakCache
+ *
+ * Cache of break boundary positions and rule status values.
+ * Break iterator API functions, next(), previous(), etc., will use cached results
+ * when possible, and otherwise cache new results as they are obtained.
+ *
+ * Uniformly caches both dictionary and rule based (non-dictionary) boundaries.
+ *
+ * The cache is implemented as a single circular buffer.
+ */
+
+/*
+ * size of the circular cache buffer.
+ */
+
+class RuleBasedBreakIterator::BreakCache: public UMemory {
+ public:
+ BreakCache(RuleBasedBreakIterator *bi, UErrorCode &status);
+ virtual ~BreakCache();
+ void reset(int32_t pos = 0, int32_t ruleStatus = 0);
+ void next() { if (fBufIdx == fEndBufIdx) {
+ nextOL();
+ } else {
+ fBufIdx = modChunkSize(fBufIdx + 1);
+ fTextIdx = fBI->fPosition = fBoundaries[fBufIdx];
+ fBI->fRuleStatusIndex = fStatuses[fBufIdx];
+ }
+ };
+
+
+ void nextOL();
+ void previous(UErrorCode &status);
+
+ // Move the iteration state to the position following the startPosition.
+ // Input position must be pinned to the input length.
+ void following(int32_t startPosition, UErrorCode &status);
+
+ void preceding(int32_t startPosition, UErrorCode &status);
+
+ /*
+ * Update the state of the public BreakIterator (fBI) to reflect the
+ * current state of the break iterator cache (this).
+ */
+ int32_t current();
+
+ /**
+ * Add boundaries to the cache near the specified position.
+ * The given position need not be a boundary itself.
+ * The input position must be within the range of the text, and
+ * on a code point boundary.
+ * If the requested position is a break boundary, leave the iteration
+ * position on it.
+ * If the requested position is not a boundary, leave the iteration
+ * position on the preceding boundary and include both the
+ * preceding and following boundaries in the cache.
+ * Additional boundaries, either preceding or following, may be added
+ * to the cache as a side effect.
+ *
+ * Return FALSE if the operation failed.
+ */
+ UBool populateNear(int32_t position, UErrorCode &status);
+
+ /**
+ * Add boundary(s) to the cache following the current last boundary.
+ * Return FALSE if at the end of the text, and no more boundaries can be added.
+ * Leave iteration position at the first newly added boundary, or unchanged if no boundary was added.
+ */
+ UBool populateFollowing();
+
+ /**
+ * Add one or more boundaries to the cache preceding the first currently cached boundary.
+ * Leave the iteration position on the first added boundary.
+ * Return false if no boundaries could be added (if at the start of the text.)
+ */
+ UBool populatePreceding(UErrorCode &status);
+
+ enum UpdatePositionValues {
+ RetainCachePosition = 0,
+ UpdateCachePosition = 1
+ };
+
+ /*
+ * Add the boundary following the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ void addFollowing(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update);
+
+
+ /*
+ * Add the boundary preceding the current position.
+ * The current position can be left as it was, or changed to the newly added boundary,
+ * as specified by the update parameter.
+ */
+ bool addPreceding(int32_t position, int32_t ruleStatusIdx, UpdatePositionValues update);
+
+ /**
+ * Set the cache position to the specified position, or, if the position
+ * falls between to cached boundaries, to the preceding boundary.
+ * Fails if the requested position is outside of the range of boundaries currently held by the cache.
+ * The startPosition must be on a code point boundary.
+ *
+ * Return TRUE if successful, FALSE if the specified position is after
+ * the last cached boundary or before the first.
+ */
+ UBool seek(int32_t startPosition);
+
+ void dumpCache();
+
+ private:
+ static inline int32_t modChunkSize(int index) { return index & (CACHE_SIZE - 1); };
+
+ static constexpr int32_t CACHE_SIZE = 128;
+ static_assert((CACHE_SIZE & (CACHE_SIZE-1)) == 0, "CACHE_SIZE must be power of two.");
+
+ RuleBasedBreakIterator *fBI;
+ int32_t fStartBufIdx;
+ int32_t fEndBufIdx; // inclusive
+
+ int32_t fTextIdx;
+ int32_t fBufIdx;
+
+ int32_t fBoundaries[CACHE_SIZE];
+ uint16_t fStatuses[CACHE_SIZE];
+
+ UVector32 fSideBuffer;
+};
+
+U_NAMESPACE_END
+
+#endif // #if !UCONFIG_NO_BREAK_ITERATION
+
+#endif // RBBI_CACHE_H
diff --git a/intl/icu/source/common/rbbidata.cpp b/intl/icu/source/common/rbbidata.cpp
index 26e11ac2eb..1d4c9e5895 100644
--- a/intl/icu/source/common/rbbidata.cpp
+++ b/intl/icu/source/common/rbbidata.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
***************************************************************************
@@ -14,7 +14,7 @@
#include "unicode/utypes.h"
#include "rbbidata.h"
#include "rbbirb.h"
-#include "utrie.h"
+#include "utrie2.h"
#include "udatamem.h"
#include "cmemory.h"
#include "cstring.h"
@@ -23,23 +23,6 @@
#include "uassert.h"
-//-----------------------------------------------------------------------------------
-//
-// Trie access folding function. Copied as-is from properties code in uchar.c
-//
-//-----------------------------------------------------------------------------------
-U_CDECL_BEGIN
-static int32_t U_CALLCONV
-getFoldingOffset(uint32_t data) {
- /* if bit 15 is set, then the folding offset is in bits 14..0 of the 16-bit trie result */
- if(data&0x8000) {
- return (int32_t)(data&0x7fff);
- } else {
- return 0;
- }
-}
-U_CDECL_END
-
U_NAMESPACE_BEGIN
//-----------------------------------------------------------------------------
@@ -71,9 +54,8 @@ RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) {
dh->info.dataFormat[0] == 0x42 && // dataFormat="Brk "
dh->info.dataFormat[1] == 0x72 &&
dh->info.dataFormat[2] == 0x6b &&
- dh->info.dataFormat[3] == 0x20)
- // Note: info.fFormatVersion is duplicated in the RBBIDataHeader, and is
- // validated when checking that.
+ dh->info.dataFormat[3] == 0x20 &&
+ isDataVersionAcceptable(dh->info.formatVersion))
) {
status = U_INVALID_FORMAT_ERROR;
return;
@@ -84,6 +66,11 @@ RBBIDataWrapper::RBBIDataWrapper(UDataMemory* udm, UErrorCode &status) {
fUDataMem = udm;
}
+UBool RBBIDataWrapper::isDataVersionAcceptable(const UVersionInfo version) {
+ return RBBI_DATA_FORMAT_VERSION[0] == version[0];
+}
+
+
//-----------------------------------------------------------------------------
//
// init(). Does most of the work of construction, shared between the
@@ -94,12 +81,11 @@ void RBBIDataWrapper::init0() {
fHeader = NULL;
fForwardTable = NULL;
fReverseTable = NULL;
- fSafeFwdTable = NULL;
- fSafeRevTable = NULL;
- fRuleSource = NULL;
+ fRuleSource = NULL;
fRuleStatusTable = NULL;
- fUDataMem = NULL;
- fRefCount = 0;
+ fTrie = NULL;
+ fUDataMem = NULL;
+ fRefCount = 0;
fDontFreeData = TRUE;
}
@@ -108,8 +94,7 @@ void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
return;
}
fHeader = data;
- if (fHeader->fMagic != 0xb1a0 || fHeader->fFormatVersion[0] != 3)
- {
+ if (fHeader->fMagic != 0xb1a0 || !isDataVersionAcceptable(fHeader->fFormatVersion)) {
status = U_INVALID_FORMAT_ERROR;
return;
}
@@ -124,23 +109,15 @@ void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
if (data->fRTableLen != 0) {
fReverseTable = (RBBIStateTable *)((char *)data + fHeader->fRTable);
}
- if (data->fSFTableLen != 0) {
- fSafeFwdTable = (RBBIStateTable *)((char *)data + fHeader->fSFTable);
- }
- if (data->fSRTableLen != 0) {
- fSafeRevTable = (RBBIStateTable *)((char *)data + fHeader->fSRTable);
- }
-
- utrie_unserialize(&fTrie,
- (uint8_t *)data + fHeader->fTrie,
- fHeader->fTrieLen,
- &status);
+ fTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
+ (uint8_t *)data + fHeader->fTrie,
+ fHeader->fTrieLen,
+ NULL, // *actual length
+ &status);
if (U_FAILURE(status)) {
return;
}
- fTrie.getFoldingOffset=getFoldingOffset;
-
fRuleSource = (UChar *)((char *)data + fHeader->fRuleSource);
fRuleString.setTo(TRUE, fRuleSource, -1);
@@ -165,6 +142,8 @@ void RBBIDataWrapper::init(const RBBIDataHeader *data, UErrorCode &status) {
//-----------------------------------------------------------------------------
RBBIDataWrapper::~RBBIDataWrapper() {
U_ASSERT(fRefCount == 0);
+ utrie2_close(fTrie);
+ fTrie = NULL;
if (fUDataMem) {
udata_close(fUDataMem);
} else if (!fDontFreeData) {
@@ -271,8 +250,8 @@ void RBBIDataWrapper::printTable(const char *heading, const RBBIStateTable *tab
#endif
-#ifdef RBBI_DEBUG
void RBBIDataWrapper::printData() {
+#ifdef RBBI_DEBUG
RBBIDebugPrintf("RBBI Data at %p\n", (void *)fHeader);
RBBIDebugPrintf(" Version = {%d %d %d %d}\n", fHeader->fFormatVersion[0], fHeader->fFormatVersion[1],
fHeader->fFormatVersion[2], fHeader->fFormatVersion[3]);
@@ -281,16 +260,14 @@ void RBBIDataWrapper::printData() {
printTable("Forward State Transition Table", fForwardTable);
printTable("Reverse State Transition Table", fReverseTable);
- printTable("Safe Forward State Transition Table", fSafeFwdTable);
- printTable("Safe Reverse State Transition Table", fSafeRevTable);
RBBIDebugPrintf("\nOrignal Rules source:\n");
for (int32_t c=0; fRuleSource[c] != 0; c++) {
RBBIDebugPrintf("%c", fRuleSource[c]);
}
RBBIDebugPrintf("\n\n");
-}
#endif
+}
U_NAMESPACE_END
@@ -323,7 +300,7 @@ ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outD
pInfo->dataFormat[1]==0x72 &&
pInfo->dataFormat[2]==0x6b &&
pInfo->dataFormat[3]==0x20 &&
- pInfo->formatVersion[0]==3 )) {
+ RBBIDataWrapper::isDataVersionAcceptable(pInfo->formatVersion) )) {
udata_printError(ds, "ubrk_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n",
pInfo->dataFormat[0], pInfo->dataFormat[1],
pInfo->dataFormat[2], pInfo->dataFormat[3],
@@ -344,17 +321,11 @@ ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outD
//
// Get the RRBI Data Header, and check that it appears to be OK.
//
- // Note: ICU 3.2 and earlier, RBBIDataHeader::fDataFormat was actually
- // an int32_t with a value of 1. Starting with ICU 3.4,
- // RBBI's fDataFormat matches the dataFormat field from the
- // UDataInfo header, four int8_t bytes. The value is {3,1,0,0}
- //
const uint8_t *inBytes =(const uint8_t *)inData+headerSize;
RBBIDataHeader *rbbiDH = (RBBIDataHeader *)inBytes;
if (ds->readUInt32(rbbiDH->fMagic) != 0xb1a0 ||
- rbbiDH->fFormatVersion[0] != 3 ||
- ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader))
- {
+ !RBBIDataWrapper::isDataVersionAcceptable(rbbiDH->fFormatVersion) ||
+ ds->readUInt32(rbbiDH->fLength) < sizeof(RBBIDataHeader)) {
udata_printError(ds, "ubrk_swap(): RBBI Data header is invalid.\n");
*status=U_UNSUPPORTED_ERROR;
return 0;
@@ -428,31 +399,9 @@ ubrk_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outD
outBytes+tableStartOffset+topSize, status);
}
- // Safe Forward state table. Same layout as forward table, above.
- tableStartOffset = ds->readUInt32(rbbiDH->fSFTable);
- tableLength = ds->readUInt32(rbbiDH->fSFTableLen);
-
- if (tableLength > 0) {
- ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
- outBytes+tableStartOffset, status);
- ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
- outBytes+tableStartOffset+topSize, status);
- }
-
- // Safe Reverse state table. Same layout as forward table, above.
- tableStartOffset = ds->readUInt32(rbbiDH->fSRTable);
- tableLength = ds->readUInt32(rbbiDH->fSRTableLen);
-
- if (tableLength > 0) {
- ds->swapArray32(ds, inBytes+tableStartOffset, topSize,
- outBytes+tableStartOffset, status);
- ds->swapArray16(ds, inBytes+tableStartOffset+topSize, tableLength-topSize,
- outBytes+tableStartOffset+topSize, status);
- }
-
// Trie table for character categories
- utrie_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
- outBytes+ds->readUInt32(rbbiDH->fTrie), status);
+ utrie2_swap(ds, inBytes+ds->readUInt32(rbbiDH->fTrie), ds->readUInt32(rbbiDH->fTrieLen),
+ outBytes+ds->readUInt32(rbbiDH->fTrie), status);
// Source Rules Text. It's UChar data
ds->swapArray16(ds, inBytes+ds->readUInt32(rbbiDH->fRuleSource), ds->readUInt32(rbbiDH->fRuleSourceLen),
diff --git a/intl/icu/source/common/rbbidata.h b/intl/icu/source/common/rbbidata.h
index 1d6f37d4b4..b7de6ce073 100644
--- a/intl/icu/source/common/rbbidata.h
+++ b/intl/icu/source/common/rbbidata.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: rbbidata.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -51,22 +51,23 @@ ubrk_swap(const UDataSwapper *ds,
#include "unicode/uobject.h"
#include "unicode/unistr.h"
+#include "unicode/uversion.h"
#include "umutex.h"
-#include "utrie.h"
+#include "utrie2.h"
U_NAMESPACE_BEGIN
+// The current RBBI data format version.
+static const uint8_t RBBI_DATA_FORMAT_VERSION[] = {5, 0, 0, 0};
+
/*
* The following structs map exactly onto the raw data from ICU common data file.
*/
struct RBBIDataHeader {
uint32_t fMagic; /* == 0xbla0 */
- uint8_t fFormatVersion[4]; /* Data Format. Same as the value in struct UDataInfo */
+ UVersionInfo fFormatVersion; /* Data Format. Same as the value in struct UDataInfo */
/* if there is one associated with this data. */
/* (version originates in rbbi, is copied to UDataInfo) */
- /* For ICU 3.2 and earlier, this field was */
- /* uint32_t fVersion */
- /* with a value of 1. */
uint32_t fLength; /* Total length in bytes of this RBBI Data, */
/* including all sections, not just the header. */
uint32_t fCatCount; /* Number of character categories. */
@@ -80,10 +81,6 @@ struct RBBIDataHeader {
uint32_t fFTableLen;
uint32_t fRTable; /* Offset to the reverse state transition table. */
uint32_t fRTableLen;
- uint32_t fSFTable; /* safe point forward transition table */
- uint32_t fSFTableLen;
- uint32_t fSRTable; /* safe point reverse transition table */
- uint32_t fSRTableLen;
uint32_t fTrie; /* Offset to Trie data for character categories */
uint32_t fTrieLen;
uint32_t fRuleSource; /* Offset to the source for for the break */
@@ -115,9 +112,10 @@ struct RBBIStateTableRow {
/* StatusTable of the set of matching */
/* tags (rule status values) */
int16_t fReserved;
- uint16_t fNextState[2]; /* Next State, indexed by char category. */
- /* This array does not have two elements */
- /* Array Size is actually fData->fHeader->fCatCount */
+ uint16_t fNextState[1]; /* Next State, indexed by char category. */
+ /* Variable-length array declared with length 1 */
+ /* to disable bounds checkers. */
+ /* Array Size is actually fData->fHeader->fCatCount*/
/* CAUTION: see RBBITableBuilder::getTableSize() */
/* before changing anything here. */
};
@@ -128,7 +126,9 @@ struct RBBIStateTable {
uint32_t fRowLen; /* Length of a state table row, in bytes. */
uint32_t fFlags; /* Option Flags for this state table */
uint32_t fReserved; /* reserved */
- char fTableData[4]; /* First RBBIStateTableRow begins here. */
+ char fTableData[1]; /* First RBBIStateTableRow begins here. */
+ /* Variable-length array declared with length 1 */
+ /* to disable bounds checkers. */
/* (making it char[] simplifies ugly address */
/* arithmetic for indexing variable length rows.) */
};
@@ -152,6 +152,8 @@ public:
RBBIDataWrapper(UDataMemory* udm, UErrorCode &status);
~RBBIDataWrapper();
+ static UBool isDataVersionAcceptable(const UVersionInfo version);
+
void init0();
void init(const RBBIDataHeader *data, UErrorCode &status);
RBBIDataWrapper *addReference();
@@ -159,13 +161,8 @@ public:
UBool operator ==(const RBBIDataWrapper &other) const;
int32_t hashCode();
const UnicodeString &getRuleSourceString() const;
-#ifdef RBBI_DEBUG
void printData();
void printTable(const char *heading, const RBBIStateTable *table);
-#else
- #define printData()
- #define printTable(heading, table)
-#endif
/* */
/* Pointers to items within the data */
@@ -173,19 +170,17 @@ public:
const RBBIDataHeader *fHeader;
const RBBIStateTable *fForwardTable;
const RBBIStateTable *fReverseTable;
- const RBBIStateTable *fSafeFwdTable;
- const RBBIStateTable *fSafeRevTable;
const UChar *fRuleSource;
const int32_t *fRuleStatusTable;
/* number of int32_t values in the rule status table. Used to sanity check indexing */
int32_t fStatusMaxIdx;
- UTrie fTrie;
+ UTrie2 *fTrie;
private:
u_atomic_int32_t fRefCount;
- UDataMemory *fUDataMem;
+ UDataMemory *fUDataMem;
UnicodeString fRuleString;
UBool fDontFreeData;
diff --git a/intl/icu/source/common/rbbinode.cpp b/intl/icu/source/common/rbbinode.cpp
index 123c21e44c..69d84151fe 100644
--- a/intl/icu/source/common/rbbinode.cpp
+++ b/intl/icu/source/common/rbbinode.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
***************************************************************************
diff --git a/intl/icu/source/common/rbbinode.h b/intl/icu/source/common/rbbinode.h
index 74096cc814..f352596022 100644
--- a/intl/icu/source/common/rbbinode.h
+++ b/intl/icu/source/common/rbbinode.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/********************************************************************
* COPYRIGHT:
diff --git a/intl/icu/source/common/rbbirb.cpp b/intl/icu/source/common/rbbirb.cpp
index 3d9fe6fd34..08c577696c 100644
--- a/intl/icu/source/common/rbbirb.cpp
+++ b/intl/icu/source/common/rbbirb.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// file: rbbirb.cpp
@@ -24,16 +24,16 @@
#include "unicode/uchriter.h"
#include "unicode/parsepos.h"
#include "unicode/parseerr.h"
+
#include "cmemory.h"
#include "cstring.h"
-
#include "rbbirb.h"
#include "rbbinode.h"
-
#include "rbbiscan.h"
#include "rbbisetb.h"
#include "rbbitblb.h"
#include "rbbidata.h"
+#include "uassert.h"
U_NAMESPACE_BEGIN
@@ -47,7 +47,7 @@ U_NAMESPACE_BEGIN
RBBIRuleBuilder::RBBIRuleBuilder(const UnicodeString &rules,
UParseError *parseErr,
UErrorCode &status)
- : fRules(rules)
+ : fRules(rules), fStrippedRules(rules)
{
fStatus = &status; // status is checked below
fParseError = parseErr;
@@ -62,10 +62,7 @@ RBBIRuleBuilder::RBBIRuleBuilder(const UnicodeString &rules,
fSafeFwdTree = NULL;
fSafeRevTree = NULL;
fDefaultTree = &fForwardTree;
- fForwardTables = NULL;
- fReverseTables = NULL;
- fSafeFwdTables = NULL;
- fSafeRevTables = NULL;
+ fForwardTable = NULL;
fRuleStatusVals = NULL;
fChainRules = FALSE;
fLBCMNoChain = FALSE;
@@ -114,11 +111,7 @@ RBBIRuleBuilder::~RBBIRuleBuilder() {
delete fUSetNodes;
delete fSetBuilder;
- delete fForwardTables;
- delete fReverseTables;
- delete fSafeFwdTables;
- delete fSafeRevTables;
-
+ delete fForwardTable;
delete fForwardTree;
delete fReverseTree;
delete fSafeFwdTree;
@@ -147,8 +140,9 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
return NULL;
}
- // Remove comments and whitespace from the rules to make it smaller.
- UnicodeString strippedRules((const UnicodeString&)RBBIRuleScanner::stripRules(fRules));
+ // Remove whitespace from the rules to make it smaller.
+ // The rule parser has already removed comments.
+ fStrippedRules = fScanner->stripRules(fStrippedRules);
// Calculate the size of each section in the data.
// Sizes here are padded up to a multiple of 8 for better memory alignment.
@@ -156,16 +150,15 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
// without the padding.
//
int32_t headerSize = align8(sizeof(RBBIDataHeader));
- int32_t forwardTableSize = align8(fForwardTables->getTableSize());
- int32_t reverseTableSize = align8(fReverseTables->getTableSize());
- int32_t safeFwdTableSize = align8(fSafeFwdTables->getTableSize());
- int32_t safeRevTableSize = align8(fSafeRevTables->getTableSize());
+ int32_t forwardTableSize = align8(fForwardTable->getTableSize());
+ int32_t reverseTableSize = align8(fForwardTable->getSafeTableSize());
int32_t trieSize = align8(fSetBuilder->getTrieSize());
int32_t statusTableSize = align8(fRuleStatusVals->size() * sizeof(int32_t));
- int32_t rulesSize = align8((strippedRules.length()+1) * sizeof(UChar));
+ int32_t rulesSize = align8((fStrippedRules.length()+1) * sizeof(UChar));
- int32_t totalSize = headerSize + forwardTableSize + reverseTableSize
- + safeFwdTableSize + safeRevTableSize
+ int32_t totalSize = headerSize
+ + forwardTableSize
+ + reverseTableSize
+ statusTableSize + trieSize + rulesSize;
RBBIDataHeader *data = (RBBIDataHeader *)uprv_malloc(totalSize);
@@ -177,35 +170,30 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
data->fMagic = 0xb1a0;
- data->fFormatVersion[0] = 3;
- data->fFormatVersion[1] = 1;
- data->fFormatVersion[2] = 0;
- data->fFormatVersion[3] = 0;
+ data->fFormatVersion[0] = RBBI_DATA_FORMAT_VERSION[0];
+ data->fFormatVersion[1] = RBBI_DATA_FORMAT_VERSION[1];
+ data->fFormatVersion[2] = RBBI_DATA_FORMAT_VERSION[2];
+ data->fFormatVersion[3] = RBBI_DATA_FORMAT_VERSION[3];
data->fLength = totalSize;
data->fCatCount = fSetBuilder->getNumCharCategories();
data->fFTable = headerSize;
data->fFTableLen = forwardTableSize;
- data->fRTable = data->fFTable + forwardTableSize;
- data->fRTableLen = reverseTableSize;
- data->fSFTable = data->fRTable + reverseTableSize;
- data->fSFTableLen = safeFwdTableSize;
- data->fSRTable = data->fSFTable + safeFwdTableSize;
- data->fSRTableLen = safeRevTableSize;
- data->fTrie = data->fSRTable + safeRevTableSize;
+ data->fRTable = data->fFTable + data->fFTableLen;
+ data->fRTableLen = reverseTableSize;
+
+ data->fTrie = data->fRTable + data->fRTableLen;
data->fTrieLen = fSetBuilder->getTrieSize();
data->fStatusTable = data->fTrie + trieSize;
data->fStatusTableLen= statusTableSize;
data->fRuleSource = data->fStatusTable + statusTableSize;
- data->fRuleSourceLen = strippedRules.length() * sizeof(UChar);
+ data->fRuleSourceLen = fStrippedRules.length() * sizeof(UChar);
uprv_memset(data->fReserved, 0, sizeof(data->fReserved));
- fForwardTables->exportTable((uint8_t *)data + data->fFTable);
- fReverseTables->exportTable((uint8_t *)data + data->fRTable);
- fSafeFwdTables->exportTable((uint8_t *)data + data->fSFTable);
- fSafeRevTables->exportTable((uint8_t *)data + data->fSRTable);
+ fForwardTable->exportTable((uint8_t *)data + data->fFTable);
+ fForwardTable->exportSafeTable((uint8_t *)data + data->fRTable);
fSetBuilder->serializeTrie ((uint8_t *)data + data->fTrie);
int32_t *ruleStatusTable = (int32_t *)((uint8_t *)data + data->fStatusTable);
@@ -213,16 +201,12 @@ RBBIDataHeader *RBBIRuleBuilder::flattenData() {
ruleStatusTable[i] = fRuleStatusVals->elementAti(i);
}
- strippedRules.extract((UChar *)((uint8_t *)data+data->fRuleSource), rulesSize/2+1, *fStatus);
+ fStrippedRules.extract((UChar *)((uint8_t *)data+data->fRuleSource), rulesSize/2+1, *fStatus);
return data;
}
-
-
-
-
//----------------------------------------------------------------------------------------
//
// createRuleBasedBreakIterator construct from source rules that are passed in
@@ -234,8 +218,6 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString &rules,
UParseError *parseError,
UErrorCode &status)
{
- // status checked below
-
//
// Read the input rules, generate a parse tree, symbol table,
// and list of all Unicode Sets referenced by the rules.
@@ -244,61 +226,13 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString &rules,
if (U_FAILURE(status)) { // status checked here bcos build below doesn't
return NULL;
}
- builder.fScanner->parse();
- //
- // UnicodeSet processing.
- // Munge the Unicode Sets to create a set of character categories.
- // Generate the mapping tables (TRIE) from input 32-bit characters to
- // the character categories.
- //
- builder.fSetBuilder->build();
+ RBBIDataHeader *data = builder.build(status);
-
- //
- // Generate the DFA state transition table.
- //
- builder.fForwardTables = new RBBITableBuilder(&builder, &builder.fForwardTree);
- builder.fReverseTables = new RBBITableBuilder(&builder, &builder.fReverseTree);
- builder.fSafeFwdTables = new RBBITableBuilder(&builder, &builder.fSafeFwdTree);
- builder.fSafeRevTables = new RBBITableBuilder(&builder, &builder.fSafeRevTree);
- if (builder.fForwardTables == NULL || builder.fReverseTables == NULL ||
- builder.fSafeFwdTables == NULL || builder.fSafeRevTables == NULL)
- {
- status = U_MEMORY_ALLOCATION_ERROR;
- delete builder.fForwardTables; builder.fForwardTables = NULL;
- delete builder.fReverseTables; builder.fReverseTables = NULL;
- delete builder.fSafeFwdTables; builder.fSafeFwdTables = NULL;
- delete builder.fSafeRevTables; builder.fSafeRevTables = NULL;
- return NULL;
+ if (U_FAILURE(status)) {
+ return nullptr;
}
- builder.fForwardTables->build();
- builder.fReverseTables->build();
- builder.fSafeFwdTables->build();
- builder.fSafeRevTables->build();
-
-#ifdef RBBI_DEBUG
- if (builder.fDebugEnv && uprv_strstr(builder.fDebugEnv, "states")) {
- builder.fForwardTables->printRuleStatusTable();
- }
-#endif
-
- //
- // Package up the compiled data into a memory image
- // in the run-time format.
- //
- RBBIDataHeader *data = builder.flattenData(); // returns NULL if error
- if (U_FAILURE(*builder.fStatus)) {
- return NULL;
- }
-
-
- //
- // Clean up the compiler related stuff
- //
-
-
//
// Create a break iterator from the compiled rules.
// (Identical to creation from stored pre-compiled rules)
@@ -315,6 +249,80 @@ RBBIRuleBuilder::createRuleBasedBreakIterator( const UnicodeString &rules,
return This;
}
+RBBIDataHeader *RBBIRuleBuilder::build(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return nullptr;
+ }
+
+ fScanner->parse();
+ if (U_FAILURE(status)) {
+ return nullptr;
+ }
+
+ //
+ // UnicodeSet processing.
+ // Munge the Unicode Sets to create a set of character categories.
+ // Generate the mapping tables (TRIE) from input code points to
+ // the character categories.
+ //
+ fSetBuilder->buildRanges();
+
+ //
+ // Generate the DFA state transition table.
+ //
+ fForwardTable = new RBBITableBuilder(this, &fForwardTree, status);
+ if (fForwardTable == nullptr) {
+ status = U_MEMORY_ALLOCATION_ERROR;
+ return nullptr;
+ }
+
+ fForwardTable->buildForwardTable();
+ optimizeTables();
+ fForwardTable->buildSafeReverseTable(status);
+
+
+#ifdef RBBI_DEBUG
+ if (fDebugEnv && uprv_strstr(fDebugEnv, "states")) {
+ fForwardTable->printStates();
+ fForwardTable->printRuleStatusTable();
+ fForwardTable->printReverseTable();
+ }
+#endif
+
+ fSetBuilder->buildTrie();
+
+ //
+ // Package up the compiled data into a memory image
+ // in the run-time format.
+ //
+ RBBIDataHeader *data = flattenData(); // returns NULL if error
+ if (U_FAILURE(status)) {
+ return nullptr;
+ }
+ return data;
+}
+
+void RBBIRuleBuilder::optimizeTables() {
+ bool didSomething;
+ do {
+ didSomething = false;
+
+ // Begin looking for duplicates with char class 3.
+ // Classes 0, 1 and 2 are special; they are unused, {bof} and {eof} respectively,
+ // and should not have other categories merged into them.
+ IntPair duplPair = {3, 0};
+ while (fForwardTable->findDuplCharClassFrom(&duplPair)) {
+ fSetBuilder->mergeCategories(duplPair);
+ fForwardTable->removeColumn(duplPair.second);
+ didSomething = true;
+ }
+
+ while (fForwardTable->removeDuplicateStates() > 0) {
+ didSomething = true;
+ }
+ } while (didSomething);
+}
+
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
diff --git a/intl/icu/source/common/rbbirb.h b/intl/icu/source/common/rbbirb.h
index e4f736c8f3..037c1dc2ce 100644
--- a/intl/icu/source/common/rbbirb.h
+++ b/intl/icu/source/common/rbbirb.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// rbbirb.h
@@ -15,6 +15,11 @@
#define RBBIRB_H
#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+#include
+
#include "unicode/uobject.h"
#include "unicode/rbbi.h"
#include "unicode/uniset.h"
@@ -22,8 +27,7 @@
#include "uhash.h"
#include "uvector.h"
#include "unicode/symtable.h"// For UnicodeSet parsing, is the interface that
- // looks up references to $variables within a set.
-
+ // looks up references to $variables within a set.
U_NAMESPACE_BEGIN
@@ -120,13 +124,28 @@ public:
RBBIRuleBuilder(const UnicodeString &rules,
UParseError *parseErr,
UErrorCode &status
- );
+ );
virtual ~RBBIRuleBuilder();
+
+ /**
+ * Build the state tables and char class Trie from the source rules.
+ */
+ RBBIDataHeader *build(UErrorCode &status);
+
+
+ /**
+ * Fold together redundant character classes (table columns) and
+ * redundant states (table rows). Done after initial table generation,
+ * before serializing the result.
+ */
+ void optimizeTables();
+
char *fDebugEnv; // controls debug trace output
UErrorCode *fStatus; // Error reporting. Keeping status
UParseError *fParseError; // here avoids passing it everywhere.
const UnicodeString &fRules; // The rule string that we are compiling
+ UnicodeString fStrippedRules; // The rule string, with comments stripped.
RBBIRuleScanner *fScanner; // The scanner.
RBBINode *fForwardTree; // The parse trees, generated by the scanner,
@@ -150,10 +169,7 @@ public:
RBBISetBuilder *fSetBuilder; // Set and Character Category builder.
UVector *fUSetNodes; // Vector of all uset nodes.
- RBBITableBuilder *fForwardTables; // State transition tables
- RBBITableBuilder *fReverseTables;
- RBBITableBuilder *fSafeFwdTables;
- RBBITableBuilder *fSafeRevTables;
+ RBBITableBuilder *fForwardTable; // State transition table, build time form.
UVector *fRuleStatusVals; // The values that can be returned
// from getRuleStatus().
@@ -188,6 +204,11 @@ struct RBBISetTableEl {
RBBINode *val;
};
+/**
+ * A pair of ints, used to bundle pairs of states or pairs of character classes.
+ */
+typedef std::pair IntPair;
+
//----------------------------------------------------------------------------
//
@@ -207,6 +228,9 @@ struct RBBISetTableEl {
#endif
U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
+
#endif
diff --git a/intl/icu/source/common/rbbirpt.h b/intl/icu/source/common/rbbirpt.h
index b5bc8f7bf3..586953c90c 100644
--- a/intl/icu/source/common/rbbirpt.h
+++ b/intl/icu/source/common/rbbirpt.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//---------------------------------------------------------------------------------
//
diff --git a/intl/icu/source/common/rbbiscan.cpp b/intl/icu/source/common/rbbiscan.cpp
index df30f2c204..f536ab5d39 100644
--- a/intl/icu/source/common/rbbiscan.cpp
+++ b/intl/icu/source/common/rbbiscan.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// file: rbbiscan.cpp
@@ -47,6 +47,7 @@
//
//------------------------------------------------------------------------------
static const UChar gRuleSet_rule_char_pattern[] = {
+ // Characters that may appear as literals in patterns without escaping or quoting.
// [ ^ [ \ p { Z } \ u 0 0 2 0
0x5b, 0x5e, 0x5b, 0x5c, 0x70, 0x7b, 0x5a, 0x7d, 0x5c, 0x75, 0x30, 0x30, 0x32, 0x30,
// - \ u 0 0 7 f ] - [ \ p
@@ -371,7 +372,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
// (forward, reverse, safe_forward, safe_reverse)
// OR this rule into the appropriate group of them.
//
- RBBINode **destRules = (fReverseRule? &fRB->fReverseTree : fRB->fDefaultTree);
+ RBBINode **destRules = (fReverseRule? &fRB->fSafeRevTree : fRB->fDefaultTree);
if (*destRules != NULL) {
// This is not the first rule encounted.
@@ -379,7 +380,7 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
// with the current rule expression (on the Node Stack)
// with the resulting OR expression going to *destRules
//
- RBBINode *thisRule = fNodeStack[fNodeStackPtr];
+ thisRule = fNodeStack[fNodeStackPtr];
RBBINode *prevRules = *destRules;
RBBINode *orNode = pushNewNode(RBBINode::opOr);
if (U_FAILURE(*fRB->fStatus)) {
@@ -558,6 +559,10 @@ UBool RBBIRuleScanner::doParseActions(int32_t action)
fRB->fDefaultTree = &fRB->fSafeRevTree;
} else if (opt == UNICODE_STRING("lookAheadHardBreak", 18)) {
fRB->fLookAheadHardBreak = TRUE;
+ } else if (opt == UNICODE_STRING("quoted_literals_only", 20)) {
+ fRuleSets[kRuleSet_rule_char-128].clear();
+ } else if (opt == UNICODE_STRING("unquoted_literals", 17)) {
+ fRuleSets[kRuleSet_rule_char-128].applyPattern(UnicodeString(gRuleSet_rule_char_pattern), *fRB->fStatus);
} else {
error(U_BRK_UNRECOGNIZED_OPTION);
}
@@ -817,27 +822,24 @@ static const UChar chRParen = 0x29;
//------------------------------------------------------------------------------
//
-// stripRules Return a rules string without unnecessary
-// characters.
+// stripRules Return a rules string without extra spaces.
+// (Comments are removed separately, during rule parsing.)
//
//------------------------------------------------------------------------------
UnicodeString RBBIRuleScanner::stripRules(const UnicodeString &rules) {
UnicodeString strippedRules;
- int rulesLength = rules.length();
- for (int idx = 0; idx < rulesLength; ) {
- UChar ch = rules[idx++];
- if (ch == chPound) {
- while (idx < rulesLength
- && ch != chCR && ch != chLF && ch != chNEL)
- {
- ch = rules[idx++];
- }
- }
- if (!u_isISOControl(ch)) {
- strippedRules.append(ch);
+ int32_t rulesLength = rules.length();
+ bool skippingSpaces = false;
+
+ for (int32_t idx=0; idxfStrippedRules.setCharAt(i, u' ');
+ }
}
if (c.fChar == (UChar32)-1) {
return;
@@ -1116,22 +1122,6 @@ void RBBIRuleScanner::parse() {
return;
}
- //
- // If there were NO user specified reverse rules, set up the equivalent of ".*;"
- //
- if (fRB->fReverseTree == NULL) {
- fRB->fReverseTree = pushNewNode(RBBINode::opStar);
- RBBINode *operand = pushNewNode(RBBINode::setRef);
- if (U_FAILURE(*fRB->fStatus)) {
- return;
- }
- findSetFor(UnicodeString(TRUE, kAny, 3), operand);
- fRB->fReverseTree->fLeftChild = operand;
- operand->fParent = fRB->fReverseTree;
- fNodeStackPtr -= 2;
- }
-
-
//
// Parsing of the input RBBI rules is complete.
// We now have a parse tree for the rule expressions
@@ -1179,13 +1169,12 @@ RBBINode *RBBIRuleScanner::pushNewNode(RBBINode::NodeType t) {
if (U_FAILURE(*fRB->fStatus)) {
return NULL;
}
- fNodeStackPtr++;
- if (fNodeStackPtr >= kStackSize) {
- error(U_BRK_INTERNAL_ERROR);
+ if (fNodeStackPtr >= kStackSize - 1) {
+ error(U_BRK_RULE_SYNTAX);
RBBIDebugPuts("RBBIRuleScanner::pushNewNode - stack overflow.");
- *fRB->fStatus = U_BRK_INTERNAL_ERROR;
return NULL;
}
+ fNodeStackPtr++;
fNodeStack[fNodeStackPtr] = new RBBINode(t);
if (fNodeStack[fNodeStackPtr] == NULL) {
*fRB->fStatus = U_MEMORY_ALLOCATION_ERROR;
diff --git a/intl/icu/source/common/rbbiscan.h b/intl/icu/source/common/rbbiscan.h
index 6f3267e21a..8bec8234e5 100644
--- a/intl/icu/source/common/rbbiscan.h
+++ b/intl/icu/source/common/rbbiscan.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// rbbiscan.h
diff --git a/intl/icu/source/common/rbbisetb.cpp b/intl/icu/source/common/rbbisetb.cpp
index ebdf8b7fb9..36e2e07e9c 100644
--- a/intl/icu/source/common/rbbisetb.cpp
+++ b/intl/icu/source/common/rbbisetb.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// rbbisetb.cpp
@@ -35,7 +35,7 @@
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/uniset.h"
-#include "utrie.h"
+#include "utrie2.h"
#include "uvector.h"
#include "uassert.h"
#include "cmemory.h"
@@ -44,43 +44,6 @@
#include "rbbisetb.h"
#include "rbbinode.h"
-
-//------------------------------------------------------------------------
-//
-// getFoldedRBBIValue Call-back function used during building of Trie table.
-// Folding value: just store the offset (16 bits)
-// if there is any non-0 entry.
-// (It'd really be nice if the Trie builder would provide a
-// simple default, so this function could go away from here.)
-//
-//------------------------------------------------------------------------
-/* folding value: just store the offset (16 bits) if there is any non-0 entry */
-U_CDECL_BEGIN
-static uint32_t U_CALLCONV
-getFoldedRBBIValue(UNewTrie *trie, UChar32 start, int32_t offset) {
- uint32_t value;
- UChar32 limit;
- UBool inBlockZero;
-
- limit=start+0x400;
- while(startfDebugEnv && uprv_strstr(fRB->fDebugEnv, "rgroup")) {printRangeGroups();}
if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "esets")) {printSets();}
-
- //
- // Build the Trie table for mapping UChar32 values to the corresponding
- // range group number
- //
- fTrie = utrie_open(NULL, // Pre-existing trie to be filled in
- NULL, // Data array (utrie will allocate one)
- 100000, // Max Data Length
- 0, // Initial value for all code points
- 0, // Lead surrogate unit value
- TRUE); // Keep Latin 1 in separately
+}
- for (rlRange = fRangeList; rlRange!=0; rlRange=rlRange->fNext) {
- utrie_setRange32(fTrie, rlRange->fStartChar, rlRange->fEndChar+1, rlRange->fNum, TRUE);
+//
+// Build the Trie table for mapping UChar32 values to the corresponding
+// range group number.
+//
+void RBBISetBuilder::buildTrie() {
+ RangeDescriptor *rlRange;
+
+ fTrie = utrie2_open(0, // Initial value for all code points.
+ 0, // Error value for out-of-range input.
+ fStatus);
+
+ for (rlRange = fRangeList; rlRange!=0 && U_SUCCESS(*fStatus); rlRange=rlRange->fNext) {
+ utrie2_setRange32(fTrie,
+ rlRange->fStartChar, // Range start
+ rlRange->fEndChar, // Range end (inclusive)
+ rlRange->fNum, // value for range
+ TRUE, // Overwrite previously written values
+ fStatus);
}
}
+void RBBISetBuilder::mergeCategories(IntPair categories) {
+ U_ASSERT(categories.first >= 1);
+ U_ASSERT(categories.second > categories.first);
+ for (RangeDescriptor *rd = fRangeList; rd != nullptr; rd = rd->fNext) {
+ int32_t rangeNum = rd->fNum & ~DICT_BIT;
+ int32_t rangeDict = rd->fNum & DICT_BIT;
+ if (rangeNum == categories.second) {
+ rd->fNum = categories.first | rangeDict;
+ } else if (rangeNum > categories.second) {
+ rd->fNum--;
+ }
+ }
+ --fGroupCount;
+}
+
//-----------------------------------------------------------------------------------
//
// getTrieSize() Return the size that will be required to serialize the Trie.
//
//-----------------------------------------------------------------------------------
-int32_t RBBISetBuilder::getTrieSize() /*const*/ {
- fTrieSize = utrie_serialize(fTrie,
- NULL, // Buffer
- 0, // Capacity
- getFoldedRBBIValue,
- TRUE, // Reduce to 16 bits
- fStatus);
+int32_t RBBISetBuilder::getTrieSize() {
+ if (U_FAILURE(*fStatus)) {
+ return 0;
+ }
+ utrie2_freeze(fTrie, UTRIE2_16_VALUE_BITS, fStatus);
+ fTrieSize = utrie2_serialize(fTrie,
+ NULL, // Buffer
+ 0, // Capacity
+ fStatus);
+ if (*fStatus == U_BUFFER_OVERFLOW_ERROR) {
+ *fStatus = U_ZERO_ERROR;
+ }
// RBBIDebugPrintf("Trie table size is %d\n", trieSize);
return fTrieSize;
}
@@ -327,12 +316,10 @@ int32_t RBBISetBuilder::getTrieSize() /*const*/ {
//
//-----------------------------------------------------------------------------------
void RBBISetBuilder::serializeTrie(uint8_t *where) {
- utrie_serialize(fTrie,
- where, // Buffer
- fTrieSize, // Capacity
- getFoldedRBBIValue,
- TRUE, // Reduce to 16 bits
- fStatus);
+ utrie2_serialize(fTrie,
+ where, // Buffer
+ fTrieSize, // Capacity
+ fStatus);
}
//------------------------------------------------------------------------
@@ -480,7 +467,7 @@ void RBBISetBuilder::printRangeGroups() {
lastPrintedGroupNum = groupNum;
RBBIDebugPrintf("%2i ", groupNum);
- if (rlRange->fNum & 0x4000) { RBBIDebugPrintf(" ");}
+ if (rlRange->fNum & DICT_BIT) { RBBIDebugPrintf(" ");}
for (i=0; ifIncludesSets->size(); i++) {
RBBINode *usetNode = (RBBINode *)rlRange->fIncludesSets->elementAt(i);
@@ -673,20 +660,20 @@ void RangeDescriptor::split(UChar32 where, UErrorCode &status) {
void RangeDescriptor::setDictionaryFlag() {
int i;
- for (i=0; ifIncludesSets->size(); i++) {
- RBBINode *usetNode = (RBBINode *)fIncludesSets->elementAt(i);
- UnicodeString setName;
- RBBINode *setRef = usetNode->fParent;
- if (setRef != NULL) {
+ static const char16_t *dictionary = u"dictionary";
+ for (i=0; isize(); i++) {
+ RBBINode *usetNode = (RBBINode *)fIncludesSets->elementAt(i);
+ RBBINode *setRef = usetNode->fParent;
+ if (setRef != nullptr) {
RBBINode *varRef = setRef->fParent;
- if (varRef != NULL && varRef->fType == RBBINode::varRef) {
- setName = varRef->fText;
+ if (varRef && varRef->fType == RBBINode::varRef) {
+ const UnicodeString *setName = &varRef->fText;
+ if (setName->compare(dictionary, -1) == 0) {
+ fNum |= RBBISetBuilder::DICT_BIT;
+ break;
+ }
}
}
- if (setName.compare(UNICODE_STRING("dictionary", 10)) == 0) { // TODO: no string literals.
- this->fNum |= 0x4000;
- break;
- }
}
}
diff --git a/intl/icu/source/common/rbbisetb.h b/intl/icu/source/common/rbbisetb.h
index 89bfb9865c..ed6a76b121 100644
--- a/intl/icu/source/common/rbbisetb.h
+++ b/intl/icu/source/common/rbbisetb.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// rbbisetb.h
@@ -13,12 +13,14 @@
#define RBBISETB_H
#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
#include "unicode/uobject.h"
#include "rbbirb.h"
+#include "utrie2.h"
#include "uvector.h"
-struct UNewTrie;
-
U_NAMESPACE_BEGIN
//
@@ -80,7 +82,8 @@ public:
RBBISetBuilder(RBBIRuleBuilder *rb);
~RBBISetBuilder();
- void build();
+ void buildRanges();
+ void buildTrie();
void addValToSets(UVector *sets, uint32_t val);
void addValToSet (RBBINode *usetNode, uint32_t val);
int32_t getNumCharCategories() const; // CharCategories are the same as input symbol set to the
@@ -91,6 +94,15 @@ public:
UChar32 getFirstChar(int32_t val) const;
UBool sawBOF() const; // Indicate whether any references to the {bof} pseudo
// character were encountered.
+ /**
+ * Merge two character categories that have been identified as having equivalent behavior.
+ * The ranges belonging to the second category (table column) will be added to the first.
+ * @param categories the pair of categories to be merged.
+ */
+ void mergeCategories(IntPair categories);
+
+ static constexpr int32_t DICT_BIT = 0x4000;
+
#ifdef RBBI_DEBUG
void printSets();
void printRanges();
@@ -109,8 +121,8 @@ private:
RangeDescriptor *fRangeList; // Head of the linked list of RangeDescriptors
- UNewTrie *fTrie; // The mapping TRIE that is the end result of processing
- uint32_t fTrieSize; // the Unicode Sets.
+ UTrie2 *fTrie; // The mapping TRIE that is the end result of processing
+ uint32_t fTrieSize; // the Unicode Sets.
// Groups correspond to character categories -
// groups of ranges that are in the same original UnicodeSets.
@@ -129,4 +141,7 @@ private:
U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
+
#endif
diff --git a/intl/icu/source/common/rbbistbl.cpp b/intl/icu/source/common/rbbistbl.cpp
index d937edbd7c..5303f76096 100644
--- a/intl/icu/source/common/rbbistbl.cpp
+++ b/intl/icu/source/common/rbbistbl.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// file: rbbistbl.cpp Implementation of the ICU RBBISymbolTable class
diff --git a/intl/icu/source/common/rbbitblb.cpp b/intl/icu/source/common/rbbitblb.cpp
index 2738c75008..a20b51777c 100644
--- a/intl/icu/source/common/rbbitblb.cpp
+++ b/intl/icu/source/common/rbbitblb.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
@@ -22,25 +22,24 @@
#include "rbbidata.h"
#include "cstring.h"
#include "uassert.h"
+#include "uvectr32.h"
#include "cmemory.h"
U_NAMESPACE_BEGIN
-RBBITableBuilder::RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode) :
- fTree(*rootNode) {
- fRB = rb;
- fStatus = fRB->fStatus;
- UErrorCode status = U_ZERO_ERROR;
- fDStates = new UVector(status);
- if (U_FAILURE(*fStatus)) {
- return;
- }
+RBBITableBuilder::RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode, UErrorCode &status) :
+ fRB(rb),
+ fTree(*rootNode),
+ fStatus(&status),
+ fDStates(nullptr),
+ fSafeTable(nullptr) {
if (U_FAILURE(status)) {
- *fStatus = status;
return;
}
- if (fDStates == NULL) {
- *fStatus = U_MEMORY_ALLOCATION_ERROR;;
+ // fDStates is UVector
+ fDStates = new UVector(status);
+ if (U_SUCCESS(status) && fDStates == nullptr ) {
+ status = U_MEMORY_ALLOCATION_ERROR;
}
}
@@ -51,17 +50,18 @@ RBBITableBuilder::~RBBITableBuilder() {
for (i=0; isize(); i++) {
delete (RBBIStateDescriptor *)fDStates->elementAt(i);
}
- delete fDStates;
+ delete fDStates;
+ delete fSafeTable;
}
//-----------------------------------------------------------------------------
//
-// RBBITableBuilder::build - This is the main function for building the DFA state transtion
-// table from the RBBI rules parse tree.
+// RBBITableBuilder::buildForwardTable - This is the main function for building
+// the DFA state transition table from the RBBI rules parse tree.
//
//-----------------------------------------------------------------------------
-void RBBITableBuilder::build() {
+void RBBITableBuilder::buildForwardTable() {
if (U_FAILURE(*fStatus)) {
return;
@@ -188,8 +188,6 @@ void RBBITableBuilder::build() {
// for all tables. Merge the ones from this table into the global set.
//
mergeRuleStatusVals();
-
- if (fRB->fDebugEnv && uprv_strstr(fRB->fDebugEnv, "states")) {printStates();};
}
@@ -430,8 +428,8 @@ void RBBITableBuilder::calcChainedFollowPos(RBBINode *tree) {
addRuleRootNodes(&ruleRootNodes, tree);
UVector matchStartNodes(*fStatus);
- for (int i=0; i(ruleRootNodes.elementAt(i));
+ for (int j=0; j(ruleRootNodes.elementAt(j));
if (node->fChainIn) {
setAdd(&matchStartNodes, node->fFirstPosSet);
}
@@ -761,7 +759,7 @@ void RBBITableBuilder::flagAcceptingStates() {
// if sd->fAccepting already had a value other than 0 or -1, leave it be.
// If the end marker node is from a look-ahead rule, set
- // the fLookAhead field or this state also.
+ // the fLookAhead field for this state also.
if (endMarker->fLookAheadEnd) {
// TODO: don't change value if already set?
// TODO: allow for more than one active look-ahead rule in engine.
@@ -1077,6 +1075,188 @@ void RBBITableBuilder::printPosSets(RBBINode *n) {
}
#endif
+//
+// findDuplCharClassFrom()
+//
+bool RBBITableBuilder::findDuplCharClassFrom(IntPair *categories) {
+ int32_t numStates = fDStates->size();
+ int32_t numCols = fRB->fSetBuilder->getNumCharCategories();
+
+ for (; categories->first < numCols-1; categories->first++) {
+ for (categories->second=categories->first+1; categories->second < numCols; categories->second++) {
+ // Initialized to different values to prevent returning true if numStates = 0 (implies no duplicates).
+ uint16_t table_base = 0;
+ uint16_t table_dupl = 1;
+ for (int32_t state=0; stateelementAt(state);
+ table_base = (uint16_t)sd->fDtran->elementAti(categories->first);
+ table_dupl = (uint16_t)sd->fDtran->elementAti(categories->second);
+ if (table_base != table_dupl) {
+ break;
+ }
+ }
+ if (table_base == table_dupl) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+
+//
+// removeColumn()
+//
+void RBBITableBuilder::removeColumn(int32_t column) {
+ int32_t numStates = fDStates->size();
+ for (int32_t state=0; stateelementAt(state);
+ U_ASSERT(column < sd->fDtran->size());
+ sd->fDtran->removeElementAt(column);
+ }
+}
+
+/*
+ * findDuplicateState
+ */
+bool RBBITableBuilder::findDuplicateState(IntPair *states) {
+ int32_t numStates = fDStates->size();
+ int32_t numCols = fRB->fSetBuilder->getNumCharCategories();
+
+ for (; states->firstfirst++) {
+ RBBIStateDescriptor *firstSD = (RBBIStateDescriptor *)fDStates->elementAt(states->first);
+ for (states->second=states->first+1; states->secondsecond++) {
+ RBBIStateDescriptor *duplSD = (RBBIStateDescriptor *)fDStates->elementAt(states->second);
+ if (firstSD->fAccepting != duplSD->fAccepting ||
+ firstSD->fLookAhead != duplSD->fLookAhead ||
+ firstSD->fTagsIdx != duplSD->fTagsIdx) {
+ continue;
+ }
+ bool rowsMatch = true;
+ for (int32_t col=0; col < numCols; ++col) {
+ int32_t firstVal = firstSD->fDtran->elementAti(col);
+ int32_t duplVal = duplSD->fDtran->elementAti(col);
+ if (!((firstVal == duplVal) ||
+ ((firstVal == states->first || firstVal == states->second) &&
+ (duplVal == states->first || duplVal == states->second)))) {
+ rowsMatch = false;
+ break;
+ }
+ }
+ if (rowsMatch) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+
+bool RBBITableBuilder::findDuplicateSafeState(IntPair *states) {
+ int32_t numStates = fSafeTable->size();
+
+ for (; states->firstfirst++) {
+ UnicodeString *firstRow = static_cast(fSafeTable->elementAt(states->first));
+ for (states->second=states->first+1; states->secondsecond++) {
+ UnicodeString *duplRow = static_cast(fSafeTable->elementAt(states->second));
+ bool rowsMatch = true;
+ int32_t numCols = firstRow->length();
+ for (int32_t col=0; col < numCols; ++col) {
+ int32_t firstVal = firstRow->charAt(col);
+ int32_t duplVal = duplRow->charAt(col);
+ if (!((firstVal == duplVal) ||
+ ((firstVal == states->first || firstVal == states->second) &&
+ (duplVal == states->first || duplVal == states->second)))) {
+ rowsMatch = false;
+ break;
+ }
+ }
+ if (rowsMatch) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+
+void RBBITableBuilder::removeState(IntPair duplStates) {
+ const int32_t keepState = duplStates.first;
+ const int32_t duplState = duplStates.second;
+ U_ASSERT(keepState < duplState);
+ U_ASSERT(duplState < fDStates->size());
+
+ RBBIStateDescriptor *duplSD = (RBBIStateDescriptor *)fDStates->elementAt(duplState);
+ fDStates->removeElementAt(duplState);
+ delete duplSD;
+
+ int32_t numStates = fDStates->size();
+ int32_t numCols = fRB->fSetBuilder->getNumCharCategories();
+ for (int32_t state=0; stateelementAt(state);
+ for (int32_t col=0; colfDtran->elementAti(col);
+ int32_t newVal = existingVal;
+ if (existingVal == duplState) {
+ newVal = keepState;
+ } else if (existingVal > duplState) {
+ newVal = existingVal - 1;
+ }
+ sd->fDtran->setElementAt(newVal, col);
+ }
+ if (sd->fAccepting == duplState) {
+ sd->fAccepting = keepState;
+ } else if (sd->fAccepting > duplState) {
+ sd->fAccepting--;
+ }
+ if (sd->fLookAhead == duplState) {
+ sd->fLookAhead = keepState;
+ } else if (sd->fLookAhead > duplState) {
+ sd->fLookAhead--;
+ }
+ }
+}
+
+void RBBITableBuilder::removeSafeState(IntPair duplStates) {
+ const int32_t keepState = duplStates.first;
+ const int32_t duplState = duplStates.second;
+ U_ASSERT(keepState < duplState);
+ U_ASSERT(duplState < fSafeTable->size());
+
+ fSafeTable->removeElementAt(duplState); // Note that fSafeTable has a deleter function
+ // and will auto-delete the removed element.
+ int32_t numStates = fSafeTable->size();
+ for (int32_t state=0; stateelementAt(state);
+ int32_t numCols = sd->length();
+ for (int32_t col=0; colcharAt(col);
+ int32_t newVal = existingVal;
+ if (existingVal == duplState) {
+ newVal = keepState;
+ } else if (existingVal > duplState) {
+ newVal = existingVal - 1;
+ }
+ sd->setCharAt(col, static_cast(newVal));
+ }
+ }
+}
+
+
+/*
+ * RemoveDuplicateStates
+ */
+int32_t RBBITableBuilder::removeDuplicateStates() {
+ IntPair dupls = {3, 0};
+ int32_t numStatesRemoved = 0;
+
+ while (findDuplicateState(&dupls)) {
+ // printf("Removing duplicate states (%d, %d)\n", dupls.first, dupls.second);
+ removeState(dupls);
+ ++numStatesRemoved;
+ }
+ return numStatesRemoved;
+}
//-----------------------------------------------------------------------------
@@ -1095,21 +1275,17 @@ int32_t RBBITableBuilder::getTableSize() const {
return 0;
}
- size = sizeof(RBBIStateTable) - 4; // The header, with no rows to the table.
+ size = offsetof(RBBIStateTable, fTableData); // The header, with no rows to the table.
numRows = fDStates->size();
numCols = fRB->fSetBuilder->getNumCharCategories();
- // Note The declaration of RBBIStateTableRow is for a table of two columns.
- // Therefore we subtract two from numCols when determining
- // how much storage to add to a row for the total columns.
- rowSize = sizeof(RBBIStateTableRow) + sizeof(uint16_t)*(numCols-2);
+ rowSize = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t)*numCols;
size += numRows * rowSize;
return size;
}
-
//-----------------------------------------------------------------------------
//
// exportTable() export the state transition table in the format required
@@ -1126,14 +1302,14 @@ void RBBITableBuilder::exportTable(void *where) {
return;
}
- if (fRB->fSetBuilder->getNumCharCategories() > 0x7fff ||
+ int32_t catCount = fRB->fSetBuilder->getNumCharCategories();
+ if (catCount > 0x7fff ||
fDStates->size() > 0x7fff) {
*fStatus = U_BRK_INTERNAL_ERROR;
return;
}
- table->fRowLen = sizeof(RBBIStateTableRow) +
- sizeof(uint16_t) * (fRB->fSetBuilder->getNumCharCategories() - 2);
+ table->fRowLen = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t) * catCount;
table->fNumStates = fDStates->size();
table->fFlags = 0;
if (fRB->fLookAheadHardBreak) {
@@ -1152,13 +1328,192 @@ void RBBITableBuilder::exportTable(void *where) {
row->fAccepting = (int16_t)sd->fAccepting;
row->fLookAhead = (int16_t)sd->fLookAhead;
row->fTagIdx = (int16_t)sd->fTagsIdx;
- for (col=0; colfSetBuilder->getNumCharCategories(); col++) {
+ for (col=0; colfNextState[col] = (uint16_t)sd->fDtran->elementAti(col);
}
}
}
+/**
+ * Synthesize a safe state table from the main state table.
+ */
+void RBBITableBuilder::buildSafeReverseTable(UErrorCode &status) {
+ // The safe table creation has three steps:
+
+ // 1. Identifiy pairs of character classes that are "safe." Safe means that boundaries
+ // following the pair do not depend on context or state before the pair. To test
+ // whether a pair is safe, run it through the main forward state table, starting
+ // from each state. If the the final state is the same, no matter what the starting state,
+ // the pair is safe.
+ //
+ // 2. Build a state table that recognizes the safe pairs. It's similar to their
+ // forward table, with a column for each input character [class], and a row for
+ // each state. Row 1 is the start state, and row 0 is the stop state. Initially
+ // create an additional state for each input character category; being in
+ // one of these states means that the character has been seen, and is potentially
+ // the first of a pair. In each of these rows, the entry for the second character
+ // of a safe pair is set to the stop state (0), indicating that a match was found.
+ // All other table entries are set to the state corresponding the current input
+ // character, allowing that charcter to be the of a start following pair.
+ //
+ // Because the safe rules are to be run in reverse, moving backwards in the text,
+ // the first and second pair categories are swapped when building the table.
+ //
+ // 3. Compress the table. There are typically many rows (states) that are
+ // equivalent - that have zeroes (match completed) in the same columns -
+ // and can be folded together.
+
+ // Each safe pair is stored as two UChars in the safePair string.
+ UnicodeString safePairs;
+
+ int32_t numCharClasses = fRB->fSetBuilder->getNumCharCategories();
+ int32_t numStates = fDStates->size();
+
+ for (int32_t c1=0; c1(fDStates->elementAt(startState));
+ int32_t s2 = startStateD->fDtran->elementAti(c1);
+ RBBIStateDescriptor *s2StateD = static_cast(fDStates->elementAt(s2));
+ endState = s2StateD->fDtran->elementAti(c2);
+ if (wantedEndState < 0) {
+ wantedEndState = endState;
+ } else {
+ if (wantedEndState != endState) {
+ break;
+ }
+ }
+ }
+ if (wantedEndState == endState) {
+ safePairs.append((char16_t)c1);
+ safePairs.append((char16_t)c2);
+ // printf("(%d, %d) ", c1, c2);
+ }
+ }
+ // printf("\n");
+ }
+
+ // Populate the initial safe table.
+ // The table as a whole is UVector
+ // Each row is represented by a UnicodeString, being used as a Vector.
+ // Row 0 is the stop state.
+ // Row 1 is the start sate.
+ // Row 2 and beyond are other states, initially one per char class, but
+ // after initial construction, many of the states will be combined, compacting the table.
+ // The String holds the nextState data only. The four leading fields of a row, fAccepting,
+ // fLookAhead, etc. are not needed for the safe table, and are omitted at this stage of building.
+
+ U_ASSERT(fSafeTable == nullptr);
+ fSafeTable = new UVector(uprv_deleteUObject, uhash_compareUnicodeString, numCharClasses + 2, status);
+ for (int32_t row=0; rowaddElement(new UnicodeString(numCharClasses, 0, numCharClasses+4), status);
+ }
+
+ // From the start state, each input char class transitions to the state for that input.
+ UnicodeString &startState = *static_cast(fSafeTable->elementAt(1));
+ for (int32_t charClass=0; charClass < numCharClasses; ++charClass) {
+ // Note: +2 for the start & stop state.
+ startState.setCharAt(charClass, static_cast(charClass+2));
+ }
+
+ // Initially make every other state table row look like the start state row,
+ for (int32_t row=2; row(fSafeTable->elementAt(row));
+ rowState = startState; // UnicodeString assignment, copies contents.
+ }
+
+ // Run through the safe pairs, set the next state to zero when pair has been seen.
+ // Zero being the stop state, meaning we found a safe point.
+ for (int32_t pairIdx=0; pairIdx(fSafeTable->elementAt(c2 + 2));
+ rowState.setCharAt(c1, 0);
+ }
+
+ // Remove duplicate or redundant rows from the table.
+ IntPair states = {1, 0};
+ while (findDuplicateSafeState(&states)) {
+ // printf("Removing duplicate safe states (%d, %d)\n", states.first, states.second);
+ removeSafeState(states);
+ }
+}
+
+
+//-----------------------------------------------------------------------------
+//
+// getSafeTableSize() Calculate the size of the runtime form of this
+// safe state table.
+//
+//-----------------------------------------------------------------------------
+int32_t RBBITableBuilder::getSafeTableSize() const {
+ int32_t size = 0;
+ int32_t numRows;
+ int32_t numCols;
+ int32_t rowSize;
+
+ if (fSafeTable == nullptr) {
+ return 0;
+ }
+
+ size = offsetof(RBBIStateTable, fTableData); // The header, with no rows to the table.
+
+ numRows = fSafeTable->size();
+ numCols = fRB->fSetBuilder->getNumCharCategories();
+
+ rowSize = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t)*numCols;
+ size += numRows * rowSize;
+ return size;
+}
+
+
+//-----------------------------------------------------------------------------
+//
+// exportSafeTable() export the state transition table in the format required
+// by the runtime engine. getTableSize() bytes of memory
+// must be available at the output address "where".
+//
+//-----------------------------------------------------------------------------
+void RBBITableBuilder::exportSafeTable(void *where) {
+ RBBIStateTable *table = (RBBIStateTable *)where;
+ uint32_t state;
+ int col;
+
+ if (U_FAILURE(*fStatus) || fSafeTable == nullptr) {
+ return;
+ }
+
+ int32_t catCount = fRB->fSetBuilder->getNumCharCategories();
+ if (catCount > 0x7fff ||
+ fSafeTable->size() > 0x7fff) {
+ *fStatus = U_BRK_INTERNAL_ERROR;
+ return;
+ }
+
+ table->fRowLen = offsetof(RBBIStateTableRow, fNextState) + sizeof(uint16_t) * catCount;
+ table->fNumStates = fSafeTable->size();
+ table->fFlags = 0;
+ table->fReserved = 0;
+
+ for (state=0; statefNumStates; state++) {
+ UnicodeString *rowString = (UnicodeString *)fSafeTable->elementAt(state);
+ RBBIStateTableRow *row = (RBBIStateTableRow *)(table->fTableData + state*table->fRowLen);
+ row->fAccepting = 0;
+ row->fLookAhead = 0;
+ row->fTagIdx = 0;
+ row->fReserved = 0;
+ for (col=0; colfNextState[col] = rowString->charAt(col);
+ }
+ }
+}
+
+
+
//-----------------------------------------------------------------------------
//
@@ -1213,6 +1568,47 @@ void RBBITableBuilder::printStates() {
#endif
+//-----------------------------------------------------------------------------
+//
+// printSafeTable Debug Function. Dump the fully constructed safe table.
+//
+//-----------------------------------------------------------------------------
+#ifdef RBBI_DEBUG
+void RBBITableBuilder::printReverseTable() {
+ int c; // input "character"
+ int n; // state number
+
+ RBBIDebugPrintf(" Safe Reverse Table \n");
+ if (fSafeTable == nullptr) {
+ RBBIDebugPrintf(" --- nullptr ---\n");
+ return;
+ }
+ RBBIDebugPrintf("state | i n p u t s y m b o l s \n");
+ RBBIDebugPrintf(" | Acc LA Tag");
+ for (c=0; cfSetBuilder->getNumCharCategories(); c++) {
+ RBBIDebugPrintf(" %2d", c);
+ }
+ RBBIDebugPrintf("\n");
+ RBBIDebugPrintf(" |---------------");
+ for (c=0; cfSetBuilder->getNumCharCategories(); c++) {
+ RBBIDebugPrintf("---");
+ }
+ RBBIDebugPrintf("\n");
+
+ for (n=0; nsize(); n++) {
+ UnicodeString *rowString = (UnicodeString *)fSafeTable->elementAt(n);
+ RBBIDebugPrintf(" %3d | " , n);
+ RBBIDebugPrintf("%3d %3d %5d ", 0, 0, 0); // Accepting, LookAhead, Tags
+ for (c=0; cfSetBuilder->getNumCharCategories(); c++) {
+ RBBIDebugPrintf(" %2d", rowString->charAt(c));
+ }
+ RBBIDebugPrintf("\n");
+ }
+ RBBIDebugPrintf("\n\n");
+}
+#endif
+
+
//-----------------------------------------------------------------------------
//
@@ -1259,7 +1655,7 @@ RBBIStateDescriptor::RBBIStateDescriptor(int lastInputSymbol, UErrorCode *fStatu
fPositions = NULL;
fDtran = NULL;
- fDtran = new UVector(lastInputSymbol+1, *fStatus);
+ fDtran = new UVector32(lastInputSymbol+1, *fStatus);
if (U_FAILURE(*fStatus)) {
return;
}
@@ -1267,7 +1663,7 @@ RBBIStateDescriptor::RBBIStateDescriptor(int lastInputSymbol, UErrorCode *fStatu
*fStatus = U_MEMORY_ALLOCATION_ERROR;
return;
}
- fDtran->setSize(lastInputSymbol+1, *fStatus); // fDtran needs to be pre-sized.
+ fDtran->setSize(lastInputSymbol+1); // fDtran needs to be pre-sized.
// It is indexed by input symbols, and will
// hold the next state number for each
// symbol.
diff --git a/intl/icu/source/common/rbbitblb.h b/intl/icu/source/common/rbbitblb.h
index d71a024587..bc6077bb4d 100644
--- a/intl/icu/source/common/rbbitblb.h
+++ b/intl/icu/source/common/rbbitblb.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// rbbitblb.h
@@ -15,8 +15,12 @@
#define RBBITBLB_H
#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
#include "unicode/uobject.h"
#include "unicode/rbbi.h"
+#include "rbbirb.h"
#include "rbbinode.h"
@@ -24,6 +28,7 @@ U_NAMESPACE_BEGIN
class RBBIRuleScanner;
class RBBIRuleBuilder;
+class UVector32;
//
// class RBBITableBuilder is part of the RBBI rule compiler.
@@ -36,15 +41,49 @@ class RBBIRuleBuilder;
class RBBITableBuilder : public UMemory {
public:
- RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode);
+ RBBITableBuilder(RBBIRuleBuilder *rb, RBBINode **rootNode, UErrorCode &status);
~RBBITableBuilder();
- void build();
- int32_t getTableSize() const; // Return the runtime size in bytes of
- // the built state table
- void exportTable(void *where); // fill in the runtime state table.
- // Sufficient memory must exist at
- // the specified location.
+ void buildForwardTable();
+
+ /** Return the runtime size in bytes of the built state table. */
+ int32_t getTableSize() const;
+
+ /** Fill in the runtime state table. Sufficient memory must exist at the specified location.
+ */
+ void exportTable(void *where);
+
+ /**
+ * Find duplicate (redundant) character classes. Begin looking with categories.first.
+ * Duplicate, if found are returned in the categories parameter.
+ * This is an iterator-like function, used to identify character classes
+ * (state table columns) that can be eliminated.
+ * @param categories in/out parameter, specifies where to start looking for duplicates,
+ * and returns the first pair of duplicates found, if any.
+ * @return true if duplicate char classes were found, false otherwise.
+ */
+ bool findDuplCharClassFrom(IntPair *categories);
+
+ /** Remove a column from the state table. Used when two character categories
+ * have been found equivalent, and merged together, to eliminate the uneeded table column.
+ */
+ void removeColumn(int32_t column);
+
+ /**
+ * Check for, and remove dupicate states (table rows).
+ * @return the number of states removed.
+ */
+ int32_t removeDuplicateStates();
+
+ /** Build the safe reverse table from the already-constructed forward table. */
+ void buildSafeReverseTable(UErrorCode &status);
+
+ /** Return the runtime size in bytes of the built safe reverse state table. */
+ int32_t getSafeTableSize() const;
+
+ /** Fill in the runtime safe state table. Sufficient memory must exist at the specified location.
+ */
+ void exportSafeTable(void *where);
private:
@@ -60,8 +99,45 @@ private:
void flagTaggedStates();
void mergeRuleStatusVals();
+ /**
+ * Merge redundant state table columns, eliminating character classes with identical behavior.
+ * Done after the state tables are generated, just before converting to their run-time format.
+ */
+ int32_t mergeColumns();
+
void addRuleRootNodes(UVector *dest, RBBINode *node);
+ /**
+ * Find duplicate (redundant) states, beginning at the specified pair,
+ * within this state table. This is an iterator-like function, used to
+ * identify states (state table rows) that can be eliminated.
+ * @param states in/out parameter, specifies where to start looking for duplicates,
+ * and returns the first pair of duplicates found, if any.
+ * @return true if duplicate states were found, false otherwise.
+ */
+ bool findDuplicateState(IntPair *states);
+
+ /** Remove a duplicate state.
+ * @param duplStates The duplicate states. The first is kept, the second is removed.
+ * All references to the second in the state table are retargeted
+ * to the first.
+ */
+ void removeState(IntPair duplStates);
+
+ /** Find the next duplicate state in the safe reverse table. An iterator function.
+ * @param states in/out parameter, specifies where to start looking for duplicates,
+ * and returns the first pair of duplicates found, if any.
+ * @return true if a duplicate pair of states was found.
+ */
+ bool findDuplicateSafeState(IntPair *states);
+
+ /** Remove a duplicate state from the safe table.
+ * @param duplStates The duplicate states. The first is kept, the second is removed.
+ * All references to the second in the state table are retargeted
+ * to the first.
+ */
+ void removeSafeState(IntPair duplStates);
+
// Set functions for UVector.
// TODO: make a USet subclass of UVector
@@ -76,11 +152,13 @@ public:
void printPosSets(RBBINode *n /* = NULL*/);
void printStates();
void printRuleStatusTable();
+ void printReverseTable();
#else
#define printSet(s)
#define printPosSets(n)
#define printStates()
#define printRuleStatusTable()
+ #define printReverseTable()
#endif
private:
@@ -89,10 +167,14 @@ private:
// table for.
UErrorCode *fStatus;
+ /** State Descriptors, UVector */
UVector *fDStates; // D states (Aho's terminology)
// Index is state number
// Contents are RBBIStateDescriptor pointers.
+ /** Synthesized safe table, UVector of UnicodeString, one string per table row. */
+ UVector *fSafeTable;
+
RBBITableBuilder(const RBBITableBuilder &other); // forbid copying of this class
RBBITableBuilder &operator=(const RBBITableBuilder &other); // forbid copying of this class
@@ -112,7 +194,7 @@ public:
// with this state. Unordered (it's a set).
// UVector contents are RBBINode *
- UVector *fDtran; // Transitions out of this state.
+ UVector32 *fDtran; // Transitions out of this state.
// indexed by input character
// contents is int index of dest state
// in RBBITableBuilder.fDStates
@@ -128,4 +210,7 @@ private:
U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
+
#endif
diff --git a/intl/icu/source/common/resbund.cpp b/intl/icu/source/common/resbund.cpp
index 7bd6b32b08..512bd531f2 100644
--- a/intl/icu/source/common/resbund.cpp
+++ b/intl/icu/source/common/resbund.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
diff --git a/intl/icu/source/common/resbund_cnv.cpp b/intl/icu/source/common/resbund_cnv.cpp
index 65c32b5207..45c0b399bf 100644
--- a/intl/icu/source/common/resbund_cnv.cpp
+++ b/intl/icu/source/common/resbund_cnv.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: resbund_cnv.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/resource.cpp b/intl/icu/source/common/resource.cpp
index 62b3aa46a5..3d41a16029 100644
--- a/intl/icu/source/common/resource.cpp
+++ b/intl/icu/source/common/resource.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
diff --git a/intl/icu/source/common/resource.h b/intl/icu/source/common/resource.h
index 43c3309b5e..3dbff785ef 100644
--- a/intl/icu/source/common/resource.h
+++ b/intl/icu/source/common/resource.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
diff --git a/intl/icu/source/common/ruleiter.cpp b/intl/icu/source/common/ruleiter.cpp
index 6e27b4dd8c..41eea23c0d 100644
--- a/intl/icu/source/common/ruleiter.cpp
+++ b/intl/icu/source/common/ruleiter.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
diff --git a/intl/icu/source/common/ruleiter.h b/intl/icu/source/common/ruleiter.h
index 1221396fe4..4e1be53823 100644
--- a/intl/icu/source/common/ruleiter.h
+++ b/intl/icu/source/common/ruleiter.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
diff --git a/intl/icu/source/common/schriter.cpp b/intl/icu/source/common/schriter.cpp
index cc413666f1..f852800aaa 100644
--- a/intl/icu/source/common/schriter.cpp
+++ b/intl/icu/source/common/schriter.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
diff --git a/intl/icu/source/common/serv.cpp b/intl/icu/source/common/serv.cpp
index c650ef3a4e..619e8c7c8b 100644
--- a/intl/icu/source/common/serv.cpp
+++ b/intl/icu/source/common/serv.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
@@ -547,16 +547,15 @@ outerEnd:
if (putInCache && cacheResult) {
serviceCache->put(result->actualDescriptor, result, status);
if (U_FAILURE(status)) {
- delete result;
return NULL;
}
if (cacheDescriptorList._obj != NULL) {
for (int32_t i = cacheDescriptorList._obj->size(); --i >= 0;) {
UnicodeString* desc = (UnicodeString*)cacheDescriptorList._obj->elementAt(i);
+
serviceCache->put(*desc, result, status);
if (U_FAILURE(status)) {
- delete result;
return NULL;
}
@@ -703,9 +702,9 @@ ICUService::getDisplayName(const UnicodeString& id, UnicodeString& result, const
}
// fallback
- UErrorCode status = U_ZERO_ERROR;
+ status = U_ZERO_ERROR;
ICUServiceKey* fallbackKey = createKey(&id, status);
- while (fallbackKey->fallback()) {
+ while (fallbackKey != NULL && fallbackKey->fallback()) {
UnicodeString us;
fallbackKey->currentID(us);
f = (ICUServiceFactory*)map->get(us);
diff --git a/intl/icu/source/common/serv.h b/intl/icu/source/common/serv.h
index 679c4435a1..e1f69cd411 100644
--- a/intl/icu/source/common/serv.h
+++ b/intl/icu/source/common/serv.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servlk.cpp b/intl/icu/source/common/servlk.cpp
index 319d7b87fa..538982ca36 100644
--- a/intl/icu/source/common/servlk.cpp
+++ b/intl/icu/source/common/servlk.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servlkf.cpp b/intl/icu/source/common/servlkf.cpp
index 114d64389b..84f2347cdd 100644
--- a/intl/icu/source/common/servlkf.cpp
+++ b/intl/icu/source/common/servlkf.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servloc.h b/intl/icu/source/common/servloc.h
index 7faec51079..ccf6433379 100644
--- a/intl/icu/source/common/servloc.h
+++ b/intl/icu/source/common/servloc.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servls.cpp b/intl/icu/source/common/servls.cpp
index 71c6c44a8f..f4579d0eec 100644
--- a/intl/icu/source/common/servls.cpp
+++ b/intl/icu/source/common/servls.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servnotf.cpp b/intl/icu/source/common/servnotf.cpp
index 69a81da252..dc77c7b857 100644
--- a/intl/icu/source/common/servnotf.cpp
+++ b/intl/icu/source/common/servnotf.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servnotf.h b/intl/icu/source/common/servnotf.h
index 05606ff6b3..dba7a0fea3 100644
--- a/intl/icu/source/common/servnotf.h
+++ b/intl/icu/source/common/servnotf.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servrbf.cpp b/intl/icu/source/common/servrbf.cpp
index 299695e2af..94279ab3a1 100644
--- a/intl/icu/source/common/servrbf.cpp
+++ b/intl/icu/source/common/servrbf.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/servslkf.cpp b/intl/icu/source/common/servslkf.cpp
index 4cf66b9d20..09154d1b91 100644
--- a/intl/icu/source/common/servslkf.cpp
+++ b/intl/icu/source/common/servslkf.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
*******************************************************************************
diff --git a/intl/icu/source/common/sharedobject.cpp b/intl/icu/source/common/sharedobject.cpp
index 8e5095e129..6eeca8605f 100644
--- a/intl/icu/source/common/sharedobject.cpp
+++ b/intl/icu/source/common/sharedobject.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -8,7 +8,10 @@
* sharedobject.cpp
*/
#include "sharedobject.h"
+#include "mutex.h"
#include "uassert.h"
+#include "umutex.h"
+#include "unifiedcache.h"
U_NAMESPACE_BEGIN
@@ -17,69 +20,41 @@ SharedObject::~SharedObject() {}
UnifiedCacheBase::~UnifiedCacheBase() {}
void
-SharedObject::addRef(UBool fromWithinCache) const {
- umtx_atomic_inc(&totalRefCount);
-
- // Although items in use may not be correct immediately, it
- // will be correct eventually.
- if (umtx_atomic_inc(&hardRefCount) == 1 && cachePtr != NULL) {
- // If this object is cached, and the hardRefCount goes from 0 to 1,
- // then the increment must happen from within the cache while the
- // cache global mutex is locked. In this way, we can be rest assured
- // that data races can't happen if the cache performs some task if
- // the hardRefCount is zero while the global cache mutex is locked.
- (void)fromWithinCache; // Suppress unused variable warning in non-debug builds.
- U_ASSERT(fromWithinCache);
- cachePtr->incrementItemsInUse();
- }
+SharedObject::addRef() const {
+ umtx_atomic_inc(&hardRefCount);
}
+// removeRef Decrement the reference count and delete if it is zero.
+// Note that SharedObjects with a non-null cachePtr are owned by the
+// unified cache, and the cache will be responsible for the actual deletion.
+// The deletion could be as soon as immediately following the
+// update to the reference count, if another thread is running
+// a cache eviction cycle concurrently.
+// NO ACCESS TO *this PERMITTED AFTER REFERENCE COUNT == 0 for cached objects.
+// THE OBJECT MAY ALREADY BE GONE.
void
-SharedObject::removeRef(UBool fromWithinCache) const {
- UBool decrementItemsInUse = (umtx_atomic_dec(&hardRefCount) == 0);
- UBool allReferencesGone = (umtx_atomic_dec(&totalRefCount) == 0);
-
- // Although items in use may not be correct immediately, it
- // will be correct eventually.
- if (decrementItemsInUse && cachePtr != NULL) {
- if (fromWithinCache) {
- cachePtr->decrementItemsInUse();
+SharedObject::removeRef() const {
+ const UnifiedCacheBase *cache = this->cachePtr;
+ int32_t updatedRefCount = umtx_atomic_dec(&hardRefCount);
+ U_ASSERT(updatedRefCount >= 0);
+ if (updatedRefCount == 0) {
+ if (cache) {
+ cache->handleUnreferencedObject();
} else {
- cachePtr->decrementItemsInUseWithLockingAndEviction();
+ delete this;
}
}
- if (allReferencesGone) {
- delete this;
- }
}
-void
-SharedObject::addSoftRef() const {
- umtx_atomic_inc(&totalRefCount);
- ++softRefCount;
-}
-
-void
-SharedObject::removeSoftRef() const {
- --softRefCount;
- if (umtx_atomic_dec(&totalRefCount) == 0) {
- delete this;
- }
-}
int32_t
SharedObject::getRefCount() const {
- return umtx_loadAcquire(totalRefCount);
-}
-
-int32_t
-SharedObject::getHardRefCount() const {
return umtx_loadAcquire(hardRefCount);
}
void
SharedObject::deleteIfZeroRefCount() const {
- if(getRefCount() == 0) {
+ if (this->cachePtr == nullptr && getRefCount() == 0) {
delete this;
}
}
diff --git a/intl/icu/source/common/sharedobject.h b/intl/icu/source/common/sharedobject.h
index 0e53cfb7ab..c0a5aba478 100644
--- a/intl/icu/source/common/sharedobject.h
+++ b/intl/icu/source/common/sharedobject.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -17,6 +17,8 @@
U_NAMESPACE_BEGIN
+class SharedObject;
+
/**
* Base class for unified cache exposing enough methods to SharedObject
* instances to allow their addRef() and removeRef() methods to
@@ -28,22 +30,12 @@ public:
UnifiedCacheBase() { }
/**
- * Called by addRefWhileHoldingCacheLock() when the hard reference count
- * of its instance goes from 0 to 1.
+ * Notify the cache implementation that an object was seen transitioning to
+ * zero hard references. The cache may use this to keep track the number of
+ * unreferenced SharedObjects, and to trigger evictions.
*/
- virtual void incrementItemsInUse() const = 0;
+ virtual void handleUnreferencedObject() const = 0;
- /**
- * Called by removeRef() when the hard reference count of its instance
- * drops from 1 to 0.
- */
- virtual void decrementItemsInUseWithLockingAndEviction() const = 0;
-
- /**
- * Called by removeRefWhileHoldingCacheLock() when the hard reference
- * count of its instance drops from 1 to 0.
- */
- virtual void decrementItemsInUse() const = 0;
virtual ~UnifiedCacheBase();
private:
UnifiedCacheBase(const UnifiedCacheBase &);
@@ -63,7 +55,6 @@ class U_COMMON_API SharedObject : public UObject {
public:
/** Initializes totalRefCount, softRefCount to 0. */
SharedObject() :
- totalRefCount(0),
softRefCount(0),
hardRefCount(0),
cachePtr(NULL) {}
@@ -71,7 +62,6 @@ public:
/** Initializes totalRefCount, softRefCount to 0. */
SharedObject(const SharedObject &other) :
UObject(other),
- totalRefCount(0),
softRefCount(0),
hardRefCount(0),
cachePtr(NULL) {}
@@ -79,93 +69,45 @@ public:
virtual ~SharedObject();
/**
- * Increments the number of references to this object. Thread-safe.
+ * Increments the number of hard references to this object. Thread-safe.
+ * Not for use from within the Unified Cache implementation.
*/
- void addRef() const { addRef(FALSE); }
+ void addRef() const;
/**
- * Increments the number of references to this object.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
+ * Decrements the number of hard references to this object, and
+ * arrange for possible cache-eviction and/or deletion if ref
+ * count goes to zero. Thread-safe.
+ *
+ * Not for use from within the UnifiedCache implementation.
*/
- void addRefWhileHoldingCacheLock() const { addRef(TRUE); }
+ void removeRef() const;
/**
- * Increments the number of soft references to this object.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
- */
- void addSoftRef() const;
-
- /**
- * Decrements the number of references to this object. Thread-safe.
- */
- void removeRef() const { removeRef(FALSE); }
-
- /**
- * Decrements the number of references to this object.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
- */
- void removeRefWhileHoldingCacheLock() const { removeRef(TRUE); }
-
- /**
- * Decrements the number of soft references to this object.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
- */
- void removeSoftRef() const;
-
- /**
- * Returns the reference counter including soft references.
+ * Returns the number of hard references for this object.
* Uses a memory barrier.
*/
int32_t getRefCount() const;
- /**
- * Returns the count of soft references only.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
- */
- int32_t getSoftRefCount() const { return softRefCount; }
-
- /**
- * Returns the count of hard references only. Uses a memory barrier.
- * Used for testing the cache. Regular clients won't need this.
- */
- int32_t getHardRefCount() const;
-
/**
* If noHardReferences() == TRUE then this object has no hard references.
* Must be called only from within the internals of UnifiedCache.
*/
- inline UBool noHardReferences() const { return getHardRefCount() == 0; }
+ inline UBool noHardReferences() const { return getRefCount() == 0; }
/**
* If hasHardReferences() == TRUE then this object has hard references.
* Must be called only from within the internals of UnifiedCache.
*/
- inline UBool hasHardReferences() const { return getHardRefCount() != 0; }
+ inline UBool hasHardReferences() const { return getRefCount() != 0; }
/**
- * If noSoftReferences() == TRUE then this object has no soft references.
- * Must be called only from within the internals of UnifiedCache and
- * only while the cache global mutex is held.
- */
- UBool noSoftReferences() const { return (softRefCount == 0); }
-
- /**
- * Deletes this object if it has no references or soft references.
+ * Deletes this object if it has no references.
+ * Available for non-cached SharedObjects only. Ownership of cached objects
+ * is with the UnifiedCache, which is solely responsible for eviction and deletion.
*/
void deleteIfZeroRefCount() const;
- /**
- * @internal For UnifedCache use only to register this object with itself.
- * Must be called before this object is exposed to multiple threads.
- */
- void registerWithCache(const UnifiedCacheBase *ptr) const {
- cachePtr = ptr;
- }
/**
* Returns a writable version of ptr.
@@ -219,15 +161,21 @@ public:
}
private:
- mutable u_atomic_int32_t totalRefCount;
-
- // Any thread modifying softRefCount must hold the global cache mutex
+ /**
+ * The number of references from the UnifiedCache, which is
+ * the number of times that the sharedObject is stored as a hash table value.
+ * For use by UnifiedCache implementation code only.
+ * All access is synchronized by UnifiedCache's gCacheMutex
+ */
mutable int32_t softRefCount;
+ friend class UnifiedCache;
+ /**
+ * Reference count, excluding references from within the UnifiedCache implementation.
+ */
mutable u_atomic_int32_t hardRefCount;
+
mutable const UnifiedCacheBase *cachePtr;
- void addRef(UBool withCacheLock) const;
- void removeRef(UBool withCacheLock) const;
};
diff --git a/intl/icu/source/common/simpleformatter.cpp b/intl/icu/source/common/simpleformatter.cpp
index eaeb60de14..f866e0a1a1 100644
--- a/intl/icu/source/common/simpleformatter.cpp
+++ b/intl/icu/source/common/simpleformatter.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
diff --git a/intl/icu/source/common/sprpimpl.h b/intl/icu/source/common/sprpimpl.h
index 6e0bad0ae6..ca0bcdb516 100644
--- a/intl/icu/source/common/sprpimpl.h
+++ b/intl/icu/source/common/sprpimpl.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: sprpimpl.h
- * encoding: US-ASCII
+ * encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -90,7 +90,6 @@ struct UStringPrepProfile{
UTrie sprepTrie;
const uint16_t* mappingData;
UDataMemory* sprepData;
- const UBiDiProps *bdp; /* used only if checkBiDi is set */
int32_t refCount;
UBool isDataLoaded;
UBool doNFKC;
diff --git a/intl/icu/source/common/static_unicode_sets.cpp b/intl/icu/source/common/static_unicode_sets.cpp
new file mode 100644
index 0000000000..5d598a0e33
--- /dev/null
+++ b/intl/icu/source/common/static_unicode_sets.cpp
@@ -0,0 +1,223 @@
+// © 2018 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_FORMATTING
+
+// Allow implicit conversion from char16_t* to UnicodeString for this file:
+// Helpful in toString methods and elsewhere.
+#define UNISTR_FROM_STRING_EXPLICIT
+
+#include "static_unicode_sets.h"
+#include "umutex.h"
+#include "ucln_cmn.h"
+#include "unicode/uniset.h"
+#include "uresimp.h"
+#include "cstring.h"
+#include "uassert.h"
+
+using namespace icu;
+using namespace icu::unisets;
+
+
+namespace {
+
+UnicodeSet* gUnicodeSets[COUNT] = {};
+
+// Save the empty instance in static memory to have well-defined behavior if a
+// regular UnicodeSet cannot be allocated.
+alignas(UnicodeSet)
+char gEmptyUnicodeSet[sizeof(UnicodeSet)];
+
+// Whether the gEmptyUnicodeSet is initialized and ready to use.
+UBool gEmptyUnicodeSetInitialized = FALSE;
+
+inline UnicodeSet* getImpl(Key key) {
+ UnicodeSet* candidate = gUnicodeSets[key];
+ if (candidate == nullptr) {
+ return reinterpret_cast(gEmptyUnicodeSet);
+ }
+ return candidate;
+}
+
+UnicodeSet* computeUnion(Key k1, Key k2) {
+ UnicodeSet* result = new UnicodeSet();
+ if (result == nullptr) {
+ return nullptr;
+ }
+ result->addAll(*getImpl(k1));
+ result->addAll(*getImpl(k2));
+ result->freeze();
+ return result;
+}
+
+UnicodeSet* computeUnion(Key k1, Key k2, Key k3) {
+ UnicodeSet* result = new UnicodeSet();
+ if (result == nullptr) {
+ return nullptr;
+ }
+ result->addAll(*getImpl(k1));
+ result->addAll(*getImpl(k2));
+ result->addAll(*getImpl(k3));
+ result->freeze();
+ return result;
+}
+
+
+void saveSet(Key key, const UnicodeString& unicodeSetPattern, UErrorCode& status) {
+ // assert unicodeSets.get(key) == null;
+ gUnicodeSets[key] = new UnicodeSet(unicodeSetPattern, status);
+}
+
+class ParseDataSink : public ResourceSink {
+ public:
+ void put(const char* key, ResourceValue& value, UBool /*noFallback*/, UErrorCode& status) U_OVERRIDE {
+ ResourceTable contextsTable = value.getTable(status);
+ if (U_FAILURE(status)) { return; }
+ for (int i = 0; contextsTable.getKeyAndValue(i, key, value); i++) {
+ if (uprv_strcmp(key, "date") == 0) {
+ // ignore
+ } else {
+ ResourceTable strictnessTable = value.getTable(status);
+ if (U_FAILURE(status)) { return; }
+ for (int j = 0; strictnessTable.getKeyAndValue(j, key, value); j++) {
+ bool isLenient = (uprv_strcmp(key, "lenient") == 0);
+ ResourceArray array = value.getArray(status);
+ if (U_FAILURE(status)) { return; }
+ for (int k = 0; k < array.getSize(); k++) {
+ array.getValue(k, value);
+ UnicodeString str = value.getUnicodeString(status);
+ if (U_FAILURE(status)) { return; }
+ // There is both lenient and strict data for comma/period,
+ // but not for any of the other symbols.
+ if (str.indexOf(u'.') != -1) {
+ saveSet(isLenient ? PERIOD : STRICT_PERIOD, str, status);
+ } else if (str.indexOf(u',') != -1) {
+ saveSet(isLenient ? COMMA : STRICT_COMMA, str, status);
+ } else if (str.indexOf(u'+') != -1) {
+ saveSet(PLUS_SIGN, str, status);
+ } else if (str.indexOf(u'‒') != -1) {
+ saveSet(MINUS_SIGN, str, status);
+ } else if (str.indexOf(u'$') != -1) {
+ saveSet(DOLLAR_SIGN, str, status);
+ } else if (str.indexOf(u'£') != -1) {
+ saveSet(POUND_SIGN, str, status);
+ } else if (str.indexOf(u'₨') != -1) {
+ saveSet(RUPEE_SIGN, str, status);
+ }
+ if (U_FAILURE(status)) { return; }
+ }
+ }
+ }
+ }
+ }
+};
+
+
+icu::UInitOnce gNumberParseUniSetsInitOnce = U_INITONCE_INITIALIZER;
+
+UBool U_CALLCONV cleanupNumberParseUniSets() {
+ if (gEmptyUnicodeSetInitialized) {
+ reinterpret_cast(gEmptyUnicodeSet)->~UnicodeSet();
+ gEmptyUnicodeSetInitialized = FALSE;
+ }
+ for (int32_t i = 0; i < COUNT; i++) {
+ delete gUnicodeSets[i];
+ gUnicodeSets[i] = nullptr;
+ }
+ gNumberParseUniSetsInitOnce.reset();
+ return TRUE;
+}
+
+void U_CALLCONV initNumberParseUniSets(UErrorCode& status) {
+ ucln_common_registerCleanup(UCLN_COMMON_NUMPARSE_UNISETS, cleanupNumberParseUniSets);
+
+ // Initialize the empty instance for well-defined fallback behavior
+ new(gEmptyUnicodeSet) UnicodeSet();
+ reinterpret_cast(gEmptyUnicodeSet)->freeze();
+ gEmptyUnicodeSetInitialized = TRUE;
+
+ // These sets were decided after discussion with icu-design@. See tickets #13084 and #13309.
+ // Zs+TAB is "horizontal whitespace" according to UTS #18 (blank property).
+ gUnicodeSets[DEFAULT_IGNORABLES] = new UnicodeSet(
+ u"[[:Zs:][\\u0009][:Bidi_Control:][:Variation_Selector:]]", status);
+ gUnicodeSets[STRICT_IGNORABLES] = new UnicodeSet(u"[[:Bidi_Control:]]", status);
+
+ LocalUResourceBundlePointer rb(ures_open(nullptr, "root", &status));
+ if (U_FAILURE(status)) { return; }
+ ParseDataSink sink;
+ ures_getAllItemsWithFallback(rb.getAlias(), "parse", sink, status);
+ if (U_FAILURE(status)) { return; }
+
+ // NOTE: It is OK for these assertions to fail if there was a no-data build.
+ U_ASSERT(gUnicodeSets[COMMA] != nullptr);
+ U_ASSERT(gUnicodeSets[STRICT_COMMA] != nullptr);
+ U_ASSERT(gUnicodeSets[PERIOD] != nullptr);
+ U_ASSERT(gUnicodeSets[STRICT_PERIOD] != nullptr);
+
+ gUnicodeSets[OTHER_GROUPING_SEPARATORS] = new UnicodeSet(
+ u"['٬‘’'\\u0020\\u00A0\\u2000-\\u200A\\u202F\\u205F\\u3000]", status);
+ gUnicodeSets[ALL_SEPARATORS] = computeUnion(COMMA, PERIOD, OTHER_GROUPING_SEPARATORS);
+ gUnicodeSets[STRICT_ALL_SEPARATORS] = computeUnion(
+ STRICT_COMMA, STRICT_PERIOD, OTHER_GROUPING_SEPARATORS);
+
+ U_ASSERT(gUnicodeSets[MINUS_SIGN] != nullptr);
+ U_ASSERT(gUnicodeSets[PLUS_SIGN] != nullptr);
+
+ gUnicodeSets[PERCENT_SIGN] = new UnicodeSet(u"[%٪]", status);
+ gUnicodeSets[PERMILLE_SIGN] = new UnicodeSet(u"[‰؉]", status);
+ gUnicodeSets[INFINITY_KEY] = new UnicodeSet(u"[∞]", status);
+
+ U_ASSERT(gUnicodeSets[DOLLAR_SIGN] != nullptr);
+ U_ASSERT(gUnicodeSets[POUND_SIGN] != nullptr);
+ U_ASSERT(gUnicodeSets[RUPEE_SIGN] != nullptr);
+ gUnicodeSets[YEN_SIGN] = new UnicodeSet(u"[¥\\uffe5]", status);
+
+ gUnicodeSets[DIGITS] = new UnicodeSet(u"[:digit:]", status);
+
+ gUnicodeSets[DIGITS_OR_ALL_SEPARATORS] = computeUnion(DIGITS, ALL_SEPARATORS);
+ gUnicodeSets[DIGITS_OR_STRICT_ALL_SEPARATORS] = computeUnion(DIGITS, STRICT_ALL_SEPARATORS);
+
+ for (auto* uniset : gUnicodeSets) {
+ if (uniset != nullptr) {
+ uniset->freeze();
+ }
+ }
+}
+
+}
+
+const UnicodeSet* unisets::get(Key key) {
+ UErrorCode localStatus = U_ZERO_ERROR;
+ umtx_initOnce(gNumberParseUniSetsInitOnce, &initNumberParseUniSets, localStatus);
+ if (U_FAILURE(localStatus)) {
+ return reinterpret_cast(gEmptyUnicodeSet);
+ }
+ return getImpl(key);
+}
+
+Key unisets::chooseFrom(UnicodeString str, Key key1) {
+ return get(key1)->contains(str) ? key1 : NONE;
+}
+
+Key unisets::chooseFrom(UnicodeString str, Key key1, Key key2) {
+ return get(key1)->contains(str) ? key1 : chooseFrom(str, key2);
+}
+
+//Key unisets::chooseCurrency(UnicodeString str) {
+// if (get(DOLLAR_SIGN)->contains(str)) {
+// return DOLLAR_SIGN;
+// } else if (get(POUND_SIGN)->contains(str)) {
+// return POUND_SIGN;
+// } else if (get(RUPEE_SIGN)->contains(str)) {
+// return RUPEE_SIGN;
+// } else if (get(YEN_SIGN)->contains(str)) {
+// return YEN_SIGN;
+// } else {
+// return NONE;
+// }
+//}
+
+
+#endif /* #if !UCONFIG_NO_FORMATTING */
diff --git a/intl/icu/source/common/static_unicode_sets.h b/intl/icu/source/common/static_unicode_sets.h
new file mode 100644
index 0000000000..5f18b3217e
--- /dev/null
+++ b/intl/icu/source/common/static_unicode_sets.h
@@ -0,0 +1,119 @@
+// © 2018 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// This file is in common instead of i18n because it is needed by ucurr.cpp.
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_FORMATTING
+#ifndef __STATIC_UNICODE_SETS_H__
+#define __STATIC_UNICODE_SETS_H__
+
+#include "unicode/uniset.h"
+#include "unicode/unistr.h"
+
+U_NAMESPACE_BEGIN
+namespace unisets {
+
+enum Key {
+ // NONE is used to indicate null in chooseFrom().
+ // EMPTY is used to get an empty UnicodeSet.
+ NONE = -1,
+ EMPTY = 0,
+
+ // Ignorables
+ DEFAULT_IGNORABLES,
+ STRICT_IGNORABLES,
+
+ // Separators
+ // Notes:
+ // - COMMA is a superset of STRICT_COMMA
+ // - PERIOD is a superset of SCRICT_PERIOD
+ // - ALL_SEPARATORS is the union of COMMA, PERIOD, and OTHER_GROUPING_SEPARATORS
+ // - STRICT_ALL_SEPARATORS is the union of STRICT_COMMA, STRICT_PERIOD, and OTHER_GRP_SEPARATORS
+ COMMA,
+ PERIOD,
+ STRICT_COMMA,
+ STRICT_PERIOD,
+ OTHER_GROUPING_SEPARATORS,
+ ALL_SEPARATORS,
+ STRICT_ALL_SEPARATORS,
+
+ // Symbols
+ MINUS_SIGN,
+ PLUS_SIGN,
+ PERCENT_SIGN,
+ PERMILLE_SIGN,
+ INFINITY_KEY, // INFINITY is defined in cmath
+
+ // Currency Symbols
+ DOLLAR_SIGN,
+ POUND_SIGN,
+ RUPEE_SIGN,
+ YEN_SIGN, // not in CLDR data, but Currency.java wants it
+
+ // Other
+ DIGITS,
+
+ // Combined Separators with Digits (for lead code points)
+ DIGITS_OR_ALL_SEPARATORS,
+ DIGITS_OR_STRICT_ALL_SEPARATORS,
+
+ // The number of elements in the enum.
+ COUNT
+};
+
+/**
+ * Gets the static-allocated UnicodeSet according to the provided key. The
+ * pointer will be deleted during u_cleanup(); the caller should NOT delete it.
+ *
+ * Exported as U_COMMON_API for ucurr.cpp
+ *
+ * @param key The desired UnicodeSet according to the enum in this file.
+ * @return The requested UnicodeSet. Guaranteed to be frozen and non-null, but
+ * may be empty if an error occurred during data loading.
+ */
+U_COMMON_API const UnicodeSet* get(Key key);
+
+/**
+ * Checks if the UnicodeSet given by key1 contains the given string.
+ *
+ * Exported as U_COMMON_API for numparse_decimal.cpp
+ *
+ * @param str The string to check.
+ * @param key1 The set to check.
+ * @return key1 if the set contains str, or NONE if not.
+ */
+U_COMMON_API Key chooseFrom(UnicodeString str, Key key1);
+
+/**
+ * Checks if the UnicodeSet given by either key1 or key2 contains the string.
+ *
+ * Exported as U_COMMON_API for numparse_decimal.cpp
+ *
+ * @param str The string to check.
+ * @param key1 The first set to check.
+ * @param key2 The second set to check.
+ * @return key1 if that set contains str; key2 if that set contains str; or
+ * NONE if neither set contains str.
+ */
+U_COMMON_API Key chooseFrom(UnicodeString str, Key key1, Key key2);
+
+// Unused in C++:
+// Key chooseCurrency(UnicodeString str);
+// Used instead:
+static const struct {
+ Key key;
+ UChar32 exemplar;
+} kCurrencyEntries[] = {
+ {DOLLAR_SIGN, u'$'},
+ {POUND_SIGN, u'£'},
+ {RUPEE_SIGN, u'₨'},
+ {YEN_SIGN, u'¥'},
+};
+
+} // namespace unisets
+U_NAMESPACE_END
+
+#endif //__STATIC_UNICODE_SETS_H__
+#endif /* #if !UCONFIG_NO_FORMATTING */
diff --git a/intl/icu/source/common/stringpiece.cpp b/intl/icu/source/common/stringpiece.cpp
index b032b474f6..d4f7f310ba 100644
--- a/intl/icu/source/common/stringpiece.cpp
+++ b/intl/icu/source/common/stringpiece.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
// Copyright (C) 2009-2013, International Business Machines
// Corporation and others. All Rights Reserved.
diff --git a/intl/icu/source/common/stringtriebuilder.cpp b/intl/icu/source/common/stringtriebuilder.cpp
index 075d7c4324..6f9cc2e5c2 100644
--- a/intl/icu/source/common/stringtriebuilder.cpp
+++ b/intl/icu/source/common/stringtriebuilder.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -6,7 +6,7 @@
* Corporation and others. All Rights Reserved.
*******************************************************************************
* file name: stringtriebuilder.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -373,7 +373,7 @@ StringTrieBuilder::registerFinalValue(int32_t value, UErrorCode &errorCode) {
return newNode;
}
-UBool
+int32_t
StringTrieBuilder::hashNode(const void *node) {
return ((const Node *)node)->hashCode();
}
diff --git a/intl/icu/source/common/uarrsort.c b/intl/icu/source/common/uarrsort.cpp
similarity index 98%
rename from intl/icu/source/common/uarrsort.c
rename to intl/icu/source/common/uarrsort.cpp
index bb1b5bdd78..03c4d4e7fc 100644
--- a/intl/icu/source/common/uarrsort.c
+++ b/intl/icu/source/common/uarrsort.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: uarrsort.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -38,16 +38,19 @@ enum {
U_CAPI int32_t U_EXPORT2
uprv_uint16Comparator(const void *context, const void *left, const void *right) {
+ (void)context;
return (int32_t)*(const uint16_t *)left - (int32_t)*(const uint16_t *)right;
}
U_CAPI int32_t U_EXPORT2
uprv_int32Comparator(const void *context, const void *left, const void *right) {
+ (void)context;
return *(const int32_t *)left - *(const int32_t *)right;
}
U_CAPI int32_t U_EXPORT2
uprv_uint32Comparator(const void *context, const void *left, const void *right) {
+ (void)context;
uint32_t l=*(const uint32_t *)left, r=*(const uint32_t *)right;
/* compare directly because (l-r) would overflow the int32_t result */
diff --git a/intl/icu/source/common/uarrsort.h b/intl/icu/source/common/uarrsort.h
index 8b25cc5d7f..a55dca5b9e 100644
--- a/intl/icu/source/common/uarrsort.h
+++ b/intl/icu/source/common/uarrsort.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: uarrsort.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/uassert.h b/intl/icu/source/common/uassert.h
index 5a048a5ce5..fd6b6c4c01 100644
--- a/intl/icu/source/common/uassert.h
+++ b/intl/icu/source/common/uassert.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
diff --git a/intl/icu/source/common/ubidi.c b/intl/icu/source/common/ubidi.cpp
similarity index 96%
rename from intl/icu/source/common/ubidi.c
rename to intl/icu/source/common/ubidi.cpp
index 3c94978626..4b65d49185 100644
--- a/intl/icu/source/common/ubidi.c
+++ b/intl/icu/source/common/ubidi.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -8,7 +8,7 @@
*
******************************************************************************
* file name: ubidi.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -152,9 +152,6 @@ ubidi_openSized(int32_t maxLength, int32_t maxRunCount, UErrorCode *pErrorCode)
/* reset the object, all pointers NULL, all flags FALSE, all sizes 0 */
uprv_memset(pBiDi, 0, sizeof(UBiDi));
- /* get BiDi properties */
- pBiDi->bdp=ubidi_getSingleton();
-
/* allocate memory for arrays as requested */
if(maxLength>0) {
if( !getInitialDirPropsMemory(pBiDi, maxLength) ||
@@ -447,12 +444,12 @@ getDirProps(UBiDi *pBiDi) {
UBool removeBiDiControls = (UBool)(pBiDi->reorderingOptions &
UBIDI_OPTION_REMOVE_CONTROLS);
- typedef enum {
+ enum State {
NOT_SEEKING_STRONG, /* 0: not contextual paraLevel, not after FSI */
SEEKING_STRONG_FOR_PARA, /* 1: looking for first strong char in para */
SEEKING_STRONG_FOR_FSI, /* 2: looking for first strong after FSI */
LOOKING_FOR_PDI /* 3: found strong after FSI, looking for PDI */
- } State;
+ };
State state;
DirProp lastStrong=ON; /* for default level & inverse BiDi */
/* The following stacks are used to manage isolate sequences. Those
@@ -466,7 +463,7 @@ getDirProps(UBiDi *pBiDi) {
int32_t isolateStartStack[UBIDI_MAX_EXPLICIT_LEVEL+1];
/* The following stack contains the last known state before
encountering the initiator of an isolate sequence */
- int8_t previousStateStack[UBIDI_MAX_EXPLICIT_LEVEL+1];
+ State previousStateStack[UBIDI_MAX_EXPLICIT_LEVEL+1];
int32_t stackLast=-1;
if(pBiDi->reorderingOptions & UBIDI_OPTION_STREAMING)
@@ -627,7 +624,7 @@ getDirProps(UBiDi *pBiDi) {
pBiDi->paras[pBiDi->paraCount-1].level=1;
}
if(isDefaultLevel) {
- pBiDi->paraLevel=pBiDi->paras[0].level;
+ pBiDi->paraLevel=static_cast(pBiDi->paras[0].level);
}
/* The following is needed to resolve the text direction for default level
paragraphs containing no strong character */
@@ -677,7 +674,9 @@ bracketInit(UBiDi *pBiDi, BracketData *bd) {
bd->isoRuns[0].start=0;
bd->isoRuns[0].limit=0;
bd->isoRuns[0].level=GET_PARALEVEL(pBiDi, 0);
- bd->isoRuns[0].lastStrong=bd->isoRuns[0].lastBase=bd->isoRuns[0].contextDir=GET_PARALEVEL(pBiDi, 0)&1;
+ UBiDiLevel t = GET_PARALEVEL(pBiDi, 0) & 1;
+ bd->isoRuns[0].lastStrong = bd->isoRuns[0].lastBase = t;
+ bd->isoRuns[0].contextDir = (UBiDiDirection)t;
bd->isoRuns[0].contextPos=0;
if(pBiDi->openingsMemory) {
bd->openings=pBiDi->openingsMemory;
@@ -696,7 +695,8 @@ bracketProcessB(BracketData *bd, UBiDiLevel level) {
bd->isoRunLast=0;
bd->isoRuns[0].limit=0;
bd->isoRuns[0].level=level;
- bd->isoRuns[0].lastStrong=bd->isoRuns[0].lastBase=bd->isoRuns[0].contextDir=level&1;
+ bd->isoRuns[0].lastStrong=bd->isoRuns[0].lastBase=level&1;
+ bd->isoRuns[0].contextDir=(UBiDiDirection)(level&1);
bd->isoRuns[0].contextPos=0;
}
@@ -712,8 +712,9 @@ bracketProcessBoundary(BracketData *bd, int32_t lastCcPos,
contextLevel=embeddingLevel;
pLastIsoRun->limit=pLastIsoRun->start;
pLastIsoRun->level=embeddingLevel;
- pLastIsoRun->lastStrong=pLastIsoRun->lastBase=pLastIsoRun->contextDir=contextLevel&1;
- pLastIsoRun->contextPos=lastCcPos;
+ pLastIsoRun->lastStrong=pLastIsoRun->lastBase=contextLevel&1;
+ pLastIsoRun->contextDir=(UBiDiDirection)(contextLevel&1);
+ pLastIsoRun->contextPos=(UBiDiDirection)lastCcPos;
}
/* LRI or RLI */
@@ -727,7 +728,8 @@ bracketProcessLRI_RLI(BracketData *bd, UBiDiLevel level) {
pLastIsoRun++;
pLastIsoRun->start=pLastIsoRun->limit=lastLimit;
pLastIsoRun->level=level;
- pLastIsoRun->lastStrong=pLastIsoRun->lastBase=pLastIsoRun->contextDir=level&1;
+ pLastIsoRun->lastStrong=pLastIsoRun->lastBase=level&1;
+ pLastIsoRun->contextDir=(UBiDiDirection)(level&1);
pLastIsoRun->contextPos=0;
}
@@ -801,7 +803,7 @@ bracketProcessClosing(BracketData *bd, int32_t openIdx, int32_t position) {
UBool stable;
DirProp newProp;
pOpening=&bd->openings[openIdx];
- direction=pLastIsoRun->level&1;
+ direction=(UBiDiDirection)(pLastIsoRun->level&1);
stable=TRUE; /* assume stable until proved otherwise */
/* The stable flag is set when brackets are paired and their
@@ -823,28 +825,28 @@ bracketProcessClosing(BracketData *bd, int32_t openIdx, int32_t position) {
N0c1. */
if((direction==0 && pOpening->flags&FOUND_L) ||
- (direction==1 && pOpening->flags&FOUND_R)) { /* N0b */
- newProp=direction;
+ (direction==1 && pOpening->flags&FOUND_R)) { /* N0b */
+ newProp=static_cast(direction);
}
- else if(pOpening->flags&(FOUND_L|FOUND_R)) { /* N0c */
+ else if(pOpening->flags&(FOUND_L|FOUND_R)) { /* N0c */
/* it is stable if there is no containing pair or in
conditions too complicated and not worth checking */
stable=(openIdx==pLastIsoRun->start);
if(direction!=pOpening->contextDir)
- newProp=pOpening->contextDir; /* N0c1 */
+ newProp= static_cast(pOpening->contextDir); /* N0c1 */
else
- newProp=direction; /* N0c2 */
+ newProp= static_cast(direction); /* N0c2 */
} else {
/* forget this and any brackets nested within this pair */
- pLastIsoRun->limit=openIdx;
- return ON; /* N0d */
+ pLastIsoRun->limit= static_cast(openIdx);
+ return ON; /* N0d */
}
bd->pBiDi->dirProps[pOpening->position]=newProp;
bd->pBiDi->dirProps[position]=newProp;
/* Update nested N0c pairs that may be affected */
fixN0c(bd, openIdx, pOpening->position, newProp);
if(stable) {
- pLastIsoRun->limit=openIdx; /* forget any brackets nested within this pair */
+ pLastIsoRun->limit= static_cast(openIdx); /* forget any brackets nested within this pair */
/* remove lower located synonyms if any */
while(pLastIsoRun->limit>pLastIsoRun->start &&
bd->openings[pLastIsoRun->limit-1].position==pOpening->position)
@@ -894,7 +896,7 @@ bracketProcessChar(BracketData *bd, int32_t position) {
break;
}
pLastIsoRun->lastBase=ON;
- pLastIsoRun->contextDir=newProp;
+ pLastIsoRun->contextDir=(UBiDiDirection)newProp;
pLastIsoRun->contextPos=position;
level=bd->pBiDi->levels[position];
if(level&UBIDI_LEVEL_OVERRIDE) { /* X4, X5 */
@@ -916,11 +918,11 @@ bracketProcessChar(BracketData *bd, int32_t position) {
bracket or it is a case of N0d */
/* Now see if it is an opening bracket */
if(c)
- match=u_getBidiPairedBracket(c); /* get the matching char */
+ match= static_cast(u_getBidiPairedBracket(c)); /* get the matching char */
else
match=0;
if(match!=c && /* has a matching char */
- ubidi_getPairedBracketType(bd->pBiDi->bdp, c)==U_BPT_OPEN) { /* opening bracket */
+ ubidi_getPairedBracketType(c)==U_BPT_OPEN) { /* opening bracket */
/* special case: process synonyms
create an opening entry for each synonym */
if(match==0x232A) { /* RIGHT-POINTING ANGLE BRACKET */
@@ -942,14 +944,14 @@ bracketProcessChar(BracketData *bd, int32_t position) {
dirProps[position]=newProp;
pLastIsoRun->lastBase=newProp;
pLastIsoRun->lastStrong=newProp;
- pLastIsoRun->contextDir=newProp;
+ pLastIsoRun->contextDir=(UBiDiDirection)newProp;
pLastIsoRun->contextPos=position;
}
else if(dirProp<=R || dirProp==AL) {
- newProp=DIR_FROM_STRONG(dirProp);
+ newProp= static_cast(DIR_FROM_STRONG(dirProp));
pLastIsoRun->lastBase=dirProp;
pLastIsoRun->lastStrong=dirProp;
- pLastIsoRun->contextDir=newProp;
+ pLastIsoRun->contextDir=(UBiDiDirection)newProp;
pLastIsoRun->contextPos=position;
}
else if(dirProp==EN) {
@@ -958,7 +960,7 @@ bracketProcessChar(BracketData *bd, int32_t position) {
newProp=L; /* W7 */
if(!bd->isNumbersSpecial)
dirProps[position]=ENL;
- pLastIsoRun->contextDir=L;
+ pLastIsoRun->contextDir=(UBiDiDirection)L;
pLastIsoRun->contextPos=position;
}
else {
@@ -967,14 +969,14 @@ bracketProcessChar(BracketData *bd, int32_t position) {
dirProps[position]=AN; /* W2 */
else
dirProps[position]=ENR;
- pLastIsoRun->contextDir=R;
+ pLastIsoRun->contextDir=(UBiDiDirection)R;
pLastIsoRun->contextPos=position;
}
}
else if(dirProp==AN) {
newProp=R; /* N0 */
pLastIsoRun->lastBase=AN;
- pLastIsoRun->contextDir=R;
+ pLastIsoRun->contextDir=(UBiDiDirection)R;
pLastIsoRun->contextPos=position;
}
else if(dirProp==NSM) {
@@ -1099,7 +1101,7 @@ resolveExplicitLevels(UBiDi *pBiDi, UErrorCode *pErrorCode) {
else
start=pBiDi->paras[paraIndex-1].limit;
limit=pBiDi->paras[paraIndex].limit;
- level=pBiDi->paras[paraIndex].level;
+ level= static_cast(pBiDi->paras[paraIndex].level);
for(i=start; iparas[paraIndex-1].limit;
limit=pBiDi->paras[paraIndex].limit;
- level=pBiDi->paras[paraIndex].level;
+ level= static_cast(pBiDi->paras[paraIndex].level);
for(i=start; idirProps;
- DirProp dirProp;
UBiDiLevel *levels=pBiDi->levels;
int32_t isolateCount=0;
- int32_t i, length=pBiDi->length;
+ int32_t length=pBiDi->length;
Flags flags=0; /* collect all directionalities in the text */
- UBiDiLevel level;
pBiDi->isolateCount=0;
- for(i=0; iparas[0].limit;
+ int32_t currentParaLevel = pBiDi->paraLevel;
+
+ for(int32_t i=0; ipBiDi->isolateCount)
@@ -1364,21 +1368,41 @@ checkExplicitLevels(UBiDi *pBiDi, UErrorCode *pErrorCode) {
isolateCount--;
else if(dirProp==B)
isolateCount=0;
- if(level&UBIDI_LEVEL_OVERRIDE) {
+
+ // optimized version of int32_t currentParaLevel = GET_PARALEVEL(pBiDi, i);
+ if (pBiDi->defaultParaLevel != 0 &&
+ i == currentParaLimit && (currentParaIndex + 1) < pBiDi->paraCount) {
+ currentParaLevel = pBiDi->paras[++currentParaIndex].level;
+ currentParaLimit = pBiDi->paras[currentParaIndex].limit;
+ }
+
+ UBiDiLevel overrideFlag = level & UBIDI_LEVEL_OVERRIDE;
+ level &= ~UBIDI_LEVEL_OVERRIDE;
+ if (level < currentParaLevel || UBIDI_MAX_EXPLICIT_LEVEL < level) {
+ if (level == 0) {
+ if (dirProp == B) {
+ // Paragraph separators are ok with explicit level 0.
+ // Prevents reordering of paragraphs.
+ } else {
+ // Treat explicit level 0 as a wildcard for the paragraph level.
+ // Avoid making the caller guess what the paragraph level would be.
+ level = (UBiDiLevel)currentParaLevel;
+ levels[i] = level | overrideFlag;
+ }
+ } else {
+ // 1 <= level < currentParaLevel or UBIDI_MAX_EXPLICIT_LEVEL < level
+ /* level out of bounds */
+ *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
+ return UBIDI_LTR;
+ }
+ }
+ if (overrideFlag != 0) {
/* keep the override flag in levels[i] but adjust the flags */
- level&=~UBIDI_LEVEL_OVERRIDE; /* make the range check below simpler */
flags|=DIRPROP_FLAG_O(level);
} else {
/* set the flags */
flags|=DIRPROP_FLAG_E(level)|DIRPROP_FLAG(dirProp);
}
- if((levelparaLevel);
@@ -1772,7 +1796,7 @@ addPoint(UBiDi *pBiDi, int32_t pos, int32_t flag)
if (pInsertPoints->capacity == 0)
{
- pInsertPoints->points=uprv_malloc(sizeof(Point)*FIRSTALLOC);
+ pInsertPoints->points=static_cast(uprv_malloc(sizeof(Point)*FIRSTALLOC));
if (pInsertPoints->points == NULL)
{
pInsertPoints->errorCode=U_MEMORY_ALLOCATION_ERROR;
@@ -1782,9 +1806,9 @@ addPoint(UBiDi *pBiDi, int32_t pos, int32_t flag)
}
if (pInsertPoints->size >= pInsertPoints->capacity) /* no room for new point */
{
- void * savePoints=pInsertPoints->points;
- pInsertPoints->points=uprv_realloc(pInsertPoints->points,
- pInsertPoints->capacity*2*sizeof(Point));
+ Point * savePoints=pInsertPoints->points;
+ pInsertPoints->points=static_cast(uprv_realloc(pInsertPoints->points,
+ pInsertPoints->capacity*2*sizeof(Point)));
if (pInsertPoints->points == NULL)
{
pInsertPoints->points=savePoints;
@@ -2342,7 +2366,7 @@ setParaSuccess(UBiDi *pBiDi) {
static void
setParaRunsOnly(UBiDi *pBiDi, const UChar *text, int32_t length,
UBiDiLevel paraLevel, UErrorCode *pErrorCode) {
- void *runsOnlyMemory = NULL;
+ int32_t *runsOnlyMemory = NULL;
int32_t *visualMap;
UChar *visualText;
int32_t saveLength, saveTrailingWSStart;
@@ -2363,7 +2387,7 @@ setParaRunsOnly(UBiDi *pBiDi, const UChar *text, int32_t length,
goto cleanup3;
}
/* obtain memory for mapping table and visual text */
- runsOnlyMemory=uprv_malloc(length*(sizeof(int32_t)+sizeof(UChar)+sizeof(UBiDiLevel)));
+ runsOnlyMemory=static_cast(uprv_malloc(length*(sizeof(int32_t)+sizeof(UChar)+sizeof(UBiDiLevel))));
if(runsOnlyMemory==NULL) {
*pErrorCode=U_MEMORY_ALLOCATION_ERROR;
goto cleanup3;
@@ -2558,7 +2582,7 @@ ubidi_setPara(UBiDi *pBiDi, const UChar *text, int32_t length,
pBiDi->text=text;
pBiDi->length=pBiDi->originalLength=pBiDi->resultLength=length;
pBiDi->paraLevel=paraLevel;
- pBiDi->direction=paraLevel&1;
+ pBiDi->direction=(UBiDiDirection)(paraLevel&1);
pBiDi->paraCount=1;
pBiDi->dirProps=NULL;
@@ -2803,7 +2827,7 @@ ubidi_setPara(UBiDi *pBiDi, const UChar *text, int32_t length,
DirProp dirProp;
for(i=0; iparaCount; i++) {
last=(pBiDi->paras[i].limit)-1;
- level=pBiDi->paras[i].level;
+ level= static_cast(pBiDi->paras[i].level);
if(level==0)
continue; /* LTR paragraph */
start= i==0 ? 0 : pBiDi->paras[i-1].limit;
@@ -3006,10 +3030,10 @@ ubidi_getCustomizedClass(UBiDi *pBiDi, UChar32 c)
if( pBiDi->fnClassCallback == NULL ||
(dir = (*pBiDi->fnClassCallback)(pBiDi->coClassCallback, c)) == U_BIDI_CLASS_DEFAULT )
{
- dir = ubidi_getClass(pBiDi->bdp, c);
+ dir = ubidi_getClass(c);
}
if(dir >= U_CHAR_DIRECTION_COUNT) {
- dir = ON;
+ dir = (UCharDirection)ON;
}
return dir;
}
diff --git a/intl/icu/source/common/ubidi_props.c b/intl/icu/source/common/ubidi_props.cpp
similarity index 65%
rename from intl/icu/source/common/ubidi_props.c
rename to intl/icu/source/common/ubidi_props.cpp
index 7c7a6ce25e..4141c21938 100644
--- a/intl/icu/source/common/ubidi_props.c
+++ b/intl/icu/source/common/ubidi_props.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: ubidi_props.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -44,17 +44,12 @@ struct UBiDiProps {
#define INCLUDED_FROM_UBIDI_PROPS_C
#include "ubidi_props_data.h"
-/* UBiDiProps singleton ----------------------------------------------------- */
-
-U_CFUNC const UBiDiProps *
-ubidi_getSingleton() {
- return &ubidi_props_singleton;
-}
-
/* set of property starts for UnicodeSet ------------------------------------ */
static UBool U_CALLCONV
_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 end, uint32_t value) {
+ (void)end;
+ (void)value;
/* add the start code point to the USet */
const USetAdder *sa=(const USetAdder *)context;
sa->add(sa->set, start);
@@ -62,7 +57,7 @@ _enumPropertyStartsRange(const void *context, UChar32 start, UChar32 end, uint32
}
U_CFUNC void
-ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *pErrorCode) {
+ubidi_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) {
int32_t i, length;
UChar32 c, start, limit;
@@ -74,19 +69,19 @@ ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *
}
/* add the start code point of each same-value range of the trie */
- utrie2_enum(&bdp->trie, NULL, _enumPropertyStartsRange, sa);
+ utrie2_enum(&ubidi_props_singleton.trie, NULL, _enumPropertyStartsRange, sa);
/* add the code points from the bidi mirroring table */
- length=bdp->indexes[UBIDI_IX_MIRROR_LENGTH];
+ length=ubidi_props_singleton.indexes[UBIDI_IX_MIRROR_LENGTH];
for(i=0; imirrors[i]);
+ c=UBIDI_GET_MIRROR_CODE_POINT(ubidi_props_singleton.mirrors[i]);
sa->addRange(sa->set, c, c+1);
}
/* add the code points from the Joining_Group array where the value changes */
- start=bdp->indexes[UBIDI_IX_JG_START];
- limit=bdp->indexes[UBIDI_IX_JG_LIMIT];
- jgArray=bdp->jgArray;
+ start=ubidi_props_singleton.indexes[UBIDI_IX_JG_START];
+ limit=ubidi_props_singleton.indexes[UBIDI_IX_JG_LIMIT];
+ jgArray=ubidi_props_singleton.jgArray;
for(;;) {
prev=0;
while(startadd(sa->set, limit);
}
- if(limit==bdp->indexes[UBIDI_IX_JG_LIMIT]) {
+ if(limit==ubidi_props_singleton.indexes[UBIDI_IX_JG_LIMIT]) {
/* switch to the second Joining_Group range */
- start=bdp->indexes[UBIDI_IX_JG_START2];
- limit=bdp->indexes[UBIDI_IX_JG_LIMIT2];
- jgArray=bdp->jgArray2;
+ start=ubidi_props_singleton.indexes[UBIDI_IX_JG_START2];
+ limit=ubidi_props_singleton.indexes[UBIDI_IX_JG_LIMIT2];
+ jgArray=ubidi_props_singleton.jgArray2;
} else {
break;
}
@@ -119,14 +114,8 @@ ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *
/* property access functions ------------------------------------------------ */
U_CFUNC int32_t
-ubidi_getMaxValue(const UBiDiProps *bdp, UProperty which) {
- int32_t max;
-
- if(bdp==NULL) {
- return -1;
- }
-
- max=bdp->indexes[UBIDI_MAX_VALUES_INDEX];
+ubidi_getMaxValue(UProperty which) {
+ int32_t max=ubidi_props_singleton.indexes[UBIDI_MAX_VALUES_INDEX];
switch(which) {
case UCHAR_BIDI_CLASS:
return (max&UBIDI_CLASS_MASK);
@@ -142,19 +131,19 @@ ubidi_getMaxValue(const UBiDiProps *bdp, UProperty which) {
}
U_CAPI UCharDirection
-ubidi_getClass(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_getClass(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UCharDirection)UBIDI_GET_CLASS(props);
}
U_CFUNC UBool
-ubidi_isMirrored(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_isMirrored(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UBool)UBIDI_GET_FLAG(props, UBIDI_IS_MIRRORED_SHIFT);
}
static UChar32
-getMirror(const UBiDiProps *bdp, UChar32 c, uint16_t props) {
+getMirror(UChar32 c, uint16_t props) {
int32_t delta=UBIDI_GET_MIRROR_DELTA(props);
if(delta!=UBIDI_ESC_MIRROR_DELTA) {
return c+delta;
@@ -165,8 +154,8 @@ getMirror(const UBiDiProps *bdp, UChar32 c, uint16_t props) {
int32_t i, length;
UChar32 c2;
- mirrors=bdp->mirrors;
- length=bdp->indexes[UBIDI_IX_MIRROR_LENGTH];
+ mirrors=ubidi_props_singleton.mirrors;
+ length=ubidi_props_singleton.indexes[UBIDI_IX_MIRROR_LENGTH];
/* linear search */
for(i=0; itrie, c);
- return getMirror(bdp, c, props);
+ubidi_getMirror(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
+ return getMirror(c, props);
}
U_CFUNC UBool
-ubidi_isBidiControl(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_isBidiControl(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UBool)UBIDI_GET_FLAG(props, UBIDI_BIDI_CONTROL_SHIFT);
}
U_CFUNC UBool
-ubidi_isJoinControl(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_isJoinControl(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UBool)UBIDI_GET_FLAG(props, UBIDI_JOIN_CONTROL_SHIFT);
}
U_CFUNC UJoiningType
-ubidi_getJoiningType(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_getJoiningType(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UJoiningType)((props&UBIDI_JT_MASK)>>UBIDI_JT_SHIFT);
}
U_CFUNC UJoiningGroup
-ubidi_getJoiningGroup(const UBiDiProps *bdp, UChar32 c) {
+ubidi_getJoiningGroup(UChar32 c) {
UChar32 start, limit;
- start=bdp->indexes[UBIDI_IX_JG_START];
- limit=bdp->indexes[UBIDI_IX_JG_LIMIT];
+ start=ubidi_props_singleton.indexes[UBIDI_IX_JG_START];
+ limit=ubidi_props_singleton.indexes[UBIDI_IX_JG_LIMIT];
if(start<=c && cjgArray[c-start];
+ return (UJoiningGroup)ubidi_props_singleton.jgArray[c-start];
}
- start=bdp->indexes[UBIDI_IX_JG_START2];
- limit=bdp->indexes[UBIDI_IX_JG_LIMIT2];
+ start=ubidi_props_singleton.indexes[UBIDI_IX_JG_START2];
+ limit=ubidi_props_singleton.indexes[UBIDI_IX_JG_LIMIT2];
if(start<=c && cjgArray2[c-start];
+ return (UJoiningGroup)ubidi_props_singleton.jgArray2[c-start];
}
return U_JG_NO_JOINING_GROUP;
}
U_CFUNC UBidiPairedBracketType
-ubidi_getPairedBracketType(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_getPairedBracketType(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
return (UBidiPairedBracketType)((props&UBIDI_BPT_MASK)>>UBIDI_BPT_SHIFT);
}
U_CFUNC UChar32
-ubidi_getPairedBracket(const UBiDiProps *bdp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&bdp->trie, c);
+ubidi_getPairedBracket(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ubidi_props_singleton.trie, c);
if((props&UBIDI_BPT_MASK)==0) {
return c;
} else {
- return getMirror(bdp, c, props);
+ return getMirror(c, props);
}
}
/* public API (see uchar.h) ------------------------------------------------- */
U_CFUNC UCharDirection
-u_charDirection(UChar32 c) {
- return ubidi_getClass(&ubidi_props_singleton, c);
+u_charDirection(UChar32 c) {
+ return ubidi_getClass(c);
}
U_CFUNC UBool
u_isMirrored(UChar32 c) {
- return ubidi_isMirrored(&ubidi_props_singleton, c);
+ return ubidi_isMirrored(c);
}
U_CFUNC UChar32
u_charMirror(UChar32 c) {
- return ubidi_getMirror(&ubidi_props_singleton, c);
+ return ubidi_getMirror(c);
}
U_STABLE UChar32 U_EXPORT2
u_getBidiPairedBracket(UChar32 c) {
- return ubidi_getPairedBracket(&ubidi_props_singleton, c);
+ return ubidi_getPairedBracket(c);
}
diff --git a/intl/icu/source/common/ubidi_props.h b/intl/icu/source/common/ubidi_props.h
index 4312230bc0..698ee9c52b 100644
--- a/intl/icu/source/common/ubidi_props.h
+++ b/intl/icu/source/common/ubidi_props.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: ubidi_props.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -31,46 +31,40 @@ U_CDECL_BEGIN
/* library API -------------------------------------------------------------- */
-struct UBiDiProps;
-typedef struct UBiDiProps UBiDiProps;
-
-U_CFUNC const UBiDiProps *
-ubidi_getSingleton(void);
-
U_CFUNC void
-ubidi_addPropertyStarts(const UBiDiProps *bdp, const USetAdder *sa, UErrorCode *pErrorCode);
+ubidi_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode);
/* property access functions */
U_CFUNC int32_t
-ubidi_getMaxValue(const UBiDiProps *bdp, UProperty which);
+ubidi_getMaxValue(UProperty which);
U_CAPI UCharDirection
-ubidi_getClass(const UBiDiProps *bdp, UChar32 c);
+ubidi_getClass(UChar32 c);
U_CFUNC UBool
-ubidi_isMirrored(const UBiDiProps *bdp, UChar32 c);
+ubidi_isMirrored(UChar32 c);
U_CFUNC UChar32
-ubidi_getMirror(const UBiDiProps *bdp, UChar32 c);
+ubidi_getMirror(UChar32 c);
U_CFUNC UBool
-ubidi_isBidiControl(const UBiDiProps *bdp, UChar32 c);
+ubidi_isBidiControl(UChar32 c);
U_CFUNC UBool
-ubidi_isJoinControl(const UBiDiProps *bdp, UChar32 c);
+ubidi_isJoinControl(UChar32 c);
U_CFUNC UJoiningType
-ubidi_getJoiningType(const UBiDiProps *bdp, UChar32 c);
+ubidi_getJoiningType(UChar32 c);
U_CFUNC UJoiningGroup
-ubidi_getJoiningGroup(const UBiDiProps *bdp, UChar32 c);
+ubidi_getJoiningGroup(UChar32 c);
U_CFUNC UBidiPairedBracketType
-ubidi_getPairedBracketType(const UBiDiProps *bdp, UChar32 c);
+ubidi_getPairedBracketType(UChar32 c);
U_CFUNC UChar32
-ubidi_getPairedBracket(const UBiDiProps *bdp, UChar32 c);
+ubidi_getPairedBracket(UChar32 c);
/* file definitions --------------------------------------------------------- */
diff --git a/intl/icu/source/common/ubidi_props_data.h b/intl/icu/source/common/ubidi_props_data.h
index 685d2b1e84..55e331b612 100644
--- a/intl/icu/source/common/ubidi_props_data.h
+++ b/intl/icu/source/common/ubidi_props_data.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
//
// Copyright (C) 1999-2016, International Business Machines
@@ -11,342 +11,346 @@
#ifdef INCLUDED_FROM_UBIDI_PROPS_C
-static const UVersionInfo ubidi_props_dataVersion={9,0,0,0};
+static const UVersionInfo ubidi_props_dataVersion={0xb,0,0,0};
-static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={0x10,0x5df0,0x5a78,0x1a,0x620,0x8c0,0x10ac0,0x10af0,0,0,0,0,0,0,0,0x5802b6};
+static const int32_t ubidi_props_indexes[UBIDI_IX_TOP]={0x10,0x647c,0x5e98,0x28,0x620,0x8c0,0x10ac0,0x10d24,0,0,0,0,0,0,0,0x6502b6};
-static const uint16_t ubidi_props_trieIndex[11572]={
-0x36a,0x372,0x37a,0x382,0x39a,0x3a2,0x3aa,0x3b2,0x38a,0x392,0x38a,0x392,0x38a,0x392,0x38a,0x392,
-0x38a,0x392,0x38a,0x392,0x3b8,0x3c0,0x3c8,0x3d0,0x3d8,0x3e0,0x3dc,0x3e4,0x3ec,0x3f4,0x3ef,0x3f7,
-0x38a,0x392,0x38a,0x392,0x3ff,0x407,0x38a,0x392,0x38a,0x392,0x38a,0x392,0x40d,0x415,0x41d,0x425,
-0x42d,0x435,0x43d,0x445,0x44b,0x453,0x45b,0x463,0x46b,0x473,0x479,0x481,0x489,0x491,0x499,0x4a1,
-0x4ad,0x4a9,0x4b5,0x41f,0x41f,0x4c5,0x4cd,0x4bd,0x4d5,0x4d7,0x4df,0x4e7,0x4ef,0x4f0,0x4f8,0x500,
-0x508,0x4f0,0x510,0x515,0x508,0x4f0,0x51d,0x525,0x4ef,0x52a,0x532,0x4e7,0x537,0x38a,0x53f,0x543,
-0x54b,0x54c,0x554,0x55c,0x4ef,0x564,0x56c,0x4e7,0x4ef,0x38a,0x4f8,0x4e7,0x38a,0x38a,0x572,0x38a,
-0x38a,0x578,0x580,0x38a,0x38a,0x584,0x58c,0x38a,0x590,0x597,0x38a,0x59f,0x5a7,0x5ae,0x536,0x38a,
-0x38a,0x5b6,0x5be,0x5c6,0x5ce,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x5d6,0x38a,0x5de,0x38a,0x38a,0x38a,
-0x5e6,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x5ee,0x38a,0x38a,0x38a,0x5f6,0x5f6,0x4fc,0x4fc,0x38a,0x5fc,0x604,0x5de,
-0x61a,0x60c,0x60c,0x622,0x629,0x612,0x38a,0x38a,0x38a,0x631,0x639,0x38a,0x38a,0x38a,0x63b,0x643,
-0x64b,0x38a,0x652,0x65a,0x38a,0x662,0x38a,0x38a,0x66a,0x66d,0x537,0x675,0x401,0x67d,0x38a,0x684,
-0x38a,0x689,0x38a,0x38a,0x38a,0x38a,0x68f,0x697,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x3d8,0x69f,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x6a7,0x6af,0x6b3,
-0x6cb,0x6d1,0x6bb,0x6c3,0x6d9,0x6e1,0x6e5,0x5b1,0x6ed,0x6f5,0x6fd,0x38a,0x705,0x643,0x643,0x643,
-0x715,0x71d,0x725,0x72d,0x732,0x73a,0x742,0x70d,0x74a,0x752,0x38a,0x758,0x75f,0x643,0x643,0x765,
-0x643,0x562,0x76a,0x643,0x772,0x38a,0x38a,0x640,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,
-0x643,0x643,0x643,0x643,0x643,0x77a,0x643,0x643,0x643,0x643,0x643,0x780,0x643,0x643,0x788,0x790,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x643,0x643,0x643,0x643,0x7a0,0x7a7,0x7af,0x798,
-0x7bf,0x7c7,0x7cf,0x7d6,0x7de,0x7e6,0x7ed,0x7b7,0x643,0x643,0x643,0x7f5,0x7fb,0x801,0x809,0x80e,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x815,0x38a,0x38a,0x38a,0x81d,0x38a,0x38a,0x38a,0x3d8,
-0x825,0x82d,0x834,0x38a,0x83c,0x643,0x643,0x646,0x643,0x643,0x643,0x643,0x643,0x643,0x843,0x849,
-0x859,0x851,0x38a,0x38a,0x861,0x5e6,0x38a,0x3b1,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x643,0x81c,
-0x3bf,0x38a,0x838,0x869,0x38a,0x871,0x80e,0x38a,0x38a,0x38a,0x38a,0x879,0x38a,0x38a,0x63b,0x3b0,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x643,0x643,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x838,0x643,0x562,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x880,0x38a,0x38a,0x885,0x54c,0x38a,0x38a,0x592,0x643,0x63a,0x38a,0x38a,0x88d,0x38a,0x38a,0x38a,
-0x895,0x89c,0x60c,0x8a4,0x38a,0x38a,0x8ab,0x8b3,0x38a,0x8ba,0x8c1,0x38a,0x4d5,0x8c6,0x38a,0x4ee,
-0x38a,0x8ce,0x8d6,0x4f0,0x38a,0x8da,0x4ef,0x8e2,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x8e9,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x8fd,0x8f1,0x8f5,0x489,0x489,0x489,0x489,0x489,
-0x489,0x489,0x489,0x489,0x489,0x489,0x489,0x489,0x489,0x905,0x489,0x489,0x489,0x489,0x90d,0x911,
-0x919,0x921,0x925,0x92d,0x489,0x489,0x489,0x931,0x939,0x37a,0x941,0x949,0x38a,0x38a,0x38a,0x951,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0xe28,0xe28,0xe68,0xea8,0xe28,0xe28,0xe28,0xe28,0xe28,0xe28,0xee0,0xf20,0xf60,0xf70,0xfb0,0xfbc,
-0xe28,0xe28,0xffc,0xe28,0xe28,0xe28,0x1034,0x1074,0x10b4,0x10f4,0x112c,0x116c,0x11ac,0x11e4,0x1224,0x1264,
-0xa40,0xa80,0xac0,0xafa,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xb23,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xb60,0x1a0,0x1a0,0xb95,0xbd5,0xc15,0xc55,0xc95,0xcd5,
+static const uint16_t ubidi_props_trieIndex[12100]={
+0x36f,0x377,0x37f,0x387,0x39f,0x3a7,0x3af,0x3b7,0x38f,0x397,0x38f,0x397,0x38f,0x397,0x38f,0x397,
+0x38f,0x397,0x38f,0x397,0x3bd,0x3c5,0x3cd,0x3d5,0x3dd,0x3e5,0x3e1,0x3e9,0x3f1,0x3f9,0x3f4,0x3fc,
+0x38f,0x397,0x38f,0x397,0x404,0x40c,0x38f,0x397,0x38f,0x397,0x38f,0x397,0x412,0x41a,0x422,0x42a,
+0x432,0x43a,0x442,0x44a,0x450,0x458,0x460,0x468,0x470,0x478,0x47e,0x486,0x48e,0x496,0x49e,0x4a6,
+0x4b2,0x4ae,0x4ba,0x4c2,0x424,0x4d2,0x4da,0x4ca,0x4e2,0x4e4,0x4ec,0x4f4,0x4fc,0x4fd,0x505,0x50d,
+0x515,0x4fd,0x51d,0x522,0x515,0x4fd,0x52a,0x532,0x4fc,0x53a,0x542,0x4f4,0x547,0x38f,0x54f,0x553,
+0x55b,0x55d,0x565,0x56d,0x4fc,0x575,0x57d,0x4f4,0x406,0x581,0x505,0x4f4,0x38f,0x38f,0x589,0x38f,
+0x38f,0x58f,0x597,0x38f,0x38f,0x59b,0x5a3,0x38f,0x5a7,0x5ae,0x38f,0x5b6,0x5be,0x5c5,0x546,0x38f,
+0x38f,0x5cd,0x5d5,0x5dd,0x5e5,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x5ed,0x38f,0x5f5,0x38f,0x38f,0x38f,
+0x5fd,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x605,0x38f,0x38f,0x38f,0x60d,0x60d,0x509,0x509,0x38f,0x613,0x61b,0x5f5,
+0x631,0x623,0x623,0x639,0x640,0x629,0x38f,0x38f,0x38f,0x648,0x650,0x38f,0x38f,0x38f,0x652,0x65a,
+0x662,0x38f,0x669,0x671,0x38f,0x679,0x38f,0x38f,0x539,0x681,0x547,0x689,0x406,0x691,0x38f,0x698,
+0x38f,0x69d,0x38f,0x38f,0x38f,0x38f,0x6a3,0x6ab,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x3dd,0x6b3,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x6bb,0x6c3,0x6c7,
+0x6df,0x6e5,0x6cf,0x6d7,0x6ed,0x6f5,0x6f9,0x5c8,0x701,0x709,0x711,0x38f,0x719,0x65a,0x65a,0x65a,
+0x729,0x731,0x739,0x741,0x746,0x74e,0x756,0x721,0x75e,0x766,0x38f,0x76c,0x773,0x65a,0x65a,0x65a,
+0x65a,0x573,0x779,0x65a,0x781,0x38f,0x38f,0x657,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,
+0x65a,0x65a,0x65a,0x65a,0x65a,0x789,0x65a,0x65a,0x65a,0x65a,0x65a,0x78f,0x65a,0x65a,0x797,0x79f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x65a,0x65a,0x65a,0x65a,0x7af,0x7b7,0x7bf,0x7a7,
+0x7cf,0x7d7,0x7df,0x7e6,0x7ed,0x7f5,0x7f9,0x7c7,0x65a,0x65a,0x65a,0x801,0x807,0x65a,0x80d,0x810,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x818,0x38f,0x38f,0x38f,0x820,0x38f,0x38f,0x38f,0x3dd,
+0x828,0x830,0x835,0x38f,0x83d,0x65a,0x65a,0x65d,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x844,0x84a,
+0x85a,0x852,0x38f,0x38f,0x862,0x5fd,0x38f,0x3b6,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x65a,0x81f,
+0x3c4,0x38f,0x839,0x86a,0x38f,0x872,0x87a,0x38f,0x38f,0x38f,0x38f,0x87e,0x38f,0x38f,0x652,0x3b5,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x65a,0x65a,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x839,0x65a,0x573,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x885,0x38f,0x38f,0x88a,0x55d,0x38f,0x38f,0x5a9,0x65a,0x651,0x38f,0x38f,0x892,0x38f,0x38f,0x38f,
+0x89a,0x8a1,0x623,0x8a9,0x38f,0x38f,0x57f,0x8b1,0x38f,0x8b9,0x8c0,0x38f,0x4e2,0x8c5,0x38f,0x4fb,
+0x38f,0x8cd,0x8d5,0x4fd,0x38f,0x8d9,0x4fc,0x8e1,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x8e8,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x8fc,0x8f0,0x8f4,0x48e,0x48e,0x48e,0x48e,0x48e,
+0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,0x904,0x48e,0x48e,0x48e,0x48e,0x90c,0x910,
+0x918,0x920,0x924,0x92c,0x48e,0x48e,0x48e,0x930,0x938,0x37f,0x940,0x948,0x38f,0x38f,0x38f,0x950,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0xe3c,0xe3c,0xe7c,0xebc,0xe3c,0xe3c,0xe3c,0xe3c,0xe3c,0xe3c,0xef4,0xf34,0xf74,0xf84,0xfc4,0xfd0,
+0xe3c,0xe3c,0x1010,0xe3c,0xe3c,0xe3c,0x1048,0x1088,0x10c8,0x1108,0x1140,0x1180,0x11c0,0x11f8,0x1238,0x1278,
+0xa40,0xa80,0xac0,0xaff,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xb37,0x1a0,0x1a0,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xb74,0x1a0,0x1a0,0xba9,0xbe9,0xc29,0xc69,0xca9,0xce9,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
-0xd55,0xd65,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
+0xd69,0xd79,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,
-0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd15,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x959,0x38a,0x643,0x643,0x961,0x5e6,0x38a,0x4e8,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x969,0x38a,0x38a,0x38a,0x970,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x978,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,
-0x980,0x984,0x41f,0x41f,0x41f,0x41f,0x994,0x98c,0x41f,0x99c,0x41f,0x41f,0x9a4,0x9aa,0x41f,0x41f,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,
-0x41f,0x41f,0x41f,0x9b2,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,
-0x4ef,0x8ad,0x9ba,0x9c1,0x401,0x9c4,0x38a,0x38a,0x4d5,0x9cc,0x38a,0x9d2,0x401,0x9d7,0x5f8,0x38a,
-0x38a,0x9df,0x38a,0x38a,0x38a,0x38a,0x81d,0x9e7,0x401,0x4f0,0x54b,0x9ee,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x8ad,0x9f6,0x38a,0x38a,0x9fa,0xa02,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0xa06,0xa0e,0x38a,
-0x38a,0xa16,0x54b,0x832,0x38a,0xa1e,0x38a,0x38a,0x5d6,0xa26,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0xa2a,0x38a,0x38a,0xa32,0xa38,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0xa3e,0x38a,0xa44,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0xa4a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x509,0xa52,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0xa59,0xa61,0xa67,0x38a,0x38a,0x643,0x643,0xa6f,0x38a,0x38a,0x38a,0x38a,0x38a,0x643,0x643,0x767,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0xa71,0x38a,0xa78,0x38a,0xa74,
-0x38a,0xa7b,0x38a,0xa83,0xa87,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x3d8,0xa8f,0x3d8,0xa96,0xa9d,0xaa5,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0xaad,0xab5,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0xabd,0x41f,0xac5,0xac5,0xacc,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,0x41f,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x489,0x489,0x489,0x489,0x489,0x489,0x489,0xad4,0x41f,0x41f,0x41f,
-0x41f,0x41f,0x41f,0x41f,0x41f,0x643,0xadc,0x643,0x643,0x646,0xae1,0xae5,0x843,0xaed,0x38a,0x38a,
-0xaf3,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x643,0x643,0x643,
-0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,
-0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x643,0x768,0xafb,0x643,0x643,0x643,
-0x646,0x643,0x643,0x830,0x38a,0xadc,0x643,0xb03,0x643,0xb0b,0x845,0x38a,0x38a,0xb1b,0xb23,0xb2b,
-0x38a,0x844,0x38a,0x5e6,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0xb13,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,0x38a,
-0x38a,0x38a,0x38a,0x38a,0xb13,0xb3b,0xb33,0xb33,0xb33,0xb3c,0xb3c,0xb3c,0xb3c,0x3d8,0x3d8,0x3d8,
-0x3d8,0x3d8,0x3d8,0x3d8,0xb44,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,
-0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,
-0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,
-0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,
-0xb3c,0xb3c,0xb3c,0xb3c,0xb3c,0x369,0x369,0x369,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
-0x12,8,7,8,9,7,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
-0x12,0x12,0x12,0x12,7,7,7,8,9,0xa,0xa,4,4,4,0xa,0xa,
-0x310a,0xf20a,0xa,3,6,3,6,6,2,2,2,2,2,2,2,2,
-2,2,6,0xa,0x500a,0xa,0xd00a,0xa,0xa,0,0,0,0,0,0,0,
+0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0x1a0,0xd29,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x958,0x38f,0x65a,0x65a,0x960,0x5fd,0x38f,0x4f5,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x968,0x38f,0x38f,0x38f,0x96f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x977,0x424,0x424,0x424,0x424,0x424,0x424,0x424,
+0x97f,0x983,0x424,0x424,0x424,0x424,0x993,0x98b,0x424,0x99b,0x424,0x424,0x9a3,0x9a9,0x424,0x424,
+0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x9b9,0x9b1,0x424,0x424,0x424,0x424,0x424,0x424,
+0x424,0x424,0x424,0x9c1,0x424,0x424,0x424,0x424,0x424,0x9c9,0x9d0,0x9d6,0x424,0x424,0x424,0x424,
+0x4fc,0x9de,0x9e5,0x9ec,0x406,0x9ef,0x38f,0x38f,0x4e2,0x9f6,0x38f,0x9fc,0x406,0xa01,0xa09,0x38f,
+0x38f,0xa0e,0x38f,0x38f,0x38f,0x38f,0x820,0xa16,0x406,0x581,0x55c,0xa1d,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x9de,0xa25,0x38f,0x38f,0xa2d,0xa35,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xa39,0xa41,0x38f,
+0x38f,0xa49,0x55c,0xa51,0x38f,0xa57,0x38f,0x38f,0x5ed,0xa5f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0xa64,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xa6c,
+0xa70,0xa78,0x38f,0xa7f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0xa86,0x38f,0x38f,0xa94,0xa8e,0x38f,0x38f,0x38f,0xa9c,0xaa4,0x38f,0xaa8,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x583,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xaae,0x38f,
+0xab4,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0xaba,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x516,0xac2,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0xac9,0xad1,0xad7,0x38f,0x38f,0x65a,0x65a,0xadf,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x65a,0x65a,0x833,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xae1,
+0x38f,0xae8,0x38f,0xae4,0x38f,0xaeb,0x38f,0xaf3,0xaf7,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x3dd,0xaff,0x3dd,0xb06,0xb0d,0xb15,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xb1d,0xb25,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x424,0x424,0x424,0x424,0x424,0x424,0xb2d,
+0x424,0xb35,0xb35,0xb3c,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,
+0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x8f4,0x48e,0x48e,0x424,
+0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,0x48e,
+0xb44,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x424,0x65a,0xb4c,0x65a,0x65a,0x65d,0xb51,0xb55,
+0x844,0xb5d,0x3b1,0x38f,0xb63,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x76a,0x38f,0x38f,0x38f,
+0x38f,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,
+0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0x65a,0xb6b,
+0xb73,0x65a,0x65a,0x65a,0x65d,0x65a,0x65a,0xb7b,0x38f,0xb4c,0x65a,0xb83,0x65a,0xb8b,0x846,0x38f,
+0x38f,0xb4c,0xb8f,0x65a,0xb97,0x65a,0xb9f,0xba7,0x65a,0x38f,0x38f,0x38f,0x846,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xbaf,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,
+0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0x38f,0xbaf,0xbbf,0xbb7,0xbb7,0xbb7,0xbc0,0xbc0,0xbc0,
+0xbc0,0x3dd,0x3dd,0x3dd,0x3dd,0x3dd,0x3dd,0x3dd,0xbc8,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,
+0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,
+0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,
+0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,
+0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0xbc0,0x36e,0x36e,0x36e,0x12,0x12,0x12,0x12,
+0x12,0x12,0x12,0x12,0x12,8,7,8,9,7,0x12,0x12,0x12,0x12,0x12,0x12,
+0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,7,7,7,8,9,0xa,0xa,4,
+4,4,0xa,0xa,0x310a,0xf20a,0xa,3,6,3,6,6,2,2,2,2,
+2,2,2,2,2,2,6,0xa,0x500a,0xa,0xd00a,0xa,0xa,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0x510a,0xa,0xd20a,0xa,0xa,0xa,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0x510a,0xa,0xd20a,0xa,0xa,0xa,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0x510a,0xa,0xd20a,0xa,0x12,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0x510a,0xa,0xd20a,0xa,0x12,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0x12,0x12,0x12,0x12,0x12,7,0x12,0x12,
-0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
-0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,6,0xa,4,4,4,4,0xa,0xa,
-0xa,0xa,0,0x900a,0xa,0xb2,0xa,0xa,4,4,2,2,0xa,0,0xa,0xa,
-0xa,2,0,0x900a,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,
+0,0,0,0,0,0,0,0,0,0,0,0,0x12,0x12,0x12,0x12,
+0x12,7,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
+0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,6,0xa,4,4,
+4,4,0xa,0xa,0xa,0xa,0,0x900a,0xa,0xb2,0xa,0xa,4,4,2,2,
+0xa,0,0xa,0xa,0xa,2,0,0x900a,0xa,0xa,0xa,0xa,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,
+0,0,0,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0xa,0xa,0,0,0,0,0,
-0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0,
+0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0,0,0,0,0xa,0xa,0,0,0,0,0,0,0,0,0xa,0,
-0,0,0,0,0xa,0xa,0,0xa,0,0,0,0,0,0,0,0,
+0xb1,0xb1,0xb1,0xb1,0,0,0,0,0xa,0xa,0,0,0,0,0,0,
+0,0,0xa,0,0,0,0,0,0xa,0xa,0,0xa,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xa,0,0,0,0,0,0,0,0,0,
+0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0,
-0,0xa,0xa,4,1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0xa,0,0,0xa,0xa,4,1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,1,0xb1,1,0xb1,0xb1,1,0xb1,0xb1,1,0xb1,1,1,1,1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,1,0xb1,1,0xb1,0xb1,1,0xb1,0xb1,1,0xb1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,5,5,5,5,5,5,0xa,0xa,0xd,4,4,0xd,
-6,0xd,0xa,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,
-0x8ad,0xd,0xd,0xd,0x4d,0xd,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x4d,0x2d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,5,5,5,5,5,5,5,5,5,5,4,5,
-5,0xd,0x4d,0x4d,0xb1,0x8d,0x8d,0x8d,0xd,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,
-0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+1,1,1,1,1,1,1,1,5,5,5,5,5,5,0xa,0xa,
+0xd,4,4,0xd,6,0xd,0xa,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xd,0x8ad,0xd,0xd,0xd,0x4d,0xd,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d,
+0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x2d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x8d,0x4d,0x4d,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,5,5,5,5,5,5,5,5,
+5,5,4,5,5,0xd,0x4d,0x4d,0xb1,0x8d,0x8d,0x8d,0xd,0x8d,0x8d,0x8d,
+0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,
+0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0x8d,
-0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,0x8d,0x8d,
-0xd,0x8d,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,5,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xd,0xd,0xb1,0xb1,0xa,0xb1,0xb1,0xb1,0xb1,0x8d,0x8d,2,2,2,2,
-2,2,2,2,2,2,0x4d,0x4d,0x4d,0xd,0xd,0x4d,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xad,0x8d,0xb1,0x4d,0x4d,
-0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,0x8d,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,
-0xd,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,
-0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,
-0x8d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x8d,0x8d,0x4d,0x4d,0x4d,0x8d,0x8d,0x4d,0x4d,
-0x4d,0x4d,0x4d,0x4d,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,1,1,1,1,1,1,1,1,1,1,0x41,0x41,
-0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,
-0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,1,1,0xa,0xa,0xa,0xa,0x21,1,
-1,1,1,1,0xb1,0xb1,0xb1,0xb1,1,0xb1,0xb1,0xb1,1,0xb1,0xb1,0xb1,
-0xb1,0xb1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,1,1,1,1,1,1,0xb1,0xb1,0xb1,0xb1,1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0x81,0x41,0x41,0x41,0x41,0x41,0x81,0x81,0x41,0x81,0x41,0x41,
-0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x81,0x41,1,1,1,0xb1,0xb1,0xb1,
-1,1,1,1,0xb1,0xb1,5,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x8d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x8d,0x4d,0x8d,
+0x4d,0x4d,0x8d,0x8d,0xd,0x8d,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,5,0xa,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xd,0xb1,0xb1,0xa,0xb1,0xb1,0xb1,0xb1,0x8d,0x8d,
+2,2,2,2,2,2,2,2,2,2,0x4d,0x4d,0x4d,0xd,0xd,0x4d,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xad,
+0x8d,0xb1,0x4d,0x4d,0x4d,0x8d,0x8d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,
+0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x8d,0x4d,0x8d,0x4d,0x4d,0x8d,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x8d,0x8d,
-0x8d,0xd,0x8d,0x4d,0x4d,0x8d,0x8d,0x4d,0x4d,0xd,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,
-0x4d,0x4d,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0xb1,0,0xb1,0,0,0,0,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0xb1,0,0,0,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0,
-0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0xb1,0,0,
+0xb1,0xb1,0xb1,0xd,0xd,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x8d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x4d,0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x8d,0x4d,0x8d,0x8d,0x4d,0x4d,0x4d,
+0x8d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,1,1,1,1,1,1,1,1,
+1,1,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,
+0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,
+0x41,0x41,0x41,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,1,1,0xa,0xa,
+0xa,0xa,0x21,1,1,0xb1,1,1,0xb1,0xb1,0xb1,0xb1,1,0xb1,0xb1,0xb1,
+1,0xb1,0xb1,0xb1,0xb1,0xb1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,0xb1,0xb1,
+0xb1,0xb1,1,0xb1,0xb1,0xb1,0xb1,0xb1,0x81,0x41,0x41,0x41,0x41,0x41,0x81,0x81,
+0x41,0x81,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x41,0x81,0x41,1,1,
+1,0xb1,0xb1,0xb1,1,1,1,1,0x4d,0xd,0x4d,0x4d,0x4d,0x4d,0xd,0x8d,
+0x4d,0x8d,0x8d,0xd,0xd,0xd,0xd,0xd,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,0xb1,0xb1,5,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x8d,0x8d,0x8d,0xd,0x8d,0x4d,0x4d,0x8d,0x8d,0x4d,0x4d,0xd,0x4d,0x4d,
+0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0xb1,0,0xb1,0,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0xb1,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,
0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,4,4,0,0,0,0,0,0,0,4,0,0,0,0,
-0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0xb1,0xb1,0,0,0,0,0xb1,0xb1,0,0,0xb1,0xb1,0xb1,0,0,
0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0xb1,0xb1,0,0,0,0xb1,0,0,0,0,0,0,
-0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0xb1,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,
0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,4,4,0,0,0,0,0,0,0,4,
+0,0,0xb1,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0xb1,0xb1,0,0,0,0,0xb1,0xb1,0,0,0xb1,
+0xb1,0xb1,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0xb1,0,0,
+0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,
+0xb1,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,
+0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,
+0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0xb1,0,0,0xb1,0,0xb1,0xb1,0xb1,0xb1,0,0,0,
0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0xb1,0,
@@ -355,79 +359,80 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,
0xa,0xa,0xa,0xa,0xa,4,0xa,0,0,0,0,0,0xb1,0,0,0,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
-0xb1,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0,0,
-0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
-0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,
+0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0xb1,
+0xb1,0xb1,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0xa0,
-0,0,0,0,0,0,0xa0,0,0,0,0,0,0xb1,0xb1,0,0,
+0xb1,0,0,0xa0,0,0,0,0,0,0,0xa0,0,0,0,0,0,
+0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,
+0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,4,0,0,0,0,
+0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,
+0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0xb1,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,4,
-0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,
-0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0xb1,0,0xb1,0,0xb1,0x310a,0xf20a,0x310a,0xf20a,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,
-0xb1,0,0xb1,0xb1,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0xb1,
+0,0xb1,0x310a,0xf20a,0x310a,0xf20a,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,
+0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,
+0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0,
+0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0,0,0,0,0,
+0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0xb1,0xb1,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0,
-0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,
+0,0,0,0,0xa,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0,0,0,0,0,0,0xa,0,0,0,0,0,0,0,
+0,0,0,0,9,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x310a,
+0xf20a,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,9,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0x310a,0xf20a,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,
-0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,4,0,0xb1,0,0,
+0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,
+0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0,0,0,0,0,0,4,0,0xb1,0,0,0x40,0x40,0x40,0x40,
0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0x40,0xb1,0x40,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x4a,
-0xa,0xa,0x2a,0xb1,0xb1,0xb1,0x12,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0,0,0,0,0,0,0,0,0,0xb1,0xb1,0x40,0x40,0x40,0x40,0x40,
-0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0x40,0x40,0x40,0x40,0xb1,0xb1,0xb1,0,0,0,0,0xb1,0xb1,0,0,0,
-0,0,0,0,0,0,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,
-0,0,0,0,0xa,0,0,0,0xa,0xa,0,0,0,0,0,0,
+0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0xb1,0x40,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,
+0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x4a,0xa,0xa,0x2a,0xb1,
+0xb1,0xb1,0x12,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
+0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0,0,0,
+0,0,0,0,0,0xb1,0xb1,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
+0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
+0xb1,0xb1,0xb1,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,
+0,0,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0,0,
+0xa,0,0,0,0xa,0xa,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0xb1,0xb1,0,0,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0,0xb1,0,0,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
+0xb1,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0xb1,0,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0,0xb1,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,
0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,
@@ -441,7 +446,7 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0xb1,0,0,0,0,0,0,
0xb1,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0xa,0,0xa,0xa,0xa,0,0,
0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0,0,0,0,
@@ -471,16 +476,16 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0x300a,0xf00a,0xa,0xa,0x300a,0xf00a,0x900a,0x900a,0x900a,0x100a,0x900a,0x900a,0x100a,0x100a,0x900a,0x900a,
0x900a,0x900a,0x900a,0x100a,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0xa,0xa,0x700a,0x700a,0x700a,0xb00a,
0xb00a,0xb00a,0xa,0xa,0xa,0x100a,3,4,0xa,0x900a,0x100a,0xa,0xa,0xa,0x100a,0x100a,
-0x100a,0x100a,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0x100a,0xa,0x100a,0xa,0xa,0xa,0xa,0x100a,
+0x100a,0x100a,0xa,0x900a,0x900a,0x900a,0x900a,0xa,0x900a,0xa,0x100a,0xa,0xa,0xa,0xa,0x100a,
0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0x100a,0xa,0x100a,
-0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x900a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,
-0x100a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,
+0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,0x900a,0x100a,0x900a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,
+0x900a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0x100a,0x100a,0xa,0x100a,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,
0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,
0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,
0x900a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0xa,0xa,0x900a,0x100a,
0x900a,0x900a,0x100a,0x900a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,
-0x100a,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0x900a,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0x300a,0xf00a,0x300a,0xf00a,0x900a,0xa,0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0x300a,0xf00a,
0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
@@ -489,28 +494,27 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
-2,2,2,2,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,
-0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0x100a,0xa,0xa,0x300a,0xf00a,0x310a,0xf20a,0xa,0x300a,0xf00a,0xa,0x500a,0x100a,0xd00a,0xa,0xa,
-0xa,0xa,0xa,0x100a,0x100a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0x100a,0x300a,0xf00a,0xa,
-0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0x100a,0xa,0x100a,0x100a,0x100a,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0x100a,0x900a,0x100a,0x100a,0x300a,0xf00a,0xa,0xa,0x310a,0xf20a,0xa,0xa,
-0xa,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x710a,0x320a,0xf10a,
-0xb20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0x100a,
-0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0xa,0xa,0xa,0xa,
+0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,
+0x310a,0xf20a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0xa,0xa,0x300a,
+0xf00a,0x310a,0xf20a,0xa,0x300a,0xf00a,0xa,0x500a,0x100a,0xd00a,0xa,0xa,0xa,0xa,0xa,0x100a,
+0x100a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0x900a,0x300a,0xf00a,0xa,0xa,0xa,0x300a,0xf00a,
+0x300a,0xf00a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0xa,0x100a,
+0x100a,0x100a,0xa,0xa,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0x100a,0x900a,0x100a,0x100a,0x300a,0xf00a,0xa,0xa,0x310a,0xf20a,0xa,0xa,0xa,0xa,0xa,0x310a,
+0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x710a,0x320a,0xf10a,0xb20a,0x310a,0xf20a,0x310a,
+0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0x900a,0x100a,0x100a,0x100a,0x100a,0x900a,0xa,0x100a,0x900a,
+0x300a,0xf00a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0x900a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x100a,0x100a,
0x300a,0xf00a,0xa,0xa,0xa,0x100a,0xa,0xa,0xa,0xa,0x100a,0x300a,0xf00a,0x300a,0xf00a,0xa,
0x300a,0xf00a,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x100a,0xa,0xa,0xa,0xa,0xa,0x100a,0x900a,
-0x900a,0x900a,0x100a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x100a,0xa,0xa,0xa,0xa,0x100a,
+0x900a,0x900a,0x100a,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0x900a,0xa,0xa,0xa,0xa,0x100a,
0xa,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,
0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0x100a,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0xa,
@@ -519,159 +523,183 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0xa,0xa,
0xa,0xa,0x100a,0x100a,0x100a,0x100a,0xa,0x100a,0x100a,0xa,0xa,0x100a,0x100a,0xa,0xa,0xa,
-0xa,0x300a,0xf00a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a,
-0x100a,0x100a,0x100a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,
-0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x300a,0xf00a,0x100a,0xa,0xa,0x300a,0xf00a,
-0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0x300a,0xf00a,0x100a,0x100a,0x300a,0xf00a,0x100a,0x100a,0x100a,
-0x100a,0x100a,0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0x100a,0x100a,0x100a,0x100a,
-0x100a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,
+0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,
+0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,
+0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x100a,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,
+0x300a,0xf00a,0xa,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,
+0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,
0x100a,0xa,0x900a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,
-0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0,
-0,0,0,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0xb1,0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0x300a,0xf00a,0xa,
-0x300a,0xf00a,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0x300a,0xf00a,0xa,0xa,0x300a,0xf00a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,
+0xa,0xa,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x900a,0,
+0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0xb1,
+0xb1,0xb1,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
+0xa,0xa,0x300a,0xf00a,0x300a,0xf00a,0xa,0xa,0xa,0x300a,0xf00a,0xa,0x300a,0xf00a,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x300a,0xf00a,0xa,0xa,
+0x300a,0xf00a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0,0,0,0,0xa,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
-0xb1,0xb1,0,0,0xa,0,0,0,0,0,0xa,0xa,0,0,0,0,
-0,0xa,0xa,0xa,9,0xa,0xa,0xa,0xa,0,0,0,0x310a,0xf20a,0x310a,0xf20a,
-0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,
-0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xa,
-0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0xb1,0xb1,0xb1,0xb1,0xa,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xa,0xa,0,0,0,0,0,0,0,0,0xa,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0xb1,0,0,0,0xb1,0,0,0,0,0xb1,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0xb1,0xb1,0,0xa,0xa,0xa,0xa,0,0,0,0,
-0,0,0,0,0,0,0,0,4,4,0,0,0,0,0,0,
-0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
-0x40,0x40,0x60,0,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
-0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,
-0xb1,0xb1,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,
-0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0xb1,0xb1,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,
-0,0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xb1,0,0xb1,0xb1,0xb1,0,0,0xb1,
-0xb1,0,0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0xb1,0,
-0,0,0,0,0,0,0,0,0,0xb1,0,0,0xb1,0,0,0,
-0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,1,1,1,1,1,1,1,1,1,3,1,1,
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,1,0xb1,1,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xa,0xa,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
-0x12,0x12,0x12,0x12,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xa,0xd,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,
-0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,6,0xa,6,0,0xa,6,0xa,0xa,0xa,0x310a,0xf20a,0x310a,
-0xf20a,0x310a,0xf20a,4,0xa,0xa,3,3,0x300a,0xf00a,0xa,0,0xa,4,4,0xa,
-0,0,0,0,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
-0xd,0xd,0xd,0xb2,0,0xa,0xa,4,4,4,0xa,0xa,0x310a,0xf20a,0xa,3,
-6,3,6,6,2,2,2,2,2,2,2,2,2,2,6,0xa,
-0x500a,0xa,0xd00a,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x510a,
-0xa,0xd20a,0xa,0x310a,0xf20a,0xa,0x310a,0xf20a,0xa,0xa,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,4,4,0xa,0xa,0xa,4,4,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0xaa,0xaa,0xaa,
-0xa,0xa,0x12,0x12,0,0xa,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0xa,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0,0,0,0,0xb1,2,2,2,2,2,2,2,2,2,2,2,
-2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0,0,0,0,0xa,0,0,0,0,0,0,0,
+0,0,0xb1,0xb1,0xb1,0xb1,0,0,0xa,0,0,0,0,0,0xa,0xa,
+0,0,0,0,0,0xa,0xa,0xa,9,0xa,0xa,0xa,0xa,0,0,0,
+0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,
+0x310a,0xf20a,0x310a,0xf20a,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,
+0,0xb1,0xb1,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
+0,0,0,0,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xa,
+0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xa,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xa,0xa,0,0,0,0,0,0,0,0,
+0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0xb1,0,
+0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0xb1,0xb1,0,0xa,0xa,0xa,0xa,
+0,0,0,0,0,0,0,0,0,0,0,0,4,4,0,0,
+0,0,0,0,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
+0x40,0x40,0x40,0x40,0x40,0x40,0x60,0,0xa,0xa,0xa,0xa,0,0,0,0,
+0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0xb1,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0,0,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0xb1,0,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0,
+0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
+0xb1,0xb1,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,0,
+0,0,0,0,0,0xb1,0,0,0xb1,0,0,0,0,0xb1,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+1,1,1,1,1,1,1,1,1,3,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0xa,
-1,0xb1,0xb1,0xb1,1,0xb1,0xb1,1,1,1,1,1,0xb1,0xb1,0xb1,0xb1,
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,1,1,1,1,0xb1,0xb1,0xb1,1,1,1,1,0xb1,
-0x41,0x81,1,1,0x81,0xb1,0xb1,1,1,1,1,0x41,0x41,0x41,0x41,0x81,
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-0x41,0x41,0x41,0x41,0x41,0x81,1,0x81,1,0x81,0x81,1,1,0x61,0x81,0x81,
-0x81,0x81,0x81,0x41,0x41,0x41,0x41,0x61,0x41,0x41,0x41,0x41,0x41,0x81,0x41,0x41,
-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,1,1,1,1,1,1,1,1,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
-0x41,0x81,0x41,0x81,0x81,0x81,0x41,0x41,0x41,0x81,0x41,0x41,0x81,0x41,0x81,0x81,
-0x41,0x81,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-1,0x81,0x81,0x81,0x81,0x41,0x41,1,1,1,1,1,1,1,1,1,
-1,1,1,1,1,1,1,1,5,5,5,5,5,5,5,5,
-5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
-5,5,5,5,5,5,5,1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,
-0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,1,0xb1,1,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xa,0xa,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xa,0xd,0xd,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+6,0xa,6,0,0xa,6,0xa,0xa,0xa,0x310a,0xf20a,0x310a,0xf20a,0x310a,0xf20a,4,
+0xa,0xa,3,3,0x300a,0xf00a,0xa,0,0xa,4,4,0xa,0,0,0,0,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xb2,
+0,0xa,0xa,4,4,4,0xa,0xa,0x310a,0xf20a,0xa,3,6,3,6,6,
+2,2,2,2,2,2,2,2,2,2,6,0xa,0x500a,0xa,0xd00a,0xa,
+0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0x510a,0xa,0xd20a,0xa,0x310a,
+0xf20a,0xa,0x310a,0xf20a,0xa,0xa,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0xa0,0,0,
-0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0,0,0,0,
+4,4,0xa,0xa,0xa,4,4,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,
+0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0xaa,0xaa,0xaa,0xa,0xa,0x12,0x12,
+0,0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,
-0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0xb1,0,0,0,0,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,
+0xb1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+2,2,2,2,2,2,2,2,2,2,2,2,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0,0xb1,0,0,0,0,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,
+0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,0xa,1,0xb1,0xb1,0xb1,
+1,0xb1,0xb1,1,1,1,1,1,0xb1,0xb1,0xb1,0xb1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,0xb1,0xb1,0xb1,1,1,1,1,0xb1,0x41,0x81,1,1,
+0x81,0xb1,0xb1,1,1,1,1,0x41,0x41,0x41,0x41,0x81,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,0x41,0x41,0x41,0x41,
+0x41,0x81,1,0x81,1,0x81,0x81,1,1,0x61,0x81,0x81,0x81,0x81,0x81,0x41,
+0x41,0x41,0x41,0x61,0x41,0x41,0x41,0x41,0x41,0x81,0x41,0x41,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0x41,0x81,0x41,0x81,
+0x81,0x81,0x41,0x41,0x41,0x81,0x41,0x41,0x81,0x41,0x81,0x81,0x41,0x81,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,0x81,0x81,0x81,
+0x81,0x41,0x41,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,0x4d,0x4d,0x8d,0x4d,0xb1,0xb1,0xb1,0xb1,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,5,5,5,5,5,5,5,5,5,5,0xd,0xd,
+0xd,0xd,0xd,0xd,0x6d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x4d,0x4d,5,5,5,5,5,5,5,5,5,5,5,5,
+5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,
+5,5,5,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,0x4d,0x4d,0x4d,0x8d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,0x4d,
+0x4d,0x4d,0x4d,0x4d,0x4d,0xd,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0x4d,0x4d,0x4d,0x8d,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,
+0xd,0xd,0xd,0xd,0xd,0xd,0xd,0xd,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,
-0,0,0,0,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
+0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0xb1,
+0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0,0,0xb1,0,0xb1,0,0,0,0,0,0,0,0,
-0,0,0,0xb1,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,
-0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0xb1,
-0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xa0,0,0,0,0,0,0,0,0,
+0,0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
-0xb1,0,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
+0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,
+0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0,0xb1,0,0xb1,0xb1,
+0,0,0,0,0,0,0xb1,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xb1,0xb1,0xb1,0,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0,
+0,0,0,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,
+0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,
+0,0xb1,0,0xb1,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0xb1,0,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0,0xb1,0,0,0,0,0,0,0,0,0,0,0xb1,0xb1,
+0xb1,0xb1,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xa0,0xa0,0xb1,0xb1,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0xb1,0xb1,0,
+0,0,0,0,0,0,0,0xb1,0,0,0,0,0,0,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0xb1,0xb1,0xb1,0,0,0,0,
+0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xa0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0xb1,0,0xb1,0xb1,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0xb1,0,0xb1,0xb1,0,0xb1,
+0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0xb1,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0xb1,0xb1,0,0,0,0xb1,0,0xb1,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0xb1,0,0,0,0,0,0,0,0,0,
@@ -719,20 +747,25 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0xa,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0xa,0xa,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0xa,
+0xa,0xa,0xa,0,0,0,0xa,0,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
+0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0,0,0xa,0xa,0xa,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0x12,0x12,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0,0,0,0,0,0,0,0,0xa,0,0,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0,0,0,0xa,0xa,0xa,0xa,
-0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0xa,0,0xb2,0xb2,0xb2,0xb2,
+0,0,0,0,0,0,0,0,0,0,0x12,0x12,0xb2,0xb2,0xb2,0xb2,
0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,
0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0xb2,0x12,0xb2,0x12,0x12,
0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12,
@@ -742,9 +775,10 @@ static const uint16_t ubidi_props_trieIndex[11572]={
0,0,0,0
};
-static const uint32_t ubidi_props_mirrors[26]={
-0x2000ab,0xbb,0x2a02215,0x1202243,0x2802298,0x2c022a6,0x30022a8,0x2e022a9,0x32022ab,0x6022cd,0x1e022f2,0x20022f3,0x22022f4,0x24022f6,0x26022f7,0x14022fa,
-0x16022fb,0x18022fc,0x1a022fd,0x1c022fe,0x8029b8,0x4029f5,0xa02ade,0xe02ae3,0xc02ae4,0x1002ae5
+static const uint32_t ubidi_props_mirrors[40]={
+0x2000ab,0xbb,0x4202215,0x4e0221f,0x3e02220,0x3a02221,0x3c02222,0x4c02224,0x2202243,0x1402245,0x120224c,0x4002298,0x44022a6,0x48022a8,0x46022a9,0x4a022ab,
+0x38022b8,0x10022cd,0x2e022f2,0x30022f3,0x32022f4,0x34022f6,0x36022f7,0x24022fa,0x26022fb,0x28022fc,0x2a022fd,0x2c022fe,0x20027dc,0xa0299b,0xc029a0,0x8029a3,
+0x16029b8,0x4029f5,0x1802ade,0x1c02ae3,0x1a02ae4,0x1e02ae5,0xe02aee,0x602bfe
};
static const uint8_t ubidi_props_jgArray[672]={
@@ -784,7 +818,7 @@ static const uint8_t ubidi_props_jgArray[672]={
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0x5d,0x5a,0x60,0x63,0x5e,0x5f,0x59,0x61,0x5b,0x5c,0x62,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
@@ -792,10 +826,46 @@ static const uint8_t ubidi_props_jgArray[672]={
0xb,0x55,0x1f,1,0x13,0,4,4,4,0x1f,0x2d,0x56,0x58,0x57,0,0
};
-static const uint8_t ubidi_props_jgArray2[48]={
+static const uint8_t ubidi_props_jgArray2[612]={
0x3a,0x3c,0x3c,0x40,0x40,0x3d,0,0x52,0,0x54,0x54,0,0,0x41,0x4f,0x53,
0x43,0x43,0x43,0x44,0x3e,0x50,0x45,0x46,0x4c,0x3b,0x3b,0x48,0x48,0x4b,0x49,0x49,
-0x49,0x4a,0,0,0x4d,0,0,0,0,0,0,0x47,0x3f,0x4e,0x51,0x42
+0x49,0x4a,0,0,0x4d,0,0,0,0,0,0,0x47,0x3f,0x4e,0x51,0x42,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0x65,0,0,0,0,0,0,0x65,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0x64,0,0,0x65,0,0x64,0,
+0x64,0,0,0x64
};
static const UBiDiProps ubidi_props_singleton={
@@ -806,16 +876,16 @@ static const UBiDiProps ubidi_props_singleton={
ubidi_props_jgArray2,
{
ubidi_props_trieIndex,
- ubidi_props_trieIndex+3496,
+ ubidi_props_trieIndex+3516,
NULL,
- 3496,
- 8076,
+ 3516,
+ 8584,
0x1a0,
- 0xe28,
+ 0xe3c,
0x0,
0x0,
0x110000,
- 0x2d30,
+ 0x2f40,
NULL, 0, FALSE, FALSE, 0, NULL
},
{ 2,2,0,0 }
diff --git a/intl/icu/source/common/ubidiimp.h b/intl/icu/source/common/ubidiimp.h
index a62d8b259e..a5d0727495 100644
--- a/intl/icu/source/common/ubidiimp.h
+++ b/intl/icu/source/common/ubidiimp.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -8,7 +8,7 @@
*
******************************************************************************
* file name: ubidiimp.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -254,8 +254,6 @@ struct UBiDi {
*/
const UBiDi * pParaBiDi;
- const UBiDiProps *bdp;
-
/* alias pointer to the current text */
const UChar *text;
diff --git a/intl/icu/source/common/ubidiln.c b/intl/icu/source/common/ubidiln.cpp
similarity index 99%
rename from intl/icu/source/common/ubidiln.c
rename to intl/icu/source/common/ubidiln.cpp
index 688ca4c31e..71c581fe1c 100644
--- a/intl/icu/source/common/ubidiln.c
+++ b/intl/icu/source/common/ubidiln.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -8,7 +8,7 @@
*
******************************************************************************
* file name: ubidiln.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/ubiditransform.c b/intl/icu/source/common/ubiditransform.cpp
similarity index 97%
rename from intl/icu/source/common/ubiditransform.c
rename to intl/icu/source/common/ubiditransform.cpp
index 5f83a2b5a6..e6d1d9c086 100644
--- a/intl/icu/source/common/ubiditransform.c
+++ b/intl/icu/source/common/ubiditransform.cpp
@@ -1,12 +1,12 @@
/*
******************************************************************************
*
-* Copyright (C) 2016 and later: Unicode, Inc. and others.
+* © 2016 and later: Unicode, Inc. and others.
* License & terms of use: http://www.unicode.org/copyright.html
*
******************************************************************************
* file name: ubiditransform.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -146,7 +146,7 @@ static UBool
action_reorder(UBiDiTransform *pTransform, UErrorCode *pErrorCode)
{
ubidi_writeReordered(pTransform->pBidi, pTransform->dest, pTransform->destSize,
- pTransform->reorderingOptions, pErrorCode);
+ static_cast(pTransform->reorderingOptions), pErrorCode);
*pTransform->pDestLength = pTransform->srcLength;
pTransform->reorderingOptions = UBIDI_REORDER_DEFAULT;
@@ -165,6 +165,7 @@ action_reorder(UBiDiTransform *pTransform, UErrorCode *pErrorCode)
static UBool
action_setInverse(UBiDiTransform *pTransform, UErrorCode *pErrorCode)
{
+ (void)pErrorCode;
ubidi_setInverse(pTransform->pBidi, TRUE);
ubidi_setReorderingMode(pTransform->pBidi, UBIDI_REORDER_INVERSE_LIKE_DIRECT);
return FALSE;
@@ -183,6 +184,7 @@ action_setInverse(UBiDiTransform *pTransform, UErrorCode *pErrorCode)
static UBool
action_setRunsOnly(UBiDiTransform *pTransform, UErrorCode *pErrorCode)
{
+ (void)pErrorCode;
ubidi_setReorderingMode(pTransform->pBidi, UBIDI_REORDER_RUNS_ONLY);
return FALSE;
}
@@ -391,9 +393,9 @@ resolveBaseDirection(const UChar *text, uint32_t length,
switch (*pInLevel) {
case UBIDI_DEFAULT_LTR:
case UBIDI_DEFAULT_RTL: {
- UBiDiLevel level = ubidi_getBaseDirection(text, length);
- *pInLevel = level != UBIDI_NEUTRAL ? level
- : *pInLevel == UBIDI_DEFAULT_RTL ? RTL : LTR;
+ UBiDiLevel level = static_cast(ubidi_getBaseDirection(text, length));
+ *pInLevel = static_cast(level != UBIDI_NEUTRAL) ? level
+ : *pInLevel == UBIDI_DEFAULT_RTL ? static_cast(RTL) : static_cast(LTR);
break;
}
default:
diff --git a/intl/icu/source/common/ubidiwrt.c b/intl/icu/source/common/ubidiwrt.cpp
similarity index 99%
rename from intl/icu/source/common/ubidiwrt.c
rename to intl/icu/source/common/ubidiwrt.cpp
index 1d0c36a5d2..a89099dad0 100644
--- a/intl/icu/source/common/ubidiwrt.c
+++ b/intl/icu/source/common/ubidiwrt.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
******************************************************************************
@@ -8,7 +8,7 @@
*
******************************************************************************
* file name: ubidiwrt.c
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
diff --git a/intl/icu/source/common/ubrk.cpp b/intl/icu/source/common/ubrk.cpp
index b02c966b10..f8bdf5a6b6 100644
--- a/intl/icu/source/common/ubrk.cpp
+++ b/intl/icu/source/common/ubrk.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
********************************************************************************
@@ -20,6 +20,7 @@
#include "unicode/rbbi.h"
#include "rbbirb.h"
#include "uassert.h"
+#include "cmemory.h"
U_NAMESPACE_USE
@@ -119,7 +120,28 @@ ubrk_openRules( const UChar *rules,
}
-
+U_CAPI UBreakIterator* U_EXPORT2
+ubrk_openBinaryRules(const uint8_t *binaryRules, int32_t rulesLength,
+ const UChar * text, int32_t textLength,
+ UErrorCode * status)
+{
+ if (U_FAILURE(*status)) {
+ return NULL;
+ }
+ if (rulesLength < 0) {
+ *status = U_ILLEGAL_ARGUMENT_ERROR;
+ return NULL;
+ }
+ LocalPointer lpRBBI(new RuleBasedBreakIterator(binaryRules, rulesLength, *status), *status);
+ if (U_FAILURE(*status)) {
+ return NULL;
+ }
+ UBreakIterator *uBI = reinterpret_cast(lpRBBI.orphan());
+ if (text != NULL) {
+ ubrk_setText(uBI, text, textLength, status);
+ }
+ return uBI;
+}
U_CAPI UBreakIterator * U_EXPORT2
@@ -288,7 +310,8 @@ ubrk_getLocaleByType(const UBreakIterator *bi,
}
-void ubrk_refreshUText(UBreakIterator *bi,
+U_CAPI void U_EXPORT2
+ubrk_refreshUText(UBreakIterator *bi,
UText *text,
UErrorCode *status)
{
@@ -296,6 +319,39 @@ void ubrk_refreshUText(UBreakIterator *bi,
bii->refreshInputText(text, *status);
}
+U_CAPI int32_t U_EXPORT2
+ubrk_getBinaryRules(UBreakIterator *bi,
+ uint8_t * binaryRules, int32_t rulesCapacity,
+ UErrorCode * status)
+{
+ if (U_FAILURE(*status)) {
+ return 0;
+ }
+ if ((binaryRules == NULL && rulesCapacity > 0) || rulesCapacity < 0) {
+ *status = U_ILLEGAL_ARGUMENT_ERROR;
+ return 0;
+ }
+ RuleBasedBreakIterator* rbbi;
+ if ((rbbi = dynamic_cast(reinterpret_cast(bi))) == NULL) {
+ *status = U_ILLEGAL_ARGUMENT_ERROR;
+ return 0;
+ }
+ uint32_t rulesLength;
+ const uint8_t * returnedRules = rbbi->getBinaryRules(rulesLength);
+ if (rulesLength > INT32_MAX) {
+ *status = U_INDEX_OUTOFBOUNDS_ERROR;
+ return 0;
+ }
+ if (binaryRules != NULL) { // if not preflighting
+ // Here we know rulesLength <= INT32_MAX and rulesCapacity >= 0, can cast safely
+ if ((int32_t)rulesLength > rulesCapacity) {
+ *status = U_BUFFER_OVERFLOW_ERROR;
+ } else {
+ uprv_memcpy(binaryRules, returnedRules, rulesLength);
+ }
+ }
+ return (int32_t)rulesLength;
+}
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
diff --git a/intl/icu/source/common/ubrkimpl.h b/intl/icu/source/common/ubrkimpl.h
index 36ca668859..8197f66339 100644
--- a/intl/icu/source/common/ubrkimpl.h
+++ b/intl/icu/source/common/ubrkimpl.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
diff --git a/intl/icu/source/common/ucase.cpp b/intl/icu/source/common/ucase.cpp
index 97ded9ee2d..8414c527d4 100644
--- a/intl/icu/source/common/ucase.cpp
+++ b/intl/icu/source/common/ucase.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: ucase.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -46,13 +46,6 @@ struct UCaseProps {
#define INCLUDED_FROM_UCASE_CPP
#include "ucase_props_data.h"
-/* UCaseProps singleton ----------------------------------------------------- */
-
-U_CAPI const UCaseProps * U_EXPORT2
-ucase_getSingleton() {
- return &ucase_props_singleton;
-}
-
/* set of property starts for UnicodeSet ------------------------------------ */
static UBool U_CALLCONV
@@ -64,13 +57,13 @@ _enumPropertyStartsRange(const void *context, UChar32 start, UChar32 /*end*/, ui
}
U_CFUNC void U_EXPORT2
-ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *pErrorCode) {
+ucase_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) {
if(U_FAILURE(*pErrorCode)) {
return;
}
/* add the start code point of each same-value range of the trie */
- utrie2_enum(&csp->trie, NULL, _enumPropertyStartsRange, sa);
+ utrie2_enum(&ucase_props_singleton.trie, NULL, _enumPropertyStartsRange, sa);
/* add code points with hardcoded properties, plus the ones following them */
@@ -84,9 +77,12 @@ ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *
/* data access primitives --------------------------------------------------- */
-#define GET_EXCEPTIONS(csp, props) ((csp)->exceptions+((props)>>UCASE_EXC_SHIFT))
+U_CFUNC const UTrie2 * U_EXPORT2
+ucase_getTrie() {
+ return &ucase_props_singleton.trie;
+}
-#define PROPS_HAS_EXCEPTION(props) ((props)&UCASE_EXCEPTION)
+#define GET_EXCEPTIONS(csp, props) ((csp)->exceptions+((props)>>UCASE_EXC_SHIFT))
/* number of bits in an 8-bit integer value */
static const uint8_t flagsOffset[256]={
@@ -133,15 +129,20 @@ static const uint8_t flagsOffset[256]={
/* simple case mappings ----------------------------------------------------- */
U_CAPI UChar32 U_EXPORT2
-ucase_tolower(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
- if(UCASE_GET_TYPE(props)>=UCASE_UPPER) {
+ucase_tolower(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
+ if(UCASE_IS_UPPER_OR_TITLE(props)) {
c+=UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
uint16_t excWord=*pe++;
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe, c);
}
@@ -150,15 +151,20 @@ ucase_tolower(const UCaseProps *csp, UChar32 c) {
}
U_CAPI UChar32 U_EXPORT2
-ucase_toupper(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
+ucase_toupper(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
if(UCASE_GET_TYPE(props)==UCASE_LOWER) {
c+=UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
uint16_t excWord=*pe++;
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) {
GET_SLOT_VALUE(excWord, UCASE_EXC_UPPER, pe, c);
}
@@ -167,15 +173,20 @@ ucase_toupper(const UCaseProps *csp, UChar32 c) {
}
U_CAPI UChar32 U_EXPORT2
-ucase_totitle(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
+ucase_totitle(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
if(UCASE_GET_TYPE(props)==UCASE_LOWER) {
c+=UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
uint16_t excWord=*pe++;
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
int32_t idx;
if(HAS_SLOT(excWord, UCASE_EXC_TITLE)) {
idx=UCASE_EXC_TITLE;
@@ -198,7 +209,7 @@ static const UChar iDotTilde[3] = { 0x69, 0x307, 0x303 };
U_CFUNC void U_EXPORT2
-ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa) {
+ucase_addCaseClosure(UChar32 c, const USetAdder *sa) {
uint16_t props;
/*
@@ -229,8 +240,8 @@ ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa) {
break;
}
- props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
+ props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
if(UCASE_GET_TYPE(props)!=UCASE_NONE) {
/* add the one simple case mapping, no matter what type it is */
int32_t delta=UCASE_GET_DELTA(props);
@@ -243,7 +254,7 @@ ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa) {
* c has exceptions, so there may be multiple simple and/or
* full case mappings. Add them all.
*/
- const uint16_t *pe0, *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe0, *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
const UChar *closure;
uint16_t excWord=*pe++;
int32_t idx, closureLength, fullLength, length;
@@ -258,6 +269,12 @@ ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa) {
sa->add(sa->set, c);
}
}
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA)) {
+ pe=pe0;
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+ sa->add(sa->set, (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta);
+ }
/* get the closure string pointer & length */
if(HAS_SLOT(excWord, UCASE_EXC_CLOSURE)) {
@@ -338,10 +355,10 @@ strcmpMax(const UChar *s, int32_t length, const UChar *t, int32_t max) {
}
U_CFUNC UBool U_EXPORT2
-ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length, const USetAdder *sa) {
+ucase_addStringCaseClosure(const UChar *s, int32_t length, const USetAdder *sa) {
int32_t i, start, limit, result, unfoldRows, unfoldRowWidth, unfoldStringWidth;
- if(csp->unfold==NULL || s==NULL) {
+ if(ucase_props_singleton.unfold==NULL || s==NULL) {
return FALSE; /* no reverse case folding data, or no string */
}
if(length<=1) {
@@ -355,7 +372,7 @@ ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length
return FALSE;
}
- const uint16_t *unfold=csp->unfold;
+ const uint16_t *unfold=ucase_props_singleton.unfold;
unfoldRows=unfold[UCASE_UNFOLD_ROWS];
unfoldRowWidth=unfold[UCASE_UNFOLD_ROW_WIDTH];
unfoldStringWidth=unfold[UCASE_UNFOLD_STRING_WIDTH];
@@ -381,7 +398,7 @@ ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length
for(i=unfoldStringWidth; iadd(sa->set, c);
- ucase_addCaseClosure(csp, c, sa);
+ ucase_addCaseClosure(c, sa);
}
return TRUE;
} else if(result<0) {
@@ -426,43 +443,180 @@ FullCaseFoldingIterator::next(UnicodeString &full) {
return c;
}
+namespace LatinCase {
+
+const int8_t TO_LOWER_NORMAL[LIMIT] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, EXC, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 0, 32, 32, 32, 32, 32, 32, 32, EXC,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ EXC, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1,
+
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, EXC, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, -121, 1, 0, 1, 0, 1, 0, EXC
+};
+
+const int8_t TO_LOWER_TR_LT[LIMIT] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 32, 32, 32, 32, 32, 32, 32, 32, EXC, EXC, 32, 32, 32, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, EXC, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, EXC, EXC, 32, 32,
+ 32, 32, 32, 32, 32, 32, 32, 0, 32, 32, 32, 32, 32, 32, 32, EXC,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, EXC, 0, 1, 0, 1, 0, EXC, 0,
+ EXC, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1,
+
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, EXC, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,
+ 1, 0, 1, 0, 1, 0, 1, 0, -121, 1, 0, 1, 0, 1, 0, EXC
+};
+
+const int8_t TO_UPPER_NORMAL[LIMIT] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32,
+ -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, EXC, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, EXC,
+ -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32,
+ -32, -32, -32, -32, -32, -32, -32, 0, -32, -32, -32, -32, -32, -32, -32, 121,
+
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, EXC, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, 0,
+
+ -1, 0, -1, 0, -1, 0, -1, 0, -1, EXC, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, EXC
+};
+
+const int8_t TO_UPPER_TR[LIMIT] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, -32, -32, -32, -32, -32, -32, -32, -32, EXC, -32, -32, -32, -32, -32, -32,
+ -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, EXC, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, EXC,
+ -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32,
+ -32, -32, -32, -32, -32, -32, -32, 0, -32, -32, -32, -32, -32, -32, -32, 121,
+
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, EXC, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, 0,
+
+ -1, 0, -1, 0, -1, 0, -1, 0, -1, EXC, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1,
+ 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, EXC
+};
+
+} // namespace LatinCase
+
U_NAMESPACE_END
/** @return UCASE_NONE, UCASE_LOWER, UCASE_UPPER, UCASE_TITLE */
U_CAPI int32_t U_EXPORT2
-ucase_getType(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
+ucase_getType(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
return UCASE_GET_TYPE(props);
}
/** @return same as ucase_getType() and set bit 2 if c is case-ignorable */
U_CAPI int32_t U_EXPORT2
-ucase_getTypeOrIgnorable(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
+ucase_getTypeOrIgnorable(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
return UCASE_GET_TYPE_AND_IGNORABLE(props);
}
/** @return UCASE_NO_DOT, UCASE_SOFT_DOTTED, UCASE_ABOVE, UCASE_OTHER_ACCENT */
static inline int32_t
-getDotType(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
+getDotType(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
return props&UCASE_DOT_MASK;
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
return (*pe>>UCASE_EXC_DOT_SHIFT)&UCASE_DOT_MASK;
}
}
U_CAPI UBool U_EXPORT2
-ucase_isSoftDotted(const UCaseProps *csp, UChar32 c) {
- return (UBool)(getDotType(csp, c)==UCASE_SOFT_DOTTED);
+ucase_isSoftDotted(UChar32 c) {
+ return (UBool)(getDotType(c)==UCASE_SOFT_DOTTED);
}
U_CAPI UBool U_EXPORT2
-ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- return (UBool)((props&UCASE_SENSITIVE)!=0);
+ucase_isCaseSensitive(UChar32 c) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
+ return (UBool)((props&UCASE_SENSITIVE)!=0);
+ } else {
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
+ return (UBool)((*pe&UCASE_EXC_SENSITIVE)!=0);
+ }
}
/* string casing ------------------------------------------------------------ */
@@ -545,12 +699,10 @@ ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c) {
* zero or more case-ignorable characters.
*/
-#define is_a(c) ((c)=='a' || (c)=='A')
#define is_d(c) ((c)=='d' || (c)=='D')
#define is_e(c) ((c)=='e' || (c)=='E')
#define is_i(c) ((c)=='i' || (c)=='I')
#define is_l(c) ((c)=='l' || (c)=='L')
-#define is_n(c) ((c)=='n' || (c)=='N')
#define is_r(c) ((c)=='r' || (c)=='R')
#define is_t(c) ((c)=='t' || (c)=='T')
#define is_u(c) ((c)=='u' || (c)=='U')
@@ -565,16 +717,7 @@ ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c) {
* Accepts both 2- and 3-letter codes and accepts case variants.
*/
U_CFUNC int32_t
-ucase_getCaseLocale(const char *locale, int32_t *locCache) {
- int32_t result;
- char c;
-
- if(locCache!=NULL && (result=*locCache)!=UCASE_LOC_UNKNOWN) {
- return result;
- }
-
- result=UCASE_LOC_ROOT;
-
+ucase_getCaseLocale(const char *locale) {
/*
* This function used to use uloc_getLanguage(), but the current code
* removes the dependency of this low-level code on uloc implementation code
@@ -584,44 +727,12 @@ ucase_getCaseLocale(const char *locale, int32_t *locCache) {
* Because this code does not want to depend on uloc, the caller must
* pass in a non-NULL locale, i.e., may need to call uloc_getDefault().
*/
- c=*locale++;
- if(is_t(c)) {
- /* tr or tur? */
- c=*locale++;
- if(is_u(c)) {
- c=*locale++;
- }
- if(is_r(c)) {
- c=*locale;
- if(is_sep(c)) {
- result=UCASE_LOC_TURKISH;
- }
- }
- } else if(is_a(c)) {
- /* az or aze? */
- c=*locale++;
- if(is_z(c)) {
- c=*locale++;
- if(is_e(c)) {
- c=*locale;
- }
- if(is_sep(c)) {
- result=UCASE_LOC_TURKISH;
- }
- }
- } else if(is_l(c)) {
- /* lt or lit? */
- c=*locale++;
- if(is_i(c)) {
- c=*locale++;
- }
- if(is_t(c)) {
- c=*locale;
- if(is_sep(c)) {
- result=UCASE_LOC_LITHUANIAN;
- }
- }
- } else if(is_e(c)) {
+ char c=*locale++;
+ // Fastpath for English "en" which is often used for default (=root locale) case mappings,
+ // and for Chinese "zh": Very common but no special case mapping behavior.
+ // Then check lowercase vs. uppercase to reduce the number of comparisons
+ // for other locales without special behavior.
+ if(c=='e') {
/* el or ell? */
c=*locale++;
if(is_l(c)) {
@@ -630,27 +741,135 @@ ucase_getCaseLocale(const char *locale, int32_t *locCache) {
c=*locale;
}
if(is_sep(c)) {
- result=UCASE_LOC_GREEK;
+ return UCASE_LOC_GREEK;
}
}
- } else if(is_n(c)) {
- /* nl or nld? */
- c=*locale++;
- if(is_l(c)) {
+ // en, es, ... -> root
+ } else if(c=='z') {
+ return UCASE_LOC_ROOT;
+#if U_CHARSET_FAMILY==U_ASCII_FAMILY
+ } else if(c>='a') { // ASCII a-z = 0x61..0x7a, after A-Z
+#elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY
+ } else if(c<='z') { // EBCDIC a-z = 0x81..0xa9 with two gaps, before A-Z
+#else
+# error Unknown charset family!
+#endif
+ // lowercase c
+ if(c=='t') {
+ /* tr or tur? */
c=*locale++;
- if(is_d(c)) {
- c=*locale;
+ if(is_u(c)) {
+ c=*locale++;
}
- if(is_sep(c)) {
- result=UCASE_LOC_DUTCH;
+ if(is_r(c)) {
+ c=*locale;
+ if(is_sep(c)) {
+ return UCASE_LOC_TURKISH;
+ }
+ }
+ } else if(c=='a') {
+ /* az or aze? */
+ c=*locale++;
+ if(is_z(c)) {
+ c=*locale++;
+ if(is_e(c)) {
+ c=*locale;
+ }
+ if(is_sep(c)) {
+ return UCASE_LOC_TURKISH;
+ }
+ }
+ } else if(c=='l') {
+ /* lt or lit? */
+ c=*locale++;
+ if(is_i(c)) {
+ c=*locale++;
+ }
+ if(is_t(c)) {
+ c=*locale;
+ if(is_sep(c)) {
+ return UCASE_LOC_LITHUANIAN;
+ }
+ }
+ } else if(c=='n') {
+ /* nl or nld? */
+ c=*locale++;
+ if(is_l(c)) {
+ c=*locale++;
+ if(is_d(c)) {
+ c=*locale;
+ }
+ if(is_sep(c)) {
+ return UCASE_LOC_DUTCH;
+ }
+ }
+ }
+ } else {
+ // uppercase c
+ // Same code as for lowercase c but also check for 'E'.
+ if(c=='T') {
+ /* tr or tur? */
+ c=*locale++;
+ if(is_u(c)) {
+ c=*locale++;
+ }
+ if(is_r(c)) {
+ c=*locale;
+ if(is_sep(c)) {
+ return UCASE_LOC_TURKISH;
+ }
+ }
+ } else if(c=='A') {
+ /* az or aze? */
+ c=*locale++;
+ if(is_z(c)) {
+ c=*locale++;
+ if(is_e(c)) {
+ c=*locale;
+ }
+ if(is_sep(c)) {
+ return UCASE_LOC_TURKISH;
+ }
+ }
+ } else if(c=='L') {
+ /* lt or lit? */
+ c=*locale++;
+ if(is_i(c)) {
+ c=*locale++;
+ }
+ if(is_t(c)) {
+ c=*locale;
+ if(is_sep(c)) {
+ return UCASE_LOC_LITHUANIAN;
+ }
+ }
+ } else if(c=='E') {
+ /* el or ell? */
+ c=*locale++;
+ if(is_l(c)) {
+ c=*locale++;
+ if(is_l(c)) {
+ c=*locale;
+ }
+ if(is_sep(c)) {
+ return UCASE_LOC_GREEK;
+ }
+ }
+ } else if(c=='N') {
+ /* nl or nld? */
+ c=*locale++;
+ if(is_l(c)) {
+ c=*locale++;
+ if(is_d(c)) {
+ c=*locale;
+ }
+ if(is_sep(c)) {
+ return UCASE_LOC_DUTCH;
+ }
}
}
}
-
- if(locCache!=NULL) {
- *locCache=result;
- }
- return result;
+ return UCASE_LOC_ROOT;
}
/*
@@ -662,7 +881,7 @@ ucase_getCaseLocale(const char *locale, int32_t *locCache) {
* it is also cased or not.
*/
static UBool
-isFollowedByCasedLetter(const UCaseProps *csp, UCaseContextIterator *iter, void *context, int8_t dir) {
+isFollowedByCasedLetter(UCaseContextIterator *iter, void *context, int8_t dir) {
UChar32 c;
if(iter==NULL) {
@@ -670,7 +889,7 @@ isFollowedByCasedLetter(const UCaseProps *csp, UCaseContextIterator *iter, void
}
for(/* dir!=0 sets direction */; (c=iter(context, dir))>=0; dir=0) {
- int32_t type=ucase_getTypeOrIgnorable(csp, c);
+ int32_t type=ucase_getTypeOrIgnorable(c);
if(type&4) {
/* case-ignorable, continue with the loop */
} else if(type!=UCASE_NONE) {
@@ -685,7 +904,7 @@ isFollowedByCasedLetter(const UCaseProps *csp, UCaseContextIterator *iter, void
/* Is preceded by Soft_Dotted character with no intervening cc=230 ? */
static UBool
-isPrecededBySoftDotted(const UCaseProps *csp, UCaseContextIterator *iter, void *context) {
+isPrecededBySoftDotted(UCaseContextIterator *iter, void *context) {
UChar32 c;
int32_t dotType;
int8_t dir;
@@ -695,7 +914,7 @@ isPrecededBySoftDotted(const UCaseProps *csp, UCaseContextIterator *iter, void *
}
for(dir=-1; (c=iter(context, dir))>=0; dir=0) {
- dotType=getDotType(csp, c);
+ dotType=getDotType(c);
if(dotType==UCASE_SOFT_DOTTED) {
return TRUE; /* preceded by TYPE_i */
} else if(dotType!=UCASE_OTHER_ACCENT) {
@@ -742,7 +961,7 @@ isPrecededBySoftDotted(const UCaseProps *csp, UCaseContextIterator *iter, void *
/* Is preceded by base character 'I' with no intervening cc=230 ? */
static UBool
-isPrecededBy_I(const UCaseProps *csp, UCaseContextIterator *iter, void *context) {
+isPrecededBy_I(UCaseContextIterator *iter, void *context) {
UChar32 c;
int32_t dotType;
int8_t dir;
@@ -755,7 +974,7 @@ isPrecededBy_I(const UCaseProps *csp, UCaseContextIterator *iter, void *context)
if(c==0x49) {
return TRUE; /* preceded by I */
}
- dotType=getDotType(csp, c);
+ dotType=getDotType(c);
if(dotType!=UCASE_OTHER_ACCENT) {
return FALSE; /* preceded by different base character (not I), or intervening cc==230 */
}
@@ -766,7 +985,7 @@ isPrecededBy_I(const UCaseProps *csp, UCaseContextIterator *iter, void *context)
/* Is followed by one or more cc==230 ? */
static UBool
-isFollowedByMoreAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *context) {
+isFollowedByMoreAbove(UCaseContextIterator *iter, void *context) {
UChar32 c;
int32_t dotType;
int8_t dir;
@@ -776,7 +995,7 @@ isFollowedByMoreAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *c
}
for(dir=1; (c=iter(context, dir))>=0; dir=0) {
- dotType=getDotType(csp, c);
+ dotType=getDotType(c);
if(dotType==UCASE_ABOVE) {
return TRUE; /* at least one cc==230 following */
} else if(dotType!=UCASE_OTHER_ACCENT) {
@@ -789,7 +1008,7 @@ isFollowedByMoreAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *c
/* Is followed by a dot above (without cc==230 in between) ? */
static UBool
-isFollowedByDotAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *context) {
+isFollowedByDotAbove(UCaseContextIterator *iter, void *context) {
UChar32 c;
int32_t dotType;
int8_t dir;
@@ -802,7 +1021,7 @@ isFollowedByDotAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *co
if(c==0x307) {
return TRUE;
}
- dotType=getDotType(csp, c);
+ dotType=getDotType(c);
if(dotType!=UCASE_OTHER_ACCENT) {
return FALSE; /* next base character or cc==230 in between */
}
@@ -812,20 +1031,20 @@ isFollowedByDotAbove(const UCaseProps *csp, UCaseContextIterator *iter, void *co
}
U_CAPI int32_t U_EXPORT2
-ucase_toFullLower(const UCaseProps *csp, UChar32 c,
+ucase_toFullLower(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache) {
+ int32_t loc) {
// The sign of the result has meaning, input must be non-negative so that it can be returned as is.
U_ASSERT(c >= 0);
UChar32 result=c;
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
- if(UCASE_GET_TYPE(props)>=UCASE_UPPER) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
+ if(UCASE_IS_UPPER_OR_TITLE(props)) {
result=c+UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2;
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props), *pe2;
uint16_t excWord=*pe++;
int32_t full;
@@ -833,7 +1052,6 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
if(excWord&UCASE_EXC_CONDITIONAL_SPECIAL) {
/* use hardcoded conditions and mappings */
- int32_t loc=ucase_getCaseLocale(locale, locCache);
/*
* Test for conditional mappings first
@@ -844,7 +1062,7 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
if( loc==UCASE_LOC_LITHUANIAN &&
/* base characters, find accents above */
(((c==0x49 || c==0x4a || c==0x12e) &&
- isFollowedByMoreAbove(csp, iter, context)) ||
+ isFollowedByMoreAbove(iter, context)) ||
/* precomposed with accent above, no need to find one */
(c==0xcc || c==0xcd || c==0x128))
) {
@@ -896,7 +1114,7 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
0130; 0069; 0130; 0130; az # LATIN CAPITAL LETTER I WITH DOT ABOVE
*/
return 0x69;
- } else if(loc==UCASE_LOC_TURKISH && c==0x307 && isPrecededBy_I(csp, iter, context)) {
+ } else if(loc==UCASE_LOC_TURKISH && c==0x307 && isPrecededBy_I(iter, context)) {
/*
# When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i.
# This matches the behavior of the canonically equivalent I-dot_above
@@ -904,8 +1122,9 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE
0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE
*/
+ *pString=nullptr;
return 0; /* remove the dot (continue without output) */
- } else if(loc==UCASE_LOC_TURKISH && c==0x49 && !isFollowedByDotAbove(csp, iter, context)) {
+ } else if(loc==UCASE_LOC_TURKISH && c==0x49 && !isFollowedByDotAbove(iter, context)) {
/*
# When lowercasing, unless an I is before a dot_above, it turns into a dotless i.
@@ -922,8 +1141,8 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
*pString=iDot;
return 2;
} else if( c==0x3a3 &&
- !isFollowedByCasedLetter(csp, iter, context, 1) &&
- isFollowedByCasedLetter(csp, iter, context, -1) /* -1=preceded */
+ !isFollowedByCasedLetter(iter, context, 1) &&
+ isFollowedByCasedLetter(iter, context, -1) /* -1=preceded */
) {
/* greek capital sigma maps depending on surrounding cased letters (see SpecialCasing.txt) */
/*
@@ -947,6 +1166,11 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
}
}
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
GET_SLOT_VALUE(excWord, UCASE_EXC_LOWER, pe2, result);
}
@@ -957,21 +1181,21 @@ ucase_toFullLower(const UCaseProps *csp, UChar32 c,
/* internal */
static int32_t
-toUpperOrTitle(const UCaseProps *csp, UChar32 c,
+toUpperOrTitle(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache,
+ int32_t loc,
UBool upperNotTitle) {
// The sign of the result has meaning, input must be non-negative so that it can be returned as is.
U_ASSERT(c >= 0);
UChar32 result=c;
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
if(UCASE_GET_TYPE(props)==UCASE_LOWER) {
result=c+UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2;
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props), *pe2;
uint16_t excWord=*pe++;
int32_t full, idx;
@@ -979,8 +1203,6 @@ toUpperOrTitle(const UCaseProps *csp, UChar32 c,
if(excWord&UCASE_EXC_CONDITIONAL_SPECIAL) {
/* use hardcoded conditions and mappings */
- int32_t loc=ucase_getCaseLocale(locale, locCache);
-
if(loc==UCASE_LOC_TURKISH && c==0x69) {
/*
# Turkish and Azeri
@@ -994,7 +1216,7 @@ toUpperOrTitle(const UCaseProps *csp, UChar32 c,
0069; 0069; 0130; 0130; az; # LATIN SMALL LETTER I
*/
return 0x130;
- } else if(loc==UCASE_LOC_LITHUANIAN && c==0x307 && isPrecededBySoftDotted(csp, iter, context)) {
+ } else if(loc==UCASE_LOC_LITHUANIAN && c==0x307 && isPrecededBySoftDotted(iter, context)) {
/*
# Lithuanian
@@ -1004,6 +1226,7 @@ toUpperOrTitle(const UCaseProps *csp, UChar32 c,
0307; 0307; ; ; lt After_Soft_Dotted; # COMBINING DOT ABOVE
*/
+ *pString=nullptr;
return 0; /* remove the dot (continue without output) */
} else {
/* no known conditional special case mapping, use a normal mapping */
@@ -1037,6 +1260,11 @@ toUpperOrTitle(const UCaseProps *csp, UChar32 c,
}
}
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_GET_TYPE(props)==UCASE_LOWER) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(!upperNotTitle && HAS_SLOT(excWord, UCASE_EXC_TITLE)) {
idx=UCASE_EXC_TITLE;
} else if(HAS_SLOT(excWord, UCASE_EXC_UPPER)) {
@@ -1052,19 +1280,19 @@ toUpperOrTitle(const UCaseProps *csp, UChar32 c,
}
U_CAPI int32_t U_EXPORT2
-ucase_toFullUpper(const UCaseProps *csp, UChar32 c,
+ucase_toFullUpper(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache) {
- return toUpperOrTitle(csp, c, iter, context, pString, locale, locCache, TRUE);
+ int32_t caseLocale) {
+ return toUpperOrTitle(c, iter, context, pString, caseLocale, TRUE);
}
U_CAPI int32_t U_EXPORT2
-ucase_toFullTitle(const UCaseProps *csp, UChar32 c,
+ucase_toFullTitle(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache) {
- return toUpperOrTitle(csp, c, iter, context, pString, locale, locCache, FALSE);
+ int32_t caseLocale) {
+ return toUpperOrTitle(c, iter, context, pString, caseLocale, FALSE);
}
/* case folding ------------------------------------------------------------- */
@@ -1110,14 +1338,14 @@ ucase_toFullTitle(const UCaseProps *csp, UChar32 c,
/* return the simple case folding mapping for c */
U_CAPI UChar32 U_EXPORT2
-ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options) {
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
- if(UCASE_GET_TYPE(props)>=UCASE_UPPER) {
+ucase_fold(UChar32 c, uint32_t options) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
+ if(UCASE_IS_UPPER_OR_TITLE(props)) {
c+=UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props);
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props);
uint16_t excWord=*pe++;
int32_t idx;
if(excWord&UCASE_EXC_CONDITIONAL_FOLD) {
@@ -1142,6 +1370,14 @@ ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options) {
}
}
}
+ if((excWord&UCASE_EXC_NO_SIMPLE_CASE_FOLDING)!=0) {
+ return c;
+ }
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) {
idx=UCASE_EXC_FOLD;
} else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
@@ -1170,19 +1406,19 @@ ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options) {
*/
U_CAPI int32_t U_EXPORT2
-ucase_toFullFolding(const UCaseProps *csp, UChar32 c,
+ucase_toFullFolding(UChar32 c,
const UChar **pString,
uint32_t options) {
// The sign of the result has meaning, input must be non-negative so that it can be returned as is.
U_ASSERT(c >= 0);
UChar32 result=c;
- uint16_t props=UTRIE2_GET16(&csp->trie, c);
- if(!PROPS_HAS_EXCEPTION(props)) {
- if(UCASE_GET_TYPE(props)>=UCASE_UPPER) {
+ uint16_t props=UTRIE2_GET16(&ucase_props_singleton.trie, c);
+ if(!UCASE_HAS_EXCEPTION(props)) {
+ if(UCASE_IS_UPPER_OR_TITLE(props)) {
result=c+UCASE_GET_DELTA(props);
}
} else {
- const uint16_t *pe=GET_EXCEPTIONS(csp, props), *pe2;
+ const uint16_t *pe=GET_EXCEPTIONS(&ucase_props_singleton, props), *pe2;
uint16_t excWord=*pe++;
int32_t full, idx;
@@ -1229,6 +1465,14 @@ ucase_toFullFolding(const UCaseProps *csp, UChar32 c,
}
}
+ if((excWord&UCASE_EXC_NO_SIMPLE_CASE_FOLDING)!=0) {
+ return ~c;
+ }
+ if(HAS_SLOT(excWord, UCASE_EXC_DELTA) && UCASE_IS_UPPER_OR_TITLE(props)) {
+ int32_t delta;
+ GET_SLOT_VALUE(excWord, UCASE_EXC_DELTA, pe2, delta);
+ return (excWord&UCASE_EXC_DELTA_IS_NEGATIVE)==0 ? c+delta : c-delta;
+ }
if(HAS_SLOT(excWord, UCASE_EXC_FOLD)) {
idx=UCASE_EXC_FOLD;
} else if(HAS_SLOT(excWord, UCASE_EXC_LOWER)) {
@@ -1244,66 +1488,59 @@ ucase_toFullFolding(const UCaseProps *csp, UChar32 c,
/* case mapping properties API ---------------------------------------------- */
-#define GET_CASE_PROPS() &ucase_props_singleton
-
/* public API (see uchar.h) */
U_CAPI UBool U_EXPORT2
u_isULowercase(UChar32 c) {
- return (UBool)(UCASE_LOWER==ucase_getType(GET_CASE_PROPS(), c));
+ return (UBool)(UCASE_LOWER==ucase_getType(c));
}
U_CAPI UBool U_EXPORT2
u_isUUppercase(UChar32 c) {
- return (UBool)(UCASE_UPPER==ucase_getType(GET_CASE_PROPS(), c));
+ return (UBool)(UCASE_UPPER==ucase_getType(c));
}
/* Transforms the Unicode character to its lower case equivalent.*/
U_CAPI UChar32 U_EXPORT2
u_tolower(UChar32 c) {
- return ucase_tolower(GET_CASE_PROPS(), c);
+ return ucase_tolower(c);
}
/* Transforms the Unicode character to its upper case equivalent.*/
U_CAPI UChar32 U_EXPORT2
u_toupper(UChar32 c) {
- return ucase_toupper(GET_CASE_PROPS(), c);
+ return ucase_toupper(c);
}
/* Transforms the Unicode character to its title case equivalent.*/
U_CAPI UChar32 U_EXPORT2
u_totitle(UChar32 c) {
- return ucase_totitle(GET_CASE_PROPS(), c);
+ return ucase_totitle(c);
}
/* return the simple case folding mapping for c */
U_CAPI UChar32 U_EXPORT2
u_foldCase(UChar32 c, uint32_t options) {
- return ucase_fold(GET_CASE_PROPS(), c, options);
+ return ucase_fold(c, options);
}
U_CFUNC int32_t U_EXPORT2
ucase_hasBinaryProperty(UChar32 c, UProperty which) {
/* case mapping properties */
const UChar *resultString;
- int32_t locCache;
- const UCaseProps *csp=GET_CASE_PROPS();
- if(csp==NULL) {
- return FALSE;
- }
switch(which) {
case UCHAR_LOWERCASE:
- return (UBool)(UCASE_LOWER==ucase_getType(csp, c));
+ return (UBool)(UCASE_LOWER==ucase_getType(c));
case UCHAR_UPPERCASE:
- return (UBool)(UCASE_UPPER==ucase_getType(csp, c));
+ return (UBool)(UCASE_UPPER==ucase_getType(c));
case UCHAR_SOFT_DOTTED:
- return ucase_isSoftDotted(csp, c);
+ return ucase_isSoftDotted(c);
case UCHAR_CASE_SENSITIVE:
- return ucase_isCaseSensitive(csp, c);
+ return ucase_isCaseSensitive(c);
case UCHAR_CASED:
- return (UBool)(UCASE_NONE!=ucase_getType(csp, c));
+ return (UBool)(UCASE_NONE!=ucase_getType(c));
case UCHAR_CASE_IGNORABLE:
- return (UBool)(ucase_getTypeOrIgnorable(csp, c)>>2);
+ return (UBool)(ucase_getTypeOrIgnorable(c)>>2);
/*
* Note: The following Changes_When_Xyz are defined as testing whether
* the NFD form of the input changes when Xyz-case-mapped.
@@ -1317,21 +1554,17 @@ ucase_hasBinaryProperty(UChar32 c, UProperty which) {
* start sets for normalization and case mappings.
*/
case UCHAR_CHANGES_WHEN_LOWERCASED:
- locCache=UCASE_LOC_ROOT;
- return (UBool)(ucase_toFullLower(csp, c, NULL, NULL, &resultString, "", &locCache)>=0);
+ return (UBool)(ucase_toFullLower(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0);
case UCHAR_CHANGES_WHEN_UPPERCASED:
- locCache=UCASE_LOC_ROOT;
- return (UBool)(ucase_toFullUpper(csp, c, NULL, NULL, &resultString, "", &locCache)>=0);
+ return (UBool)(ucase_toFullUpper(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0);
case UCHAR_CHANGES_WHEN_TITLECASED:
- locCache=UCASE_LOC_ROOT;
- return (UBool)(ucase_toFullTitle(csp, c, NULL, NULL, &resultString, "", &locCache)>=0);
+ return (UBool)(ucase_toFullTitle(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0);
/* case UCHAR_CHANGES_WHEN_CASEFOLDED: -- in uprops.c */
case UCHAR_CHANGES_WHEN_CASEMAPPED:
- locCache=UCASE_LOC_ROOT;
return (UBool)(
- ucase_toFullLower(csp, c, NULL, NULL, &resultString, "", &locCache)>=0 ||
- ucase_toFullUpper(csp, c, NULL, NULL, &resultString, "", &locCache)>=0 ||
- ucase_toFullTitle(csp, c, NULL, NULL, &resultString, "", &locCache)>=0);
+ ucase_toFullLower(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0 ||
+ ucase_toFullUpper(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0 ||
+ ucase_toFullTitle(c, NULL, NULL, &resultString, UCASE_LOC_ROOT)>=0);
default:
return FALSE;
}
diff --git a/intl/icu/source/common/ucase.h b/intl/icu/source/common/ucase.h
index 29ea71a533..b0a453b87e 100644
--- a/intl/icu/source/common/ucase.h
+++ b/intl/icu/source/common/ucase.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -8,7 +8,7 @@
*
*******************************************************************************
* file name: ucase.h
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -26,6 +26,7 @@
#include "putilimp.h"
#include "uset_imp.h"
#include "udataswp.h"
+#include "utrie2.h"
#ifdef __cplusplus
U_NAMESPACE_BEGIN
@@ -37,18 +38,8 @@ U_NAMESPACE_END
/* library API -------------------------------------------------------------- */
-U_CDECL_BEGIN
-
-struct UCaseProps;
-typedef struct UCaseProps UCaseProps;
-
-U_CDECL_END
-
-U_CAPI const UCaseProps * U_EXPORT2
-ucase_getSingleton(void);
-
U_CFUNC void U_EXPORT2
-ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *pErrorCode);
+ucase_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode);
/**
* Requires non-NULL locale ID but otherwise does the equivalent of
@@ -56,7 +47,7 @@ ucase_addPropertyStarts(const UCaseProps *csp, const USetAdder *sa, UErrorCode *
* Accepts both 2- and 3-letter codes and accepts case variants.
*/
U_CFUNC int32_t
-ucase_getCaseLocale(const char *locale, int32_t *locCache);
+ucase_getCaseLocale(const char *locale);
/* Casing locale types for ucase_getCaseLocale */
enum {
@@ -71,7 +62,7 @@ enum {
/**
* Bit mask for getting just the options from a string compare options word
* that are relevant for case-insensitive string comparison.
- * See uchar.h. Also include _STRNCMP_STYLE and U_COMPARE_CODE_POINT_ORDER.
+ * See stringoptions.h. Also include _STRNCMP_STYLE and U_COMPARE_CODE_POINT_ORDER.
* @internal
*/
#define _STRCASECMP_OPTIONS_MASK 0xffff
@@ -79,24 +70,30 @@ enum {
/**
* Bit mask for getting just the options from a string compare options word
* that are relevant for case folding (of a single string or code point).
- * See uchar.h.
+ *
+ * Currently only bit 0 for U_FOLD_CASE_EXCLUDE_SPECIAL_I.
+ * It is conceivable that at some point we might use one more bit for using uppercase sharp s.
+ * It is conceivable that at some point we might want the option to use only simple case foldings
+ * when operating on strings.
+ *
+ * See stringoptions.h.
* @internal
*/
-#define _FOLD_CASE_OPTIONS_MASK 0xff
+#define _FOLD_CASE_OPTIONS_MASK 7
/* single-code point functions */
U_CAPI UChar32 U_EXPORT2
-ucase_tolower(const UCaseProps *csp, UChar32 c);
+ucase_tolower(UChar32 c);
U_CAPI UChar32 U_EXPORT2
-ucase_toupper(const UCaseProps *csp, UChar32 c);
+ucase_toupper(UChar32 c);
U_CAPI UChar32 U_EXPORT2
-ucase_totitle(const UCaseProps *csp, UChar32 c);
+ucase_totitle(UChar32 c);
U_CAPI UChar32 U_EXPORT2
-ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options);
+ucase_fold(UChar32 c, uint32_t options);
/**
* Adds all simple case mappings and the full case folding for c to sa,
@@ -108,7 +105,7 @@ ucase_fold(const UCaseProps *csp, UChar32 c, uint32_t options);
* - for k include the Kelvin sign
*/
U_CFUNC void U_EXPORT2
-ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa);
+ucase_addCaseClosure(UChar32 c, const USetAdder *sa);
/**
* Maps the string to single code points and adds the associated case closure
@@ -123,7 +120,7 @@ ucase_addCaseClosure(const UCaseProps *csp, UChar32 c, const USetAdder *sa);
* @return TRUE if the string was found
*/
U_CFUNC UBool U_EXPORT2
-ucase_addStringCaseClosure(const UCaseProps *csp, const UChar *s, int32_t length, const USetAdder *sa);
+ucase_addStringCaseClosure(const UChar *s, int32_t length, const USetAdder *sa);
#ifdef __cplusplus
U_NAMESPACE_BEGIN
@@ -152,22 +149,49 @@ private:
int32_t rowCpIndex;
};
+/**
+ * Fast case mapping data for ASCII/Latin.
+ * Linear arrays of delta bytes: 0=no mapping; EXC=exception.
+ * Deltas must not cross the ASCII boundary, or else they cannot be easily used
+ * in simple UTF-8 code.
+ */
+namespace LatinCase {
+
+/** Case mapping/folding data for code points up to U+017F. */
+constexpr UChar LIMIT = 0x180;
+/** U+017F case-folds and uppercases crossing the ASCII boundary. */
+constexpr UChar LONG_S = 0x17f;
+/** Exception: Complex mapping, or too-large delta. */
+constexpr int8_t EXC = -0x80;
+
+/** Deltas for lowercasing for most locales, and default case folding. */
+extern const int8_t TO_LOWER_NORMAL[LIMIT];
+/** Deltas for lowercasing for tr/az/lt, and Turkic case folding. */
+extern const int8_t TO_LOWER_TR_LT[LIMIT];
+
+/** Deltas for uppercasing for most locales. */
+extern const int8_t TO_UPPER_NORMAL[LIMIT];
+/** Deltas for uppercasing for tr/az. */
+extern const int8_t TO_UPPER_TR[LIMIT];
+
+} // namespace LatinCase
+
U_NAMESPACE_END
#endif
/** @return UCASE_NONE, UCASE_LOWER, UCASE_UPPER, UCASE_TITLE */
U_CAPI int32_t U_EXPORT2
-ucase_getType(const UCaseProps *csp, UChar32 c);
+ucase_getType(UChar32 c);
/** @return like ucase_getType() but also sets UCASE_IGNORABLE if c is case-ignorable */
U_CAPI int32_t U_EXPORT2
-ucase_getTypeOrIgnorable(const UCaseProps *csp, UChar32 c);
+ucase_getTypeOrIgnorable(UChar32 c);
U_CAPI UBool U_EXPORT2
-ucase_isSoftDotted(const UCaseProps *csp, UChar32 c);
+ucase_isSoftDotted(UChar32 c);
U_CAPI UBool U_EXPORT2
-ucase_isCaseSensitive(const UCaseProps *csp, UChar32 c);
+ucase_isCaseSensitive(UChar32 c);
/* string case mapping functions */
@@ -240,10 +264,7 @@ enum {
* @param context Pointer to be passed into iter.
* @param pString If the mapping result is a string, then the pointer is
* written to *pString.
- * @param locale Locale ID for locale-dependent mappings.
- * @param locCache Initialize to 0; may be used to cache the result of parsing
- * the locale ID for subsequent calls.
- * Can be NULL.
+ * @param caseLocale Case locale value from ucase_getCaseLocale().
* @return Output code point or string length, see UCASE_MAX_STRING_LENGTH.
*
* @see UCaseContextIterator
@@ -251,25 +272,25 @@ enum {
* @internal
*/
U_CAPI int32_t U_EXPORT2
-ucase_toFullLower(const UCaseProps *csp, UChar32 c,
+ucase_toFullLower(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache);
+ int32_t caseLocale);
U_CAPI int32_t U_EXPORT2
-ucase_toFullUpper(const UCaseProps *csp, UChar32 c,
+ucase_toFullUpper(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache);
+ int32_t caseLocale);
U_CAPI int32_t U_EXPORT2
-ucase_toFullTitle(const UCaseProps *csp, UChar32 c,
+ucase_toFullTitle(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache);
+ int32_t caseLocale);
U_CAPI int32_t U_EXPORT2
-ucase_toFullFolding(const UCaseProps *csp, UChar32 c,
+ucase_toFullFolding(UChar32 c,
const UChar **pString,
uint32_t options);
@@ -283,10 +304,10 @@ U_CDECL_BEGIN
* @internal
*/
typedef int32_t U_CALLCONV
-UCaseMapFull(const UCaseProps *csp, UChar32 c,
+UCaseMapFull(UChar32 c,
UCaseContextIterator *iter, void *context,
const UChar **pString,
- const char *locale, int32_t *locCache);
+ int32_t caseLocale);
U_CDECL_END
@@ -315,6 +336,9 @@ enum {
/* definitions for 16-bit case properties word ------------------------------ */
+U_CFUNC const UTrie2 * U_EXPORT2
+ucase_getTrie();
+
/* 2-bit constants for types of cased characters */
#define UCASE_TYPE_MASK 3
enum {
@@ -327,9 +351,13 @@ enum {
#define UCASE_GET_TYPE(props) ((props)&UCASE_TYPE_MASK)
#define UCASE_GET_TYPE_AND_IGNORABLE(props) ((props)&7)
+#define UCASE_IS_UPPER_OR_TITLE(props) ((props)&2)
+
#define UCASE_IGNORABLE 4
-#define UCASE_SENSITIVE 8
-#define UCASE_EXCEPTION 0x10
+#define UCASE_EXCEPTION 8
+#define UCASE_SENSITIVE 0x10
+
+#define UCASE_HAS_EXCEPTION(props) ((props)&UCASE_EXCEPTION)
#define UCASE_DOT_MASK 0x60
enum {
@@ -351,9 +379,9 @@ enum {
# define UCASE_GET_DELTA(props) (int16_t)(((props)&0x8000) ? (((props)>>UCASE_DELTA_SHIFT)|0xfe00) : ((uint16_t)(props)>>UCASE_DELTA_SHIFT))
#endif
-/* exception: bits 15..5 are an unsigned 11-bit index into the exceptions array */
-#define UCASE_EXC_SHIFT 5
-#define UCASE_EXC_MASK 0xffe0
+/* exception: bits 15..4 are an unsigned 12-bit index into the exceptions array */
+#define UCASE_EXC_SHIFT 4
+#define UCASE_EXC_MASK 0xfff0
#define UCASE_MAX_EXCEPTIONS ((UCASE_EXC_MASK>>UCASE_EXC_SHIFT)+1)
/* definitions for 16-bit main exceptions word ------------------------------ */
@@ -364,7 +392,7 @@ enum {
UCASE_EXC_FOLD,
UCASE_EXC_UPPER,
UCASE_EXC_TITLE,
- UCASE_EXC_4, /* reserved */
+ UCASE_EXC_DELTA,
UCASE_EXC_5, /* reserved */
UCASE_EXC_CLOSURE,
UCASE_EXC_FULL_MAPPINGS,
@@ -374,7 +402,11 @@ enum {
/* each slot is 2 uint16_t instead of 1 */
#define UCASE_EXC_DOUBLE_SLOTS 0x100
-/* reserved: exception bits 11..9 */
+enum {
+ UCASE_EXC_NO_SIMPLE_CASE_FOLDING=0x200,
+ UCASE_EXC_DELTA_IS_NEGATIVE=0x400,
+ UCASE_EXC_SENSITIVE=0x800
+};
/* UCASE_EXC_DOT_MASK=UCASE_DOT_MASK<csp=ucase_getSingleton();
- ucasemap_setLocale(csm, locale, pErrorCode);
- if(U_FAILURE(*pErrorCode)) {
- uprv_free(csm);
- return NULL;
- }
-
- csm->options=options;
return csm;
}
U_CAPI void U_EXPORT2
ucasemap_close(UCaseMap *csm) {
- if(csm!=NULL) {
-#if !UCONFIG_NO_BREAK_ITERATION
- // Do not call ubrk_close() so that we do not depend on all of the BreakIterator code.
- delete reinterpret_cast(csm->iter);
-#endif
- uprv_free(csm);
- }
+ delete csm;
}
U_CAPI const char * U_EXPORT2
@@ -87,13 +94,16 @@ ucasemap_getOptions(const UCaseMap *csm) {
U_CAPI void U_EXPORT2
ucasemap_setLocale(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode) {
- int32_t length;
-
if(U_FAILURE(*pErrorCode)) {
return;
}
+ if (locale != NULL && *locale == 0) {
+ csm->locale[0] = 0;
+ csm->caseLocale = UCASE_LOC_ROOT;
+ return;
+ }
- length=uloc_getName(locale, csm->locale, (int32_t)sizeof(csm->locale), pErrorCode);
+ int32_t length=uloc_getName(locale, csm->locale, (int32_t)sizeof(csm->locale), pErrorCode);
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR || length==sizeof(csm->locale)) {
*pErrorCode=U_ZERO_ERROR;
/* we only really need the language code for case mappings */
@@ -102,126 +112,60 @@ ucasemap_setLocale(UCaseMap *csm, const char *locale, UErrorCode *pErrorCode) {
if(length==sizeof(csm->locale)) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
- csm->locCache=0;
if(U_SUCCESS(*pErrorCode)) {
- ucase_getCaseLocale(csm->locale, &csm->locCache);
+ csm->caseLocale=UCASE_LOC_UNKNOWN;
+ csm->caseLocale = ucase_getCaseLocale(csm->locale);
} else {
csm->locale[0]=0;
+ csm->caseLocale = UCASE_LOC_ROOT;
}
}
U_CAPI void U_EXPORT2
-ucasemap_setOptions(UCaseMap *csm, uint32_t options, UErrorCode * /*pErrorCode*/) {
+ucasemap_setOptions(UCaseMap *csm, uint32_t options, UErrorCode *pErrorCode) {
+ if(U_FAILURE(*pErrorCode)) {
+ return;
+ }
csm->options=options;
}
/* UTF-8 string case mappings ----------------------------------------------- */
-/* TODO(markus): Move to a new, separate utf8case.c file. */
+/* TODO(markus): Move to a new, separate utf8case.cpp file. */
+
+namespace {
/* append a full case mapping result, see UCASE_MAX_STRING_LENGTH */
-static inline int32_t
-appendResult(uint8_t *dest, int32_t destIndex, int32_t destCapacity,
- int32_t result, const UChar *s) {
- UChar32 c;
- int32_t length;
- UErrorCode errorCode;
+inline UBool
+appendResult(int32_t cpLength, int32_t result, const UChar *s,
+ ByteSink &sink, uint32_t options, icu::Edits *edits, UErrorCode &errorCode) {
+ U_ASSERT(U_SUCCESS(errorCode));
/* decode the result */
if(result<0) {
/* (not) original code point */
- c=~result;
- length=U8_LENGTH(c);
- } else if(result<=UCASE_MAX_STRING_LENGTH) {
- c=U_SENTINEL;
- length=result;
- } else {
- c=result;
- length=U8_LENGTH(c);
- }
- if(length>(INT32_MAX-destIndex)) {
- return -1; // integer overflow
- }
-
- if(destIndex=0) {
- /* code point */
- UBool isError=FALSE;
- U8_APPEND(dest, destIndex, destCapacity, c, isError);
- if(isError) {
- /* overflow, nothing written */
- destIndex+=length;
- }
- } else {
- /* string */
- int32_t destLength;
- errorCode=U_ZERO_ERROR;
- u_strToUTF8(
- (char *)(dest+destIndex), destCapacity-destIndex, &destLength,
- s, length,
- &errorCode);
- if(U_FAILURE(errorCode) && errorCode != U_BUFFER_OVERFLOW_ERROR) {
- return -1;
- }
- if(destLength>(INT32_MAX-destIndex)) {
- return -1; // integer overflow
- }
- destIndex+=destLength;
- /* we might have an overflow, but we know the actual length */
+ if(edits!=NULL) {
+ edits->addUnchanged(cpLength);
+ }
+ if((options & U_OMIT_UNCHANGED_TEXT) == 0) {
+ ByteSinkUtil::appendCodePoint(cpLength, ~result, sink);
}
} else {
- /* preflight */
- if(c>=0) {
- destIndex+=length;
+ if(result<=UCASE_MAX_STRING_LENGTH) {
+ // string: "result" is the UTF-16 length
+ return ByteSinkUtil::appendChange(cpLength, s, result, sink, edits, errorCode);
} else {
- int32_t destLength;
- errorCode=U_ZERO_ERROR;
- u_strToUTF8(
- NULL, 0, &destLength,
- s, length,
- &errorCode);
- if(U_FAILURE(errorCode) && errorCode != U_BUFFER_OVERFLOW_ERROR) {
- return -1;
- }
- if(destLength>(INT32_MAX-destIndex)) {
- return -1; // integer overflow
- }
- destIndex+=destLength;
+ ByteSinkUtil::appendCodePoint(cpLength, result, sink, edits);
}
}
- return destIndex;
+ return TRUE;
}
-static inline int32_t
-appendUChar(uint8_t *dest, int32_t destIndex, int32_t destCapacity, UChar c) {
- int32_t length=U8_LENGTH(c);
- if(length>(INT32_MAX-destIndex)) {
- return -1; // integer overflow
- }
- int32_t limit=destIndex+length;
- if(limit<=destCapacity) {
- U8_APPEND_UNSAFE(dest, destIndex, c);
- }
- return limit;
-}
+// See unicode/utf8.h U8_APPEND_UNSAFE().
+inline uint8_t getTwoByteLead(UChar32 c) { return (uint8_t)((c >> 6) | 0xc0); }
+inline uint8_t getTwoByteTrail(UChar32 c) { return (uint8_t)((c & 0x3f) | 0x80); }
-static inline int32_t
-appendString(uint8_t *dest, int32_t destIndex, int32_t destCapacity,
- const uint8_t *s, int32_t length) {
- if(length>0) {
- if(length>(INT32_MAX-destIndex)) {
- return -1; // integer overflow
- }
- if((destIndex+length)<=destCapacity) {
- uprv_memcpy(dest+destIndex, s, length);
- }
- destIndex+=length;
- }
- return destIndex;
-}
-
-static UChar32 U_CALLCONV
+UChar32 U_CALLCONV
utf8_caseContextIterator(void *context, int8_t dir) {
UCaseContext *csc=(UCaseContext *)context;
UChar32 c;
@@ -253,137 +197,291 @@ utf8_caseContextIterator(void *context, int8_t dir) {
return U_SENTINEL;
}
-/*
- * Case-maps [srcStart..srcLimit[ but takes
- * context [0..srcLength[ into account.
+/**
+ * caseLocale >= 0: Lowercases [srcStart..srcLimit[ but takes context [0..srcLength[ into account.
+ * caseLocale < 0: Case-folds [srcStart..srcLimit[.
*/
-static int32_t
-_caseMap(const UCaseMap *csm, UCaseMapFull *map,
- uint8_t *dest, int32_t destCapacity,
- const uint8_t *src, UCaseContext *csc,
- int32_t srcStart, int32_t srcLimit,
- UErrorCode *pErrorCode) {
- const UChar *s = NULL;
- UChar32 c, c2 = 0;
- int32_t srcIndex, destIndex;
- int32_t locCache;
-
- locCache=csm->locCache;
-
- /* case mapping loop */
- srcIndex=srcStart;
- destIndex=0;
- while(srcIndexcpStart=srcIndex;
- U8_NEXT(src, srcIndex, srcLimit, c);
- csc->cpLimit=srcIndex;
- if(c<0) {
- // Malformed UTF-8.
- destIndex=appendString(dest, destIndex, destCapacity, src+csc->cpStart, srcIndex-csc->cpStart);
- if(destIndex<0) {
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
+void toLower(int32_t caseLocale, uint32_t options,
+ const uint8_t *src, UCaseContext *csc, int32_t srcStart, int32_t srcLimit,
+ icu::ByteSink &sink, icu::Edits *edits, UErrorCode &errorCode) {
+ const int8_t *latinToLower;
+ if (caseLocale == UCASE_LOC_ROOT ||
+ (caseLocale >= 0 ?
+ !(caseLocale == UCASE_LOC_TURKISH || caseLocale == UCASE_LOC_LITHUANIAN) :
+ (options & _FOLD_CASE_OPTIONS_MASK) == U_FOLD_CASE_DEFAULT)) {
+ latinToLower = LatinCase::TO_LOWER_NORMAL;
+ } else {
+ latinToLower = LatinCase::TO_LOWER_TR_LT;
+ }
+ const UTrie2 *trie = ucase_getTrie();
+ int32_t prev = srcStart;
+ int32_t srcIndex = srcStart;
+ for (;;) {
+ // fast path for simple cases
+ int32_t cpStart;
+ UChar32 c;
+ for (;;) {
+ if (U_FAILURE(errorCode) || srcIndex >= srcLimit) {
+ c = U_SENTINEL;
+ break;
}
- continue;
+ uint8_t lead = src[srcIndex++];
+ if (lead <= 0x7f) {
+ int8_t d = latinToLower[lead];
+ if (d == LatinCase::EXC) {
+ cpStart = srcIndex - 1;
+ c = lead;
+ break;
+ }
+ if (d == 0) { continue; }
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - 1 - prev,
+ sink, options, edits, errorCode);
+ char ascii = (char)(lead + d);
+ sink.Append(&ascii, 1);
+ if (edits != nullptr) {
+ edits->addReplace(1, 1);
+ }
+ prev = srcIndex;
+ continue;
+ } else if (lead < 0xe3) {
+ uint8_t t;
+ if (0xc2 <= lead && lead <= 0xc5 && srcIndex < srcLimit &&
+ (t = src[srcIndex] - 0x80) <= 0x3f) {
+ // U+0080..U+017F
+ ++srcIndex;
+ c = ((lead - 0xc0) << 6) | t;
+ int8_t d = latinToLower[c];
+ if (d == LatinCase::EXC) {
+ cpStart = srcIndex - 2;
+ break;
+ }
+ if (d == 0) { continue; }
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - 2 - prev,
+ sink, options, edits, errorCode);
+ ByteSinkUtil::appendTwoBytes(c + d, sink);
+ if (edits != nullptr) {
+ edits->addReplace(2, 2);
+ }
+ prev = srcIndex;
+ continue;
+ }
+ } else if ((lead <= 0xe9 || lead == 0xeb || lead == 0xec) &&
+ (srcIndex + 2) <= srcLimit &&
+ U8_IS_TRAIL(src[srcIndex]) && U8_IS_TRAIL(src[srcIndex + 1])) {
+ // most of CJK: no case mappings
+ srcIndex += 2;
+ continue;
+ }
+ cpStart = --srcIndex;
+ U8_NEXT(src, srcIndex, srcLimit, c);
+ if (c < 0) {
+ // ill-formed UTF-8
+ continue;
+ }
+ uint16_t props = UTRIE2_GET16(trie, c);
+ if (UCASE_HAS_EXCEPTION(props)) { break; }
+ int32_t delta;
+ if (!UCASE_IS_UPPER_OR_TITLE(props) || (delta = UCASE_GET_DELTA(props)) == 0) {
+ continue;
+ }
+ ByteSinkUtil::appendUnchanged(src + prev, cpStart - prev,
+ sink, options, edits, errorCode);
+ ByteSinkUtil::appendCodePoint(srcIndex - cpStart, c + delta, sink, edits);
+ prev = srcIndex;
}
- c=map(csm->csp, c, utf8_caseContextIterator, csc, &s, csm->locale, &locCache);
- if((destIndex= 0) {
+ csc->cpStart = cpStart;
+ csc->cpLimit = srcIndex;
+ c = ucase_toFullLower(c, utf8_caseContextIterator, csc, &s, caseLocale);
} else {
- destIndex=appendResult(dest, destIndex, destCapacity, c, s);
- if(destIndex<0) {
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
- }
+ c = ucase_toFullFolding(c, &s, options);
+ }
+ if (c >= 0) {
+ ByteSinkUtil::appendUnchanged(src + prev, cpStart - prev,
+ sink, options, edits, errorCode);
+ appendResult(srcIndex - cpStart, c, s, sink, options, edits, errorCode);
+ prev = srcIndex;
}
}
-
- if(destIndex>destCapacity) {
- *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
- }
- return destIndex;
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - prev,
+ sink, options, edits, errorCode);
}
+void toUpper(int32_t caseLocale, uint32_t options,
+ const uint8_t *src, UCaseContext *csc, int32_t srcLength,
+ icu::ByteSink &sink, icu::Edits *edits, UErrorCode &errorCode) {
+ const int8_t *latinToUpper;
+ if (caseLocale == UCASE_LOC_TURKISH) {
+ latinToUpper = LatinCase::TO_UPPER_TR;
+ } else {
+ latinToUpper = LatinCase::TO_UPPER_NORMAL;
+ }
+ const UTrie2 *trie = ucase_getTrie();
+ int32_t prev = 0;
+ int32_t srcIndex = 0;
+ for (;;) {
+ // fast path for simple cases
+ int32_t cpStart;
+ UChar32 c;
+ for (;;) {
+ if (U_FAILURE(errorCode) || srcIndex >= srcLength) {
+ c = U_SENTINEL;
+ break;
+ }
+ uint8_t lead = src[srcIndex++];
+ if (lead <= 0x7f) {
+ int8_t d = latinToUpper[lead];
+ if (d == LatinCase::EXC) {
+ cpStart = srcIndex - 1;
+ c = lead;
+ break;
+ }
+ if (d == 0) { continue; }
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - 1 - prev,
+ sink, options, edits, errorCode);
+ char ascii = (char)(lead + d);
+ sink.Append(&ascii, 1);
+ if (edits != nullptr) {
+ edits->addReplace(1, 1);
+ }
+ prev = srcIndex;
+ continue;
+ } else if (lead < 0xe3) {
+ uint8_t t;
+ if (0xc2 <= lead && lead <= 0xc5 && srcIndex < srcLength &&
+ (t = src[srcIndex] - 0x80) <= 0x3f) {
+ // U+0080..U+017F
+ ++srcIndex;
+ c = ((lead - 0xc0) << 6) | t;
+ int8_t d = latinToUpper[c];
+ if (d == LatinCase::EXC) {
+ cpStart = srcIndex - 2;
+ break;
+ }
+ if (d == 0) { continue; }
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - 2 - prev,
+ sink, options, edits, errorCode);
+ ByteSinkUtil::appendTwoBytes(c + d, sink);
+ if (edits != nullptr) {
+ edits->addReplace(2, 2);
+ }
+ prev = srcIndex;
+ continue;
+ }
+ } else if ((lead <= 0xe9 || lead == 0xeb || lead == 0xec) &&
+ (srcIndex + 2) <= srcLength &&
+ U8_IS_TRAIL(src[srcIndex]) && U8_IS_TRAIL(src[srcIndex + 1])) {
+ // most of CJK: no case mappings
+ srcIndex += 2;
+ continue;
+ }
+ cpStart = --srcIndex;
+ U8_NEXT(src, srcIndex, srcLength, c);
+ if (c < 0) {
+ // ill-formed UTF-8
+ continue;
+ }
+ uint16_t props = UTRIE2_GET16(trie, c);
+ if (UCASE_HAS_EXCEPTION(props)) { break; }
+ int32_t delta;
+ if (UCASE_GET_TYPE(props) != UCASE_LOWER || (delta = UCASE_GET_DELTA(props)) == 0) {
+ continue;
+ }
+ ByteSinkUtil::appendUnchanged(src + prev, cpStart - prev,
+ sink, options, edits, errorCode);
+ ByteSinkUtil::appendCodePoint(srcIndex - cpStart, c + delta, sink, edits);
+ prev = srcIndex;
+ }
+ if (c < 0) {
+ break;
+ }
+ // slow path
+ csc->cpStart = cpStart;
+ csc->cpLimit = srcIndex;
+ const UChar *s;
+ c = ucase_toFullUpper(c, utf8_caseContextIterator, csc, &s, caseLocale);
+ if (c >= 0) {
+ ByteSinkUtil::appendUnchanged(src + prev, cpStart - prev,
+ sink, options, edits, errorCode);
+ appendResult(srcIndex - cpStart, c, s, sink, options, edits, errorCode);
+ prev = srcIndex;
+ }
+ }
+ ByteSinkUtil::appendUnchanged(src + prev, srcIndex - prev,
+ sink, options, edits, errorCode);
+}
+
+} // namespace
+
#if !UCONFIG_NO_BREAK_ITERATION
-U_CFUNC int32_t U_CALLCONV
-ucasemap_internalUTF8ToTitle(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
- const uint8_t *src, int32_t srcLength,
- UErrorCode *pErrorCode) {
- const UChar *s;
- UChar32 c;
- int32_t prev, titleStart, titleLimit, idx, destIndex;
- UBool isFirstIndex;
-
- if(U_FAILURE(*pErrorCode)) {
- return 0;
+U_CFUNC void U_CALLCONV
+ucasemap_internalUTF8ToTitle(
+ int32_t caseLocale, uint32_t options, BreakIterator *iter,
+ const uint8_t *src, int32_t srcLength,
+ ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode) {
+ if (!ustrcase_checkTitleAdjustmentOptions(options, errorCode)) {
+ return;
}
- // Use the C++ abstract base class to minimize dependencies.
- // TODO: Change UCaseMap.iter to store a BreakIterator directly.
- BreakIterator *bi=reinterpret_cast(csm->iter);
-
/* set up local variables */
- int32_t locCache=csm->locCache;
UCaseContext csc=UCASECONTEXT_INITIALIZER;
csc.p=(void *)src;
csc.limit=srcLength;
- destIndex=0;
- prev=0;
- isFirstIndex=TRUE;
+ int32_t prev=0;
+ UBool isFirstIndex=TRUE;
/* titlecasing loop */
while(prevfirst();
+ index=iter->first();
} else {
- idx=bi->next();
+ index=iter->next();
}
- if(idx==UBRK_DONE || idx>srcLength) {
- idx=srcLength;
+ if(index==UBRK_DONE || index>srcLength) {
+ index=srcLength;
}
/*
- * Unicode 4 & 5 section 3.13 Default Case Operations:
- *
- * R3 toTitlecase(X): Find the word boundaries based on Unicode Standard Annex
- * #29, "Text Boundaries." Between each pair of word boundaries, find the first
- * cased character F. If F exists, map F to default_title(F); then map each
- * subsequent character C to default_lower(C).
- *
- * In this implementation, segment [prev..index[ into 3 parts:
- * a) uncased characters (copy as-is) [prev..titleStart[
- * b) first case letter (titlecase) [titleStart..titleLimit[
+ * Segment [prev..index[ into 3 parts:
+ * a) skipped characters (copy as-is) [prev..titleStart[
+ * b) first letter (titlecase) [titleStart..titleLimit[
* c) subsequent characters (lowercase) [titleLimit..index[
*/
- if(prevoptions&U_TITLECASE_NO_BREAK_ADJUSTMENT)==0 && UCASE_NONE==ucase_getType(csm->csp, c)) {
- /* Adjust the titlecasing index (titleStart) to the next cased character. */
- for(;;) {
+ if(prevcsp, c)) {
- break; /* cased letter at [titleStart..titleLimit[ */
- }
+ U8_NEXT(src, titleLimit, index, c);
}
- destIndex=appendString(dest, destIndex, destCapacity, src+prev, titleStart-prev);
- if(destIndex<0) {
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
+ if (prev < titleStart) {
+ if (!ByteSinkUtil::appendUnchanged(src+prev, titleStart-prev,
+ sink, options, edits, errorCode)) {
+ return;
+ }
}
}
@@ -392,61 +490,59 @@ ucasemap_internalUTF8ToTitle(const UCaseMap *csm,
if(c>=0) {
csc.cpStart=titleStart;
csc.cpLimit=titleLimit;
- c=ucase_toFullTitle(csm->csp, c, utf8_caseContextIterator, &csc, &s, csm->locale, &locCache);
- destIndex=appendResult(dest, destIndex, destCapacity, c, s);
+ const UChar *s;
+ c=ucase_toFullTitle(c, utf8_caseContextIterator, &csc, &s, caseLocale);
+ if (!appendResult(titleLimit-titleStart, c, s, sink, options, edits, errorCode)) {
+ return;
+ }
} else {
// Malformed UTF-8.
- destIndex=appendString(dest, destIndex, destCapacity, src+titleStart, titleLimit-titleStart);
- }
- if(destIndex<0) {
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
+ if (!ByteSinkUtil::appendUnchanged(src+titleStart, titleLimit-titleStart,
+ sink, options, edits, errorCode)) {
+ return;
+ }
}
/* Special case Dutch IJ titlecasing */
- if (titleStart+1 < idx &&
- ucase_getCaseLocale(csm->locale, &locCache) == UCASE_LOC_DUTCH &&
- (src[titleStart] == 0x0049 || src[titleStart] == 0x0069) &&
- (src[titleStart+1] == 0x004A || src[titleStart+1] == 0x006A)) {
- destIndex=appendUChar(dest, destIndex, destCapacity, 0x004A);
- titleLimit++;
- }
- /* lowercase [titleLimit..index[ */
- if(titleLimitoptions&U_TITLECASE_NO_LOWERCASE)==0) {
- /* Normal operation: Lowercase the rest of the word. */
- destIndex+=
- _caseMap(
- csm, ucase_toFullLower,
- dest+destIndex, destCapacity-destIndex,
- src, &csc,
- titleLimit, idx,
- pErrorCode);
- if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
- *pErrorCode=U_ZERO_ERROR;
+ if (titleStart+1 < index &&
+ caseLocale == UCASE_LOC_DUTCH &&
+ (src[titleStart] == 0x0049 || src[titleStart] == 0x0069)) {
+ if (src[titleStart+1] == 0x006A) {
+ ByteSinkUtil::appendCodePoint(1, 0x004A, sink, edits);
+ titleLimit++;
+ } else if (src[titleStart+1] == 0x004A) {
+ // Keep the capital J from getting lowercased.
+ if (!ByteSinkUtil::appendUnchanged(src+titleStart+1, 1,
+ sink, options, edits, errorCode)) {
+ return;
}
- if(U_FAILURE(*pErrorCode)) {
- return destIndex;
+ titleLimit++;
+ }
+ }
+
+ /* lowercase [titleLimit..index[ */
+ if(titleLimitdestCapacity) {
- *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
- }
- return destIndex;
}
#endif
@@ -454,11 +550,11 @@ ucasemap_internalUTF8ToTitle(const UCaseMap *csm,
U_NAMESPACE_BEGIN
namespace GreekUpper {
-UBool isFollowedByCasedLetter(const UCaseProps *csp, const uint8_t *s, int32_t i, int32_t length) {
+UBool isFollowedByCasedLetter(const uint8_t *s, int32_t i, int32_t length) {
while (i < length) {
UChar32 c;
U8_NEXT(s, i, length, c);
- int32_t type = ucase_getTypeOrIgnorable(csp, c);
+ int32_t type = ucase_getTypeOrIgnorable(c);
if ((type & UCASE_IGNORABLE) != 0) {
// Case-ignorable, continue with the loop.
} else if (type != UCASE_NONE) {
@@ -471,19 +567,17 @@ UBool isFollowedByCasedLetter(const UCaseProps *csp, const uint8_t *s, int32_t i
}
// Keep this consistent with the UTF-16 version in ustrcase.cpp and the Java version in CaseMap.java.
-int32_t toUpper(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
- const uint8_t *src, int32_t srcLength,
- UErrorCode *pErrorCode) {
- int32_t locCache = UCASE_LOC_GREEK;
- int32_t destIndex=0;
+void toUpper(uint32_t options,
+ const uint8_t *src, int32_t srcLength,
+ ByteSink &sink, Edits *edits,
+ UErrorCode &errorCode) {
uint32_t state = 0;
for (int32_t i = 0; i < srcLength;) {
int32_t nextIndex = i;
UChar32 c;
U8_NEXT(src, nextIndex, srcLength, c);
uint32_t nextState = 0;
- int32_t type = ucase_getTypeOrIgnorable(csm->csp, c);
+ int32_t type = ucase_getTypeOrIgnorable(c);
if ((type & UCASE_IGNORABLE) != 0) {
// c is case-ignorable
nextState |= (state & AFTER_CASED);
@@ -533,7 +627,7 @@ int32_t toUpper(const UCaseMap *csm,
(data & HAS_ACCENT) != 0 &&
numYpogegrammeni == 0 &&
(state & AFTER_CASED) == 0 &&
- !isFollowedByCasedLetter(csm->csp, src, nextIndex, srcLength)) {
+ !isFollowedByCasedLetter(src, nextIndex, srcLength)) {
// Keep disjunctive "or" with (only) a tonos.
// We use the same "word boundary" conditions as for the Final_Sigma test.
if (i == nextIndex) {
@@ -551,161 +645,171 @@ int32_t toUpper(const UCaseMap *csm,
data &= ~HAS_EITHER_DIALYTIKA;
}
}
- destIndex=appendUChar(dest, destIndex, destCapacity, (UChar)upper);
- if (destIndex >= 0 && (data & HAS_EITHER_DIALYTIKA) != 0) {
- destIndex=appendUChar(dest, destIndex, destCapacity, 0x308); // restore or add a dialytika
+
+ UBool change;
+ if (edits == nullptr && (options & U_OMIT_UNCHANGED_TEXT) == 0) {
+ change = TRUE; // common, simple usage
+ } else {
+ // Find out first whether we are changing the text.
+ U_ASSERT(0x370 <= upper && upper <= 0x3ff); // 2-byte UTF-8, main Greek block
+ change = (i + 2) > nextIndex ||
+ src[i] != getTwoByteLead(upper) || src[i + 1] != getTwoByteTrail(upper) ||
+ numYpogegrammeni > 0;
+ int32_t i2 = i + 2;
+ if ((data & HAS_EITHER_DIALYTIKA) != 0) {
+ change |= (i2 + 2) > nextIndex ||
+ src[i2] != (uint8_t)u8"\u0308"[0] ||
+ src[i2 + 1] != (uint8_t)u8"\u0308"[1];
+ i2 += 2;
+ }
+ if (addTonos) {
+ change |= (i2 + 2) > nextIndex ||
+ src[i2] != (uint8_t)u8"\u0301"[0] ||
+ src[i2 + 1] != (uint8_t)u8"\u0301"[1];
+ i2 += 2;
+ }
+ int32_t oldLength = nextIndex - i;
+ int32_t newLength = (i2 - i) + numYpogegrammeni * 2; // 2 bytes per U+0399
+ change |= oldLength != newLength;
+ if (change) {
+ if (edits != NULL) {
+ edits->addReplace(oldLength, newLength);
+ }
+ } else {
+ if (edits != NULL) {
+ edits->addUnchanged(oldLength);
+ }
+ // Write unchanged text?
+ change = (options & U_OMIT_UNCHANGED_TEXT) == 0;
+ }
}
- if (destIndex >= 0 && addTonos) {
- destIndex=appendUChar(dest, destIndex, destCapacity, 0x301);
- }
- while (destIndex >= 0 && numYpogegrammeni > 0) {
- destIndex=appendUChar(dest, destIndex, destCapacity, 0x399);
- --numYpogegrammeni;
- }
- if(destIndex<0) {
- *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
- return 0;
+
+ if (change) {
+ ByteSinkUtil::appendTwoBytes(upper, sink);
+ if ((data & HAS_EITHER_DIALYTIKA) != 0) {
+ sink.Append(u8"\u0308", 2); // restore or add a dialytika
+ }
+ if (addTonos) {
+ sink.Append(u8"\u0301", 2);
+ }
+ while (numYpogegrammeni > 0) {
+ sink.Append(u8"\u0399", 2);
+ --numYpogegrammeni;
+ }
}
} else if(c>=0) {
const UChar *s;
- UChar32 c2 = 0;
- c=ucase_toFullUpper(csm->csp, c, NULL, NULL, &s, csm->locale, &locCache);
- if((destIndexdestCapacity) {
- *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
- }
- return destIndex;
}
} // namespace GreekUpper
U_NAMESPACE_END
-static int32_t U_CALLCONV
-ucasemap_internalUTF8ToLower(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
+static void U_CALLCONV
+ucasemap_internalUTF8ToLower(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_UNUSED
const uint8_t *src, int32_t srcLength,
- UErrorCode *pErrorCode) {
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode) {
UCaseContext csc=UCASECONTEXT_INITIALIZER;
csc.p=(void *)src;
csc.limit=srcLength;
- return _caseMap(
- csm, ucase_toFullLower,
- dest, destCapacity,
+ toLower(
+ caseLocale, options,
src, &csc, 0, srcLength,
- pErrorCode);
+ sink, edits, errorCode);
}
-static int32_t U_CALLCONV
-ucasemap_internalUTF8ToUpper(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
+static void U_CALLCONV
+ucasemap_internalUTF8ToUpper(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_UNUSED
const uint8_t *src, int32_t srcLength,
- UErrorCode *pErrorCode) {
- int32_t locCache = csm->locCache;
- if (ucase_getCaseLocale(csm->locale, &locCache) == UCASE_LOC_GREEK) {
- return GreekUpper::toUpper(csm, dest, destCapacity, src, srcLength, pErrorCode);
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode) {
+ if (caseLocale == UCASE_LOC_GREEK) {
+ GreekUpper::toUpper(options, src, srcLength, sink, edits, errorCode);
+ } else {
+ UCaseContext csc=UCASECONTEXT_INITIALIZER;
+ csc.p=(void *)src;
+ csc.limit=srcLength;
+ toUpper(
+ caseLocale, options,
+ src, &csc, srcLength,
+ sink, edits, errorCode);
}
- UCaseContext csc=UCASECONTEXT_INITIALIZER;
- csc.p=(void *)src;
- csc.limit=srcLength;
- return _caseMap(
- csm, ucase_toFullUpper,
- dest, destCapacity,
- src, &csc, 0, srcLength,
- pErrorCode);
}
-static int32_t
-utf8_foldCase(const UCaseProps *csp,
- uint8_t *dest, int32_t destCapacity,
- const uint8_t *src, int32_t srcLength,
- uint32_t options,
- UErrorCode *pErrorCode) {
- int32_t srcIndex, destIndex;
-
- const UChar *s;
- UChar32 c, c2;
- int32_t start;
-
- /* case mapping loop */
- srcIndex=destIndex=0;
- while(srcIndexdestCapacity) {
- *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
- }
- return destIndex;
-}
-
-static int32_t U_CALLCONV
-ucasemap_internalUTF8Fold(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
+static void U_CALLCONV
+ucasemap_internalUTF8Fold(int32_t /* caseLocale */, uint32_t options, UCASEMAP_BREAK_ITERATOR_UNUSED
const uint8_t *src, int32_t srcLength,
- UErrorCode *pErrorCode) {
- return utf8_foldCase(csm->csp, dest, destCapacity, src, srcLength, csm->options, pErrorCode);
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode) {
+ toLower(
+ -1, options,
+ src, nullptr, 0, srcLength,
+ sink, edits, errorCode);
}
-U_CFUNC int32_t
-ucasemap_mapUTF8(const UCaseMap *csm,
- uint8_t *dest, int32_t destCapacity,
- const uint8_t *src, int32_t srcLength,
+void
+ucasemap_mapUTF8(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ const char *src, int32_t srcLength,
UTF8CaseMapper *stringCaseMapper,
- UErrorCode *pErrorCode) {
- int32_t destLength;
-
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode) {
/* check argument values */
- if(U_FAILURE(*pErrorCode)) {
+ if (U_FAILURE(errorCode)) {
+ return;
+ }
+ if ((src == nullptr && srcLength != 0) || srcLength < -1) {
+ errorCode = U_ILLEGAL_ARGUMENT_ERROR;
+ return;
+ }
+
+ // Get the string length.
+ if (srcLength == -1) {
+ srcLength = (int32_t)uprv_strlen((const char *)src);
+ }
+
+ if (edits != nullptr && (options & U_EDITS_NO_RESET) == 0) {
+ edits->reset();
+ }
+ stringCaseMapper(caseLocale, options, UCASEMAP_BREAK_ITERATOR
+ (const uint8_t *)src, srcLength, sink, edits, errorCode);
+ sink.Flush();
+ if (U_SUCCESS(errorCode)) {
+ if (edits != nullptr) {
+ edits->copyErrorTo(errorCode);
+ }
+ }
+}
+
+int32_t
+ucasemap_mapUTF8(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ char *dest, int32_t destCapacity,
+ const char *src, int32_t srcLength,
+ UTF8CaseMapper *stringCaseMapper,
+ icu::Edits *edits,
+ UErrorCode &errorCode) {
+ /* check argument values */
+ if(U_FAILURE(errorCode)) {
return 0;
}
if( destCapacity<0 ||
(dest==NULL && destCapacity>0) ||
- src==NULL ||
- srcLength<-1
+ (src==NULL && srcLength!=0) || srcLength<-1
) {
- *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
+ errorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
@@ -719,12 +823,25 @@ ucasemap_mapUTF8(const UCaseMap *csm,
((src>=dest && src<(dest+destCapacity)) ||
(dest>=src && dest<(src+srcLength)))
) {
- *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
+ errorCode=U_ILLEGAL_ARGUMENT_ERROR;
return 0;
}
- destLength=stringCaseMapper(csm, dest, destCapacity, src, srcLength, pErrorCode);
- return u_terminateChars((char *)dest, destCapacity, destLength, pErrorCode);
+ CheckedArrayByteSink sink(dest, destCapacity);
+ if (edits != nullptr && (options & U_EDITS_NO_RESET) == 0) {
+ edits->reset();
+ }
+ stringCaseMapper(caseLocale, options, UCASEMAP_BREAK_ITERATOR
+ (const uint8_t *)src, srcLength, sink, edits, errorCode);
+ sink.Flush();
+ if (U_SUCCESS(errorCode)) {
+ if (sink.Overflowed()) {
+ errorCode = U_BUFFER_OVERFLOW_ERROR;
+ } else if (edits != nullptr) {
+ edits->copyErrorTo(errorCode);
+ }
+ }
+ return u_terminateChars(dest, destCapacity, sink.NumberOfBytesAppended(), &errorCode);
}
/* public API functions */
@@ -734,10 +851,11 @@ ucasemap_utf8ToLower(const UCaseMap *csm,
char *dest, int32_t destCapacity,
const char *src, int32_t srcLength,
UErrorCode *pErrorCode) {
- return ucasemap_mapUTF8(csm,
- (uint8_t *)dest, destCapacity,
- (const uint8_t *)src, srcLength,
- ucasemap_internalUTF8ToLower, pErrorCode);
+ return ucasemap_mapUTF8(
+ csm->caseLocale, csm->options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8ToLower, NULL, *pErrorCode);
}
U_CAPI int32_t U_EXPORT2
@@ -745,10 +863,11 @@ ucasemap_utf8ToUpper(const UCaseMap *csm,
char *dest, int32_t destCapacity,
const char *src, int32_t srcLength,
UErrorCode *pErrorCode) {
- return ucasemap_mapUTF8(csm,
- (uint8_t *)dest, destCapacity,
- (const uint8_t *)src, srcLength,
- ucasemap_internalUTF8ToUpper, pErrorCode);
+ return ucasemap_mapUTF8(
+ csm->caseLocale, csm->options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8ToUpper, NULL, *pErrorCode);
}
U_CAPI int32_t U_EXPORT2
@@ -756,8 +875,79 @@ ucasemap_utf8FoldCase(const UCaseMap *csm,
char *dest, int32_t destCapacity,
const char *src, int32_t srcLength,
UErrorCode *pErrorCode) {
- return ucasemap_mapUTF8(csm,
- (uint8_t *)dest, destCapacity,
- (const uint8_t *)src, srcLength,
- ucasemap_internalUTF8Fold, pErrorCode);
+ return ucasemap_mapUTF8(
+ UCASE_LOC_ROOT, csm->options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8Fold, NULL, *pErrorCode);
}
+
+U_NAMESPACE_BEGIN
+
+void CaseMap::utf8ToLower(
+ const char *locale, uint32_t options,
+ StringPiece src, ByteSink &sink, Edits *edits,
+ UErrorCode &errorCode) {
+ ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, UCASEMAP_BREAK_ITERATOR_NULL
+ src.data(), src.length(),
+ ucasemap_internalUTF8ToLower, sink, edits, errorCode);
+}
+
+void CaseMap::utf8ToUpper(
+ const char *locale, uint32_t options,
+ StringPiece src, ByteSink &sink, Edits *edits,
+ UErrorCode &errorCode) {
+ ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, UCASEMAP_BREAK_ITERATOR_NULL
+ src.data(), src.length(),
+ ucasemap_internalUTF8ToUpper, sink, edits, errorCode);
+}
+
+void CaseMap::utf8Fold(
+ uint32_t options,
+ StringPiece src, ByteSink &sink, Edits *edits,
+ UErrorCode &errorCode) {
+ ucasemap_mapUTF8(
+ UCASE_LOC_ROOT, options, UCASEMAP_BREAK_ITERATOR_NULL
+ src.data(), src.length(),
+ ucasemap_internalUTF8Fold, sink, edits, errorCode);
+}
+
+int32_t CaseMap::utf8ToLower(
+ const char *locale, uint32_t options,
+ const char *src, int32_t srcLength,
+ char *dest, int32_t destCapacity, Edits *edits,
+ UErrorCode &errorCode) {
+ return ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8ToLower, edits, errorCode);
+}
+
+int32_t CaseMap::utf8ToUpper(
+ const char *locale, uint32_t options,
+ const char *src, int32_t srcLength,
+ char *dest, int32_t destCapacity, Edits *edits,
+ UErrorCode &errorCode) {
+ return ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8ToUpper, edits, errorCode);
+}
+
+int32_t CaseMap::utf8Fold(
+ uint32_t options,
+ const char *src, int32_t srcLength,
+ char *dest, int32_t destCapacity, Edits *edits,
+ UErrorCode &errorCode) {
+ return ucasemap_mapUTF8(
+ UCASE_LOC_ROOT, options, UCASEMAP_BREAK_ITERATOR_NULL
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8Fold, edits, errorCode);
+}
+
+U_NAMESPACE_END
diff --git a/intl/icu/source/common/ucasemap_imp.h b/intl/icu/source/common/ucasemap_imp.h
new file mode 100644
index 0000000000..7788fd9371
--- /dev/null
+++ b/intl/icu/source/common/ucasemap_imp.h
@@ -0,0 +1,282 @@
+// © 2017 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+
+// ucasemap_imp.h
+// created: 2017feb08 Markus W. Scherer
+
+#ifndef __UCASEMAP_IMP_H__
+#define __UCASEMAP_IMP_H__
+
+#include "unicode/utypes.h"
+#include "unicode/ucasemap.h"
+#include "unicode/uchar.h"
+#include "ucase.h"
+
+/**
+ * Bit mask for the titlecasing iterator options bit field.
+ * Currently only 3 out of 8 values are used:
+ * 0 (words), U_TITLECASE_WHOLE_STRING, U_TITLECASE_SENTENCES.
+ * See stringoptions.h.
+ * @internal
+ */
+#define U_TITLECASE_ITERATOR_MASK 0xe0
+
+/**
+ * Bit mask for the titlecasing index adjustment options bit set.
+ * Currently two bits are defined:
+ * U_TITLECASE_NO_BREAK_ADJUSTMENT, U_TITLECASE_ADJUST_TO_CASED.
+ * See stringoptions.h.
+ * @internal
+ */
+#define U_TITLECASE_ADJUSTMENT_MASK 0x600
+
+/**
+ * Internal API, used by u_strcasecmp() etc.
+ * Compare strings case-insensitively,
+ * in code point order or code unit order.
+ */
+U_CFUNC int32_t
+u_strcmpFold(const UChar *s1, int32_t length1,
+ const UChar *s2, int32_t length2,
+ uint32_t options,
+ UErrorCode *pErrorCode);
+
+/**
+ * Internal API, used for detecting length of
+ * shared prefix case-insensitively.
+ * @param s1 input string 1
+ * @param length1 length of string 1, or -1 (NULL terminated)
+ * @param s2 input string 2
+ * @param length2 length of string 2, or -1 (NULL terminated)
+ * @param options compare options
+ * @param matchLen1 (output) length of partial prefix match in s1
+ * @param matchLen2 (output) length of partial prefix match in s2
+ * @param pErrorCode receives error status
+ */
+U_CAPI void
+u_caseInsensitivePrefixMatch(const UChar *s1, int32_t length1,
+ const UChar *s2, int32_t length2,
+ uint32_t options,
+ int32_t *matchLen1, int32_t *matchLen2,
+ UErrorCode *pErrorCode);
+
+#ifdef __cplusplus
+
+U_NAMESPACE_BEGIN
+
+class BreakIterator; // unicode/brkiter.h
+class ByteSink;
+class Locale; // unicode/locid.h
+
+/** Returns TRUE if the options are valid. Otherwise FALSE, and sets an error. */
+inline UBool ustrcase_checkTitleAdjustmentOptions(uint32_t options, UErrorCode &errorCode) {
+ if (U_FAILURE(errorCode)) { return FALSE; }
+ if ((options & U_TITLECASE_ADJUSTMENT_MASK) == U_TITLECASE_ADJUSTMENT_MASK) {
+ // Both options together.
+ errorCode = U_ILLEGAL_ARGUMENT_ERROR;
+ return FALSE;
+ }
+ return TRUE;
+}
+
+inline UBool ustrcase_isLNS(UChar32 c) {
+ // Letter, number, symbol,
+ // or a private use code point because those are typically used as letters or numbers.
+ // Consider modifier letters only if they are cased.
+ const uint32_t LNS = (U_GC_L_MASK|U_GC_N_MASK|U_GC_S_MASK|U_GC_CO_MASK) & ~U_GC_LM_MASK;
+ int gc = u_charType(c);
+ return (U_MASK(gc) & LNS) != 0 || (gc == U_MODIFIER_LETTER && ucase_getType(c) != UCASE_NONE);
+}
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+/** Returns nullptr if error. Pass in either locale or locID, not both. */
+U_CFUNC
+BreakIterator *ustrcase_getTitleBreakIterator(
+ const Locale *locale, const char *locID, uint32_t options, BreakIterator *iter,
+ LocalPointer &ownedIter, UErrorCode &errorCode);
+
+#endif
+
+U_NAMESPACE_END
+
+#include "unicode/unistr.h" // for UStringCaseMapper
+
+/*
+ * Internal string casing functions implementing
+ * ustring.h/ustrcase.cpp and UnicodeString case mapping functions.
+ */
+
+struct UCaseMap : public icu::UMemory {
+ /** Implements most of ucasemap_open(). */
+ UCaseMap(const char *localeID, uint32_t opts, UErrorCode *pErrorCode);
+ ~UCaseMap();
+
+#if !UCONFIG_NO_BREAK_ITERATION
+ icu::BreakIterator *iter; /* We adopt the iterator, so we own it. */
+#endif
+ char locale[32];
+ int32_t caseLocale;
+ uint32_t options;
+};
+
+#if UCONFIG_NO_BREAK_ITERATION
+# define UCASEMAP_BREAK_ITERATOR_PARAM
+# define UCASEMAP_BREAK_ITERATOR_UNUSED
+# define UCASEMAP_BREAK_ITERATOR
+# define UCASEMAP_BREAK_ITERATOR_NULL
+#else
+# define UCASEMAP_BREAK_ITERATOR_PARAM icu::BreakIterator *iter,
+# define UCASEMAP_BREAK_ITERATOR_UNUSED icu::BreakIterator *,
+# define UCASEMAP_BREAK_ITERATOR iter,
+# define UCASEMAP_BREAK_ITERATOR_NULL NULL,
+#endif
+
+U_CFUNC int32_t
+ustrcase_getCaseLocale(const char *locale);
+
+// TODO: swap src / dest if approved for new public api
+/** Implements UStringCaseMapper. */
+U_CFUNC int32_t U_CALLCONV
+ustrcase_internalToLower(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+/** Implements UStringCaseMapper. */
+U_CFUNC int32_t U_CALLCONV
+ustrcase_internalToUpper(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+/** Implements UStringCaseMapper. */
+U_CFUNC int32_t U_CALLCONV
+ustrcase_internalToTitle(int32_t caseLocale, uint32_t options,
+ icu::BreakIterator *iter,
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+#endif
+
+/** Implements UStringCaseMapper. */
+U_CFUNC int32_t U_CALLCONV
+ustrcase_internalFold(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+/**
+ * Common string case mapping implementation for ucasemap_toXyz() and UnicodeString::toXyz().
+ * Implements argument checking.
+ */
+U_CFUNC int32_t
+ustrcase_map(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ UStringCaseMapper *stringCaseMapper,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+/**
+ * Common string case mapping implementation for old-fashioned u_strToXyz() functions
+ * that allow the source string to overlap the destination buffer.
+ * Implements argument checking and internally works with an intermediate buffer if necessary.
+ */
+U_CFUNC int32_t
+ustrcase_mapWithOverlap(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ UStringCaseMapper *stringCaseMapper,
+ UErrorCode &errorCode);
+
+/**
+ * UTF-8 string case mapping function type, used by ucasemap_mapUTF8().
+ * UTF-8 version of UStringCaseMapper.
+ * All error checking must be done.
+ * The UCaseMap must be fully initialized, with locale and/or iter set as needed.
+ */
+typedef void U_CALLCONV
+UTF8CaseMapper(int32_t caseLocale, uint32_t options,
+#if !UCONFIG_NO_BREAK_ITERATION
+ icu::BreakIterator *iter,
+#endif
+ const uint8_t *src, int32_t srcLength,
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode);
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+/** Implements UTF8CaseMapper. */
+U_CFUNC void U_CALLCONV
+ucasemap_internalUTF8ToTitle(int32_t caseLocale, uint32_t options,
+ icu::BreakIterator *iter,
+ const uint8_t *src, int32_t srcLength,
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode);
+
+#endif
+
+void
+ucasemap_mapUTF8(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ const char *src, int32_t srcLength,
+ UTF8CaseMapper *stringCaseMapper,
+ icu::ByteSink &sink, icu::Edits *edits,
+ UErrorCode &errorCode);
+
+/**
+ * Implements argument checking and buffer handling
+ * for UTF-8 string case mapping as a common function.
+ */
+int32_t
+ucasemap_mapUTF8(int32_t caseLocale, uint32_t options, UCASEMAP_BREAK_ITERATOR_PARAM
+ char *dest, int32_t destCapacity,
+ const char *src, int32_t srcLength,
+ UTF8CaseMapper *stringCaseMapper,
+ icu::Edits *edits,
+ UErrorCode &errorCode);
+
+U_NAMESPACE_BEGIN
+namespace GreekUpper {
+
+// Data bits.
+static const uint32_t UPPER_MASK = 0x3ff;
+static const uint32_t HAS_VOWEL = 0x1000;
+static const uint32_t HAS_YPOGEGRAMMENI = 0x2000;
+static const uint32_t HAS_ACCENT = 0x4000;
+static const uint32_t HAS_DIALYTIKA = 0x8000;
+// Further bits during data building and processing, not stored in the data map.
+static const uint32_t HAS_COMBINING_DIALYTIKA = 0x10000;
+static const uint32_t HAS_OTHER_GREEK_DIACRITIC = 0x20000;
+
+static const uint32_t HAS_VOWEL_AND_ACCENT = HAS_VOWEL | HAS_ACCENT;
+static const uint32_t HAS_VOWEL_AND_ACCENT_AND_DIALYTIKA =
+ HAS_VOWEL_AND_ACCENT | HAS_DIALYTIKA;
+static const uint32_t HAS_EITHER_DIALYTIKA = HAS_DIALYTIKA | HAS_COMBINING_DIALYTIKA;
+
+// State bits.
+static const uint32_t AFTER_CASED = 1;
+static const uint32_t AFTER_VOWEL_WITH_ACCENT = 2;
+
+uint32_t getLetterData(UChar32 c);
+
+/**
+ * Returns a non-zero value for each of the Greek combining diacritics
+ * listed in The Unicode Standard, version 8, chapter 7.2 Greek,
+ * plus some perispomeni look-alikes.
+ */
+uint32_t getDiacriticData(UChar32 c);
+
+} // namespace GreekUpper
+U_NAMESPACE_END
+
+#endif // __cplusplus
+
+#endif // __UCASEMAP_IMP_H__
diff --git a/intl/icu/source/common/ucasemap_titlecase_brkiter.cpp b/intl/icu/source/common/ucasemap_titlecase_brkiter.cpp
index ab61e21765..c21dfb7698 100644
--- a/intl/icu/source/common/ucasemap_titlecase_brkiter.cpp
+++ b/intl/icu/source/common/ucasemap_titlecase_brkiter.cpp
@@ -1,4 +1,4 @@
-// Copyright (C) 2016 and later: Unicode, Inc. and others.
+// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
@@ -6,7 +6,7 @@
* Corporation and others. All Rights Reserved.
*******************************************************************************
* file name: ucasemap_titlecase_brkiter.cpp
-* encoding: US-ASCII
+* encoding: UTF-8
* tab size: 8 (not used)
* indentation:4
*
@@ -23,23 +23,79 @@
#include "unicode/brkiter.h"
#include "unicode/ubrk.h"
+#include "unicode/casemap.h"
#include "unicode/ucasemap.h"
#include "cmemory.h"
#include "ucase.h"
-#include "ustr_imp.h"
+#include "ucasemap_imp.h"
+
+U_NAMESPACE_BEGIN
+
+void CaseMap::utf8ToTitle(
+ const char *locale, uint32_t options, BreakIterator *iter,
+ StringPiece src, ByteSink &sink, Edits *edits,
+ UErrorCode &errorCode) {
+ if (U_FAILURE(errorCode)) {
+ return;
+ }
+ UText utext = UTEXT_INITIALIZER;
+ utext_openUTF8(&utext, src.data(), src.length(), &errorCode);
+ LocalPointer ownedIter;
+ iter = ustrcase_getTitleBreakIterator(nullptr, locale, options, iter, ownedIter, errorCode);
+ if (iter == nullptr) {
+ utext_close(&utext);
+ return;
+ }
+ iter->setText(&utext, errorCode);
+ ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, iter,
+ src.data(), src.length(),
+ ucasemap_internalUTF8ToTitle, sink, edits, errorCode);
+ utext_close(&utext);
+}
+
+int32_t CaseMap::utf8ToTitle(
+ const char *locale, uint32_t options, BreakIterator *iter,
+ const char *src, int32_t srcLength,
+ char *dest, int32_t destCapacity, Edits *edits,
+ UErrorCode &errorCode) {
+ if (U_FAILURE(errorCode)) {
+ return 0;
+ }
+ UText utext=UTEXT_INITIALIZER;
+ utext_openUTF8(&utext, src, srcLength, &errorCode);
+ LocalPointer ownedIter;
+ iter = ustrcase_getTitleBreakIterator(nullptr, locale, options, iter, ownedIter, errorCode);
+ if(iter==NULL) {
+ utext_close(&utext);
+ return 0;
+ }
+ iter->setText(&utext, errorCode);
+ int32_t length=ucasemap_mapUTF8(
+ ustrcase_getCaseLocale(locale), options, iter,
+ dest, destCapacity,
+ src, srcLength,
+ ucasemap_internalUTF8ToTitle, edits, errorCode);
+ utext_close(&utext);
+ return length;
+}
+
+U_NAMESPACE_END
U_NAMESPACE_USE
U_CAPI const UBreakIterator * U_EXPORT2
ucasemap_getBreakIterator(const UCaseMap *csm) {
- return csm->iter;
+ return reinterpret_cast(csm->iter);
}
U_CAPI void U_EXPORT2
-ucasemap_setBreakIterator(UCaseMap *csm, UBreakIterator *iterToAdopt, UErrorCode * /*pErrorCode*/) {
- // Do not call ubrk_close() so that we do not depend on all of the BreakIterator code.
- delete reinterpret_cast(csm->iter);
- csm->iter=iterToAdopt;
+ucasemap_setBreakIterator(UCaseMap *csm, UBreakIterator *iterToAdopt, UErrorCode *pErrorCode) {
+ if(U_FAILURE(*pErrorCode)) {
+ return;
+ }
+ delete csm->iter;
+ csm->iter=reinterpret_cast(iterToAdopt);
}
U_CAPI int32_t U_EXPORT2
@@ -47,21 +103,30 @@ ucasemap_utf8ToTitle(UCaseMap *csm,
char *dest, int32_t destCapacity,
const char *src, int32_t srcLength,
UErrorCode *pErrorCode) {
+ if (U_FAILURE(*pErrorCode)) {
+ return 0;
+ }
UText utext=UTEXT_INITIALIZER;
utext_openUTF8(&utext, (const char *)src, srcLength, pErrorCode);
- if(U_FAILURE(*pErrorCode)) {
+ if (U_FAILURE(*pErrorCode)) {
return 0;
}
if(csm->iter==NULL) {
- csm->iter=ubrk_open(UBRK_WORD, csm->locale,
- NULL, 0,
- pErrorCode);
+ LocalPointer