mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-09-21 15:57:31 +09:00
Issue #1971 - Part 2: Update ICU source to 63.2.
This commit is contained in:
Job Bautista
roytam1
parent
8df84683be
commit
1e69214382
3160 changed files with 275815 additions and 234203 deletions
|
|
@ -24,10 +24,10 @@ CLEANFILES = *~ $(DEPS)
|
|||
## Target information
|
||||
TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT)
|
||||
|
||||
CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil
|
||||
CPPFLAGS += -I$(srcdir) -I$(top_srcdir)/common -I$(srcdir)/../toolutil
|
||||
LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M)
|
||||
|
||||
OBJECTS = gennorm2.o n2builder.o
|
||||
OBJECTS = gennorm2.o n2builder.o extradata.o norms.o
|
||||
|
||||
DEPS = $(OBJECTS:.o=.d)
|
||||
|
||||
|
|
|
|||
253
intl/icu/source/tools/gennorm2/extradata.cpp
Normal file
253
intl/icu/source/tools/gennorm2/extradata.cpp
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
// © 2017 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
// extradata.cpp
|
||||
// created: 2017jun04 Markus W. Scherer
|
||||
// (pulled out of n2builder.cpp)
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/utf16.h"
|
||||
#include "extradata.h"
|
||||
#include "normalizer2impl.h"
|
||||
#include "norms.h"
|
||||
#include "toolutil.h"
|
||||
#include "utrie2.h"
|
||||
#include "uvectr32.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
ExtraData::ExtraData(Norms &n, UBool fast) :
|
||||
Norms::Enumerator(n),
|
||||
yesYesCompositions(1000, (UChar32)0xffff, 2), // 0=inert, 1=Jamo L, 2=start of compositions
|
||||
yesNoMappingsAndCompositions(1000, (UChar32)0, 1), // 0=Hangul LV, 1=start of normal data
|
||||
yesNoMappingsOnly(1000, (UChar32)0, 1), // 0=Hangul LVT, 1=start of normal data
|
||||
optimizeFast(fast) {
|
||||
// Hangul LV algorithmically decomposes to two Jamo.
|
||||
// Some code may harmlessly read this firstUnit.
|
||||
yesNoMappingsAndCompositions.setCharAt(0, 2);
|
||||
// Hangul LVT algorithmically decomposes to three Jamo.
|
||||
// Some code may harmlessly read this firstUnit.
|
||||
yesNoMappingsOnly.setCharAt(0, 3);
|
||||
}
|
||||
|
||||
int32_t ExtraData::writeMapping(UChar32 c, const Norm &norm, UnicodeString &dataString) {
|
||||
UnicodeString &m=*norm.mapping;
|
||||
int32_t length=m.length();
|
||||
// Write the mapping & raw mapping extraData.
|
||||
int32_t firstUnit=length|(norm.trailCC<<8);
|
||||
int32_t preMappingLength=0;
|
||||
if(norm.rawMapping!=NULL) {
|
||||
UnicodeString &rm=*norm.rawMapping;
|
||||
int32_t rmLength=rm.length();
|
||||
if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"raw mapping for U+%04lX longer than maximum of %d\n",
|
||||
(long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
UChar rm0=rm.charAt(0);
|
||||
if( rmLength==length-1 &&
|
||||
// 99: overlong substring lengths get pinned to remainder lengths anyway
|
||||
0==rm.compare(1, 99, m, 2, 99) &&
|
||||
rm0>Normalizer2Impl::MAPPING_LENGTH_MASK
|
||||
) {
|
||||
// Compression:
|
||||
// rawMapping=rm0+mapping.substring(2) -> store only rm0
|
||||
//
|
||||
// The raw mapping is the same as the final mapping after replacing
|
||||
// the final mapping's first two code units with the raw mapping's first one.
|
||||
// In this case, we store only that first unit, rm0.
|
||||
// This helps with a few hundred mappings.
|
||||
dataString.append(rm0);
|
||||
preMappingLength=1;
|
||||
} else {
|
||||
// Store the raw mapping with its length.
|
||||
dataString.append(rm);
|
||||
dataString.append((UChar)rmLength);
|
||||
preMappingLength=rmLength+1;
|
||||
}
|
||||
firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING;
|
||||
}
|
||||
int32_t cccLccc=norm.cc|(norm.leadCC<<8);
|
||||
if(cccLccc!=0) {
|
||||
dataString.append((UChar)cccLccc);
|
||||
++preMappingLength;
|
||||
firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD;
|
||||
}
|
||||
dataString.append((UChar)firstUnit);
|
||||
dataString.append(m);
|
||||
return preMappingLength;
|
||||
}
|
||||
|
||||
int32_t ExtraData::writeNoNoMapping(UChar32 c, const Norm &norm,
|
||||
UnicodeString &dataString,
|
||||
Hashtable &previousMappings) {
|
||||
UnicodeString newMapping;
|
||||
int32_t offset=writeMapping(c, norm, newMapping);
|
||||
int32_t previousOffset=previousMappings.geti(newMapping);
|
||||
if(previousOffset!=0) {
|
||||
// Duplicate, point to the identical mapping that has already been stored.
|
||||
offset=previousOffset-1;
|
||||
} else {
|
||||
// Append this new mapping and
|
||||
// enter it into the hashtable, avoiding value 0 which is "not found".
|
||||
offset=dataString.length()+offset;
|
||||
dataString.append(newMapping);
|
||||
IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.puti()");
|
||||
previousMappings.puti(newMapping, offset+1, errorCode);
|
||||
}
|
||||
return offset;
|
||||
}
|
||||
|
||||
UBool ExtraData::setNoNoDelta(UChar32 c, Norm &norm) const {
|
||||
// Try a compact, algorithmic encoding to a single compYesAndZeroCC code point.
|
||||
// Do not map from ASCII to non-ASCII.
|
||||
if(norm.mappingCP>=0 &&
|
||||
!(c<=0x7f && norm.mappingCP>0x7f) &&
|
||||
norms.getNormRef(norm.mappingCP).type<Norm::NO_NO_COMP_YES) {
|
||||
int32_t delta=norm.mappingCP-c;
|
||||
if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) {
|
||||
norm.type=Norm::NO_NO_DELTA;
|
||||
norm.offset=delta;
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
void ExtraData::writeCompositions(UChar32 c, const Norm &norm, UnicodeString &dataString) {
|
||||
if(norm.cc!=0) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n",
|
||||
(long)c);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
int32_t length;
|
||||
const CompositionPair *pairs=norm.getCompositionPairs(length);
|
||||
for(int32_t i=0; i<length; ++i) {
|
||||
const CompositionPair &pair=pairs[i];
|
||||
// 22 bits for the composite character and whether it combines forward.
|
||||
UChar32 compositeAndFwd=pair.composite<<1;
|
||||
if(norms.getNormRef(pair.composite).compositions!=NULL) {
|
||||
compositeAndFwd|=1; // The composite character also combines-forward.
|
||||
}
|
||||
// Encode most pairs in two units and some in three.
|
||||
int32_t firstUnit, secondUnit, thirdUnit;
|
||||
if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) {
|
||||
if(compositeAndFwd<=0xffff) {
|
||||
firstUnit=pair.trail<<1;
|
||||
secondUnit=compositeAndFwd;
|
||||
thirdUnit=-1;
|
||||
} else {
|
||||
firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE;
|
||||
secondUnit=compositeAndFwd>>16;
|
||||
thirdUnit=compositeAndFwd;
|
||||
}
|
||||
} else {
|
||||
firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+
|
||||
(pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))|
|
||||
Normalizer2Impl::COMP_1_TRIPLE;
|
||||
secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)|
|
||||
(compositeAndFwd>>16);
|
||||
thirdUnit=compositeAndFwd;
|
||||
}
|
||||
// Set the high bit of the first unit if this is the last composition pair.
|
||||
if(i==(length-1)) {
|
||||
firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE;
|
||||
}
|
||||
dataString.append((UChar)firstUnit).append((UChar)secondUnit);
|
||||
if(thirdUnit>=0) {
|
||||
dataString.append((UChar)thirdUnit);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ExtraData::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
|
||||
if(start!=end) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: unexpected shared data for "
|
||||
"multiple code points U+%04lX..U+%04lX\n",
|
||||
(long)start, (long)end);
|
||||
exit(U_INTERNAL_PROGRAM_ERROR);
|
||||
}
|
||||
if(norm.error!=nullptr) {
|
||||
fprintf(stderr, "gennorm2 error: U+%04lX %s\n", (long)start, norm.error);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
writeExtraData(start, norm);
|
||||
}
|
||||
|
||||
// Ticket #13342 - Disable optimizations on MSVC for this function as a workaround.
|
||||
#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210))
|
||||
#pragma optimize( "", off )
|
||||
#endif
|
||||
|
||||
void ExtraData::writeExtraData(UChar32 c, Norm &norm) {
|
||||
switch(norm.type) {
|
||||
case Norm::INERT:
|
||||
break; // no extra data
|
||||
case Norm::YES_YES_COMBINES_FWD:
|
||||
norm.offset=yesYesCompositions.length();
|
||||
writeCompositions(c, norm, yesYesCompositions);
|
||||
break;
|
||||
case Norm::YES_NO_COMBINES_FWD:
|
||||
norm.offset=yesNoMappingsAndCompositions.length()+
|
||||
writeMapping(c, norm, yesNoMappingsAndCompositions);
|
||||
writeCompositions(c, norm, yesNoMappingsAndCompositions);
|
||||
break;
|
||||
case Norm::YES_NO_MAPPING_ONLY:
|
||||
norm.offset=yesNoMappingsOnly.length()+
|
||||
writeMapping(c, norm, yesNoMappingsOnly);
|
||||
break;
|
||||
case Norm::NO_NO_COMP_YES:
|
||||
if(!optimizeFast && setNoNoDelta(c, norm)) {
|
||||
break;
|
||||
}
|
||||
norm.offset=writeNoNoMapping(c, norm, noNoMappingsCompYes, previousNoNoMappingsCompYes);
|
||||
break;
|
||||
case Norm::NO_NO_COMP_BOUNDARY_BEFORE:
|
||||
if(!optimizeFast && setNoNoDelta(c, norm)) {
|
||||
break;
|
||||
}
|
||||
norm.offset=writeNoNoMapping(
|
||||
c, norm, noNoMappingsCompBoundaryBefore, previousNoNoMappingsCompBoundaryBefore);
|
||||
break;
|
||||
case Norm::NO_NO_COMP_NO_MAYBE_CC:
|
||||
norm.offset=writeNoNoMapping(
|
||||
c, norm, noNoMappingsCompNoMaybeCC, previousNoNoMappingsCompNoMaybeCC);
|
||||
break;
|
||||
case Norm::NO_NO_EMPTY:
|
||||
// There can be multiple extra data entries for mappings to the empty string
|
||||
// if they have different raw mappings.
|
||||
norm.offset=writeNoNoMapping(c, norm, noNoMappingsEmpty, previousNoNoMappingsEmpty);
|
||||
break;
|
||||
case Norm::MAYBE_YES_COMBINES_FWD:
|
||||
norm.offset=maybeYesCompositions.length();
|
||||
writeCompositions(c, norm, maybeYesCompositions);
|
||||
break;
|
||||
case Norm::MAYBE_YES_SIMPLE:
|
||||
break; // no extra data
|
||||
case Norm::YES_YES_WITH_CC:
|
||||
break; // no extra data
|
||||
default: // Should not occur.
|
||||
exit(U_INTERNAL_PROGRAM_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
// Ticket #13342 - Turn optimization back on.
|
||||
#if (defined(_MSC_VER) && (_MSC_VER >= 1900) && defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 190024210))
|
||||
#pragma optimize( "", on )
|
||||
#endif
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif // #if !UCONFIG_NO_NORMALIZATION
|
||||
70
intl/icu/source/tools/gennorm2/extradata.h
Normal file
70
intl/icu/source/tools/gennorm2/extradata.h
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
// © 2017 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
// extradata.h
|
||||
// created: 2017jun04 Markus W. Scherer
|
||||
// (pulled out of n2builder.cpp)
|
||||
|
||||
// Write mappings and compositions in compact form for Normalizer2 "extra data",
|
||||
// the data that does not fit into the trie itself.
|
||||
|
||||
#ifndef __EXTRADATA_H__
|
||||
#define __EXTRADATA_H__
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/utf16.h"
|
||||
#include "hash.h"
|
||||
#include "norms.h"
|
||||
#include "toolutil.h"
|
||||
#include "utrie2.h"
|
||||
#include "uvectr32.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class ExtraData : public Norms::Enumerator {
|
||||
public:
|
||||
ExtraData(Norms &n, UBool fast);
|
||||
|
||||
void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE;
|
||||
|
||||
UnicodeString maybeYesCompositions;
|
||||
UnicodeString yesYesCompositions;
|
||||
UnicodeString yesNoMappingsAndCompositions;
|
||||
UnicodeString yesNoMappingsOnly;
|
||||
UnicodeString noNoMappingsCompYes;
|
||||
UnicodeString noNoMappingsCompBoundaryBefore;
|
||||
UnicodeString noNoMappingsCompNoMaybeCC;
|
||||
UnicodeString noNoMappingsEmpty;
|
||||
|
||||
private:
|
||||
/**
|
||||
* Requires norm.hasMapping().
|
||||
* Returns the offset of the "first unit" from the beginning of the extraData for c.
|
||||
* That is the same as the length of the optional data
|
||||
* for the raw mapping and the ccc/lccc word.
|
||||
*/
|
||||
int32_t writeMapping(UChar32 c, const Norm &norm, UnicodeString &dataString);
|
||||
int32_t writeNoNoMapping(UChar32 c, const Norm &norm,
|
||||
UnicodeString &dataString, Hashtable &previousMappings);
|
||||
UBool setNoNoDelta(UChar32 c, Norm &norm) const;
|
||||
/** Requires norm.compositions!=nullptr. */
|
||||
void writeCompositions(UChar32 c, const Norm &norm, UnicodeString &dataString);
|
||||
void writeExtraData(UChar32 c, Norm &norm);
|
||||
|
||||
UBool optimizeFast;
|
||||
Hashtable previousNoNoMappingsCompYes; // If constructed in runtime code, pass in UErrorCode.
|
||||
Hashtable previousNoNoMappingsCompBoundaryBefore;
|
||||
Hashtable previousNoNoMappingsCompNoMaybeCC;
|
||||
Hashtable previousNoNoMappingsEmpty;
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif // #if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#endif // __EXTRADATA_H__
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright (C) 2016 and later: Unicode, Inc. and others.
|
||||
// © 2016 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
/*
|
||||
*******************************************************************************
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
*
|
||||
*******************************************************************************
|
||||
* file name: gennorm2.cpp
|
||||
* encoding: US-ASCII
|
||||
* encoding: UTF-8
|
||||
* tab size: 8 (not used)
|
||||
* indentation:4
|
||||
*
|
||||
|
|
@ -22,8 +22,10 @@
|
|||
#include "unicode/utypes.h"
|
||||
#include "n2builder.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string>
|
||||
#include <string.h>
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/localpointer.h"
|
||||
|
|
@ -44,10 +46,8 @@ U_NAMESPACE_BEGIN
|
|||
|
||||
UBool beVerbose=FALSE, haveCopyright=TRUE;
|
||||
|
||||
U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose);
|
||||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
void parseFile(FILE *f, Normalizer2DataBuilder &builder);
|
||||
void parseFile(std::ifstream &f, Normalizer2DataBuilder &builder);
|
||||
#endif
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
|
@ -61,6 +61,7 @@ enum {
|
|||
OUTPUT_FILENAME,
|
||||
UNICODE_VERSION,
|
||||
WRITE_C_SOURCE,
|
||||
WRITE_COMBINED_DATA,
|
||||
OPT_FAST
|
||||
};
|
||||
|
||||
|
|
@ -73,6 +74,7 @@ static UOption options[]={
|
|||
UOPTION_DEF("output", 'o', UOPT_REQUIRES_ARG),
|
||||
UOPTION_DEF("unicode", 'u', UOPT_REQUIRES_ARG),
|
||||
UOPTION_DEF("csource", '\1', UOPT_NO_ARG),
|
||||
UOPTION_DEF("combined", '\1', UOPT_NO_ARG),
|
||||
UOPTION_DEF("fast", '\1', UOPT_NO_ARG)
|
||||
};
|
||||
|
||||
|
|
@ -96,17 +98,22 @@ main(int argc, char* argv[]) {
|
|||
if( argc<2 ||
|
||||
options[HELP_H].doesOccur || options[HELP_QUESTION_MARK].doesOccur
|
||||
) {
|
||||
/*
|
||||
* Broken into chunks because the C89 standard says the minimum
|
||||
* required supported string length is 509 bytes.
|
||||
*/
|
||||
fprintf(stderr,
|
||||
"Usage: %s [-options] infiles+ -o outputfilename\n"
|
||||
"\n"
|
||||
"Reads the infiles with normalization data and\n"
|
||||
"creates a binary or C source file (outputfilename) with the data.\n"
|
||||
"creates a binary file, or a C source file (--csource), with the data,\n"
|
||||
"or writes a data file with the combined data (--combined).\n"
|
||||
"See http://userguide.icu-project.org/transforms/normalization#TOC-Data-File-Syntax\n"
|
||||
"\n"
|
||||
"Alternate usage: %s [-options] a.txt b.txt minus p.txt q.txt -o outputfilename\n"
|
||||
"\n"
|
||||
"Computes the difference of (a, b) minus (p, q) and writes the diff data\n"
|
||||
"in input-file syntax to the outputfilename.\n"
|
||||
"It is then possible to build (p, q, diff) to get the same data as (a, b).\n"
|
||||
"(Useful for computing minimal incremental mapping data files.)\n"
|
||||
"\n",
|
||||
argv[0]);
|
||||
argv[0], argv[0]);
|
||||
fprintf(stderr,
|
||||
"Options:\n"
|
||||
"\t-h or -? or --help this usage text\n"
|
||||
|
|
@ -116,7 +123,9 @@ main(int argc, char* argv[]) {
|
|||
fprintf(stderr,
|
||||
"\t-s or --sourcedir source directory, followed by the path\n"
|
||||
"\t-o or --output output filename\n"
|
||||
"\t --csource writes a C source file with initializers\n");
|
||||
"\t --csource writes a C source file with initializers\n"
|
||||
"\t --combined writes a .txt file (input-file syntax) with the\n"
|
||||
"\t combined data from all of the input files\n");
|
||||
fprintf(stderr,
|
||||
"\t --fast optimize the data for fast normalization,\n"
|
||||
"\t which might increase its size (Writes fully decomposed\n"
|
||||
|
|
@ -144,7 +153,10 @@ main(int argc, char* argv[]) {
|
|||
|
||||
#else
|
||||
|
||||
LocalPointer<Normalizer2DataBuilder> builder(new Normalizer2DataBuilder(errorCode), errorCode);
|
||||
LocalPointer<Normalizer2DataBuilder> b1(new Normalizer2DataBuilder(errorCode), errorCode);
|
||||
LocalPointer<Normalizer2DataBuilder> b2;
|
||||
LocalPointer<Normalizer2DataBuilder> diff;
|
||||
Normalizer2DataBuilder *builder = b1.getAlias();
|
||||
errorCode.assertSuccess();
|
||||
|
||||
if(options[UNICODE_VERSION].doesOccur) {
|
||||
|
|
@ -166,20 +178,46 @@ main(int argc, char* argv[]) {
|
|||
pathLength=filename.length();
|
||||
}
|
||||
|
||||
bool doMinus = false;
|
||||
for(int i=1; i<argc; ++i) {
|
||||
printf("gennorm2: processing %s\n", argv[i]);
|
||||
if(strcmp(argv[i], "minus") == 0) {
|
||||
if(doMinus) {
|
||||
fprintf(stderr, "gennorm2 error: only one 'minus' can be specified\n");
|
||||
exit(U_ILLEGAL_ARGUMENT_ERROR);
|
||||
}
|
||||
// Data from previous input files has been collected in b1.
|
||||
// Collect data from further input files in b2.
|
||||
b2.adoptInsteadAndCheckErrorCode(new Normalizer2DataBuilder(errorCode), errorCode);
|
||||
diff.adoptInsteadAndCheckErrorCode(new Normalizer2DataBuilder(errorCode), errorCode);
|
||||
errorCode.assertSuccess();
|
||||
builder = b2.getAlias();
|
||||
if(options[UNICODE_VERSION].doesOccur) {
|
||||
builder->setUnicodeVersion(options[UNICODE_VERSION].value);
|
||||
}
|
||||
if(options[OPT_FAST].doesOccur) {
|
||||
builder->setOptimization(Normalizer2DataBuilder::OPTIMIZE_FAST);
|
||||
}
|
||||
doMinus = true;
|
||||
continue;
|
||||
}
|
||||
filename.append(argv[i], errorCode);
|
||||
LocalStdioFilePointer f(fopen(filename.data(), "r"));
|
||||
if(f==NULL) {
|
||||
std::ifstream f(filename.data());
|
||||
if(f.fail()) {
|
||||
fprintf(stderr, "gennorm2 error: unable to open %s\n", filename.data());
|
||||
exit(U_FILE_ACCESS_ERROR);
|
||||
}
|
||||
builder->setOverrideHandling(Normalizer2DataBuilder::OVERRIDE_PREVIOUS);
|
||||
parseFile(f.getAlias(), *builder);
|
||||
parseFile(f, *builder);
|
||||
filename.truncate(pathLength);
|
||||
}
|
||||
|
||||
if(options[WRITE_C_SOURCE].doesOccur) {
|
||||
if(doMinus) {
|
||||
Normalizer2DataBuilder::computeDiff(*b1, *b2, *diff);
|
||||
diff->writeDataFile(options[OUTPUT_FILENAME].value, /* writeRemoved= */ true);
|
||||
} else if(options[WRITE_COMBINED_DATA].doesOccur) {
|
||||
builder->writeDataFile(options[OUTPUT_FILENAME].value, /* writeRemoved= */ false);
|
||||
} else if(options[WRITE_C_SOURCE].doesOccur) {
|
||||
builder->writeCSourceFile(options[OUTPUT_FILENAME].value);
|
||||
} else {
|
||||
builder->writeBinaryFile(options[OUTPUT_FILENAME].value);
|
||||
|
|
@ -192,11 +230,19 @@ main(int argc, char* argv[]) {
|
|||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
void parseFile(FILE *f, Normalizer2DataBuilder &builder) {
|
||||
void parseFile(std::ifstream &f, Normalizer2DataBuilder &builder) {
|
||||
IcuToolErrorCode errorCode("gennorm2/parseFile()");
|
||||
char line[300];
|
||||
std::string lineString;
|
||||
uint32_t startCP, endCP;
|
||||
while(NULL!=fgets(line, (int)sizeof(line), f)) {
|
||||
while(std::getline(f, lineString)) {
|
||||
if (lineString.empty()) {
|
||||
continue; // skip empty lines.
|
||||
}
|
||||
#if (U_CPLUSPLUS_VERSION >= 11)
|
||||
char *line = &lineString.front();
|
||||
#else
|
||||
char *line = &lineString.at(0);
|
||||
#endif
|
||||
char *comment=(char *)strchr(line, '#');
|
||||
if(comment!=NULL) {
|
||||
*comment=0;
|
||||
|
|
@ -220,6 +266,11 @@ void parseFile(FILE *f, Normalizer2DataBuilder &builder) {
|
|||
fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line);
|
||||
exit(errorCode.reset());
|
||||
}
|
||||
if (endCP >= 0xd800 && startCP <= 0xdfff) {
|
||||
fprintf(stderr, "gennorm2 error: value or mapping for surrogate code points: %s\n",
|
||||
line);
|
||||
exit(U_ILLEGAL_ARGUMENT_ERROR);
|
||||
}
|
||||
delimiter=u_skipWhitespace(delimiter);
|
||||
if(*delimiter==':') {
|
||||
const char *s=u_skipWhitespace(delimiter+1);
|
||||
|
|
|
|||
|
|
@ -1,50 +1,20 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|Win32">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|Win32">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>Win32</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<!-- The following import will include the 'default' configuration options for VS projects. -->
|
||||
<Import Project="..\..\allinone\Build.Windows.ProjectConfiguration.props" />
|
||||
|
||||
<PropertyGroup Label="Globals">
|
||||
<ProjectGuid>{C7891A65-80AB-4245-912E-5F1E17B0E6C4}</ProjectGuid>
|
||||
<RootNamespace>gennorm2</RootNamespace>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
|
||||
<PropertyGroup Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<PlatformToolset>v140</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -77,6 +47,14 @@
|
|||
<IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">.\x64\Debug\</IntDir>
|
||||
<LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
|
||||
</PropertyGroup>
|
||||
<!-- Options that are common to *all* project configurations -->
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<DisableLanguageExtensions>false</DisableLanguageExtensions>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
</ClCompile>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
<CustomBuildStep>
|
||||
<Command>copy "$(TargetPath)" ..\..\..\bin
|
||||
|
|
@ -84,31 +62,22 @@
|
|||
<Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
|
||||
</CustomBuildStep>
|
||||
<ClCompile>
|
||||
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<StringPooling>true</StringPooling>
|
||||
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<DisableLanguageExtensions>false</DisableLanguageExtensions>
|
||||
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
|
||||
<PrecompiledHeaderOutputFile>.\x86\Release\gennorm2.pch</PrecompiledHeaderOutputFile>
|
||||
<AssemblerListingLocation>.\x86\Release\</AssemblerListingLocation>
|
||||
<ObjectFileName>.\x86\Release\</ObjectFileName>
|
||||
<ProgramDataBaseFileName>.\x86\Release\</ProgramDataBaseFileName>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<CompileAs>Default</CompileAs>
|
||||
</ClCompile>
|
||||
<ResourceCompile>
|
||||
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Culture>0x0409</Culture>
|
||||
</ResourceCompile>
|
||||
<Link>
|
||||
<OutputFile>.\x86\Release\gennorm2.exe</OutputFile>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
<ProgramDatabaseFile>.\x86\Release\gennorm2.pdb</ProgramDatabaseFile>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<SubSystem>Console</SubSystem>
|
||||
|
|
@ -126,38 +95,23 @@
|
|||
<Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
|
||||
</CustomBuildStep>
|
||||
<ClCompile>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<StringPooling>true</StringPooling>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
<BufferSecurityCheck>true</BufferSecurityCheck>
|
||||
<DisableLanguageExtensions>false</DisableLanguageExtensions>
|
||||
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
|
||||
<PrecompiledHeaderOutputFile>.\x86\Debug\gennorm2.pch</PrecompiledHeaderOutputFile>
|
||||
<AssemblerListingLocation>.\x86\Debug\</AssemblerListingLocation>
|
||||
<ObjectFileName>.\x86\Debug\</ObjectFileName>
|
||||
<ProgramDataBaseFileName>.\x86\Debug\</ProgramDataBaseFileName>
|
||||
<BrowseInformation>true</BrowseInformation>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
|
||||
<CompileAs>Default</CompileAs>
|
||||
</ClCompile>
|
||||
<ResourceCompile>
|
||||
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Culture>0x0409</Culture>
|
||||
</ResourceCompile>
|
||||
<Link>
|
||||
<OutputFile>.\x86\Debug\gennorm2.exe</OutputFile>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<ProgramDatabaseFile>.\x86\Debug\gennorm2.pdb</ProgramDatabaseFile>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<TargetMachine>MachineX86</TargetMachine>
|
||||
<RandomizedBaseAddress>false</RandomizedBaseAddress>
|
||||
<DataExecutionPrevention>false</DataExecutionPrevention>
|
||||
</Link>
|
||||
|
|
@ -169,37 +123,25 @@
|
|||
<Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
|
||||
</CustomBuildStep>
|
||||
<ClCompile>
|
||||
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN64;WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Optimization>MaxSpeed</Optimization>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<StringPooling>true</StringPooling>
|
||||
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<DisableLanguageExtensions>false</DisableLanguageExtensions>
|
||||
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
|
||||
<PrecompiledHeaderOutputFile>.\x64\Release\gennorm2.pch</PrecompiledHeaderOutputFile>
|
||||
<AssemblerListingLocation>.\x64\Release\</AssemblerListingLocation>
|
||||
<ObjectFileName>.\x64\Release\</ObjectFileName>
|
||||
<ProgramDataBaseFileName>.\x64\Release\</ProgramDataBaseFileName>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<CompileAs>Default</CompileAs>
|
||||
</ClCompile>
|
||||
<ResourceCompile>
|
||||
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Culture>0x0409</Culture>
|
||||
</ResourceCompile>
|
||||
<Link>
|
||||
<OutputFile>.\x64\Release\gennorm2.exe</OutputFile>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
<ProgramDatabaseFile>.\x64\Release\gennorm2.pdb</ProgramDatabaseFile>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<TargetMachine>MachineX64</TargetMachine>
|
||||
<RandomizedBaseAddress>false</RandomizedBaseAddress>
|
||||
<DataExecutionPrevention>false</DataExecutionPrevention>
|
||||
</Link>
|
||||
|
|
@ -211,58 +153,38 @@
|
|||
<Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
|
||||
</CustomBuildStep>
|
||||
<ClCompile>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN64;WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<MinimalRebuild>true</MinimalRebuild>
|
||||
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
|
||||
<StringPooling>true</StringPooling>
|
||||
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
|
||||
<BufferSecurityCheck>true</BufferSecurityCheck>
|
||||
<DisableLanguageExtensions>false</DisableLanguageExtensions>
|
||||
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
|
||||
<PrecompiledHeaderOutputFile>.\x64\Debug\gennorm2.pch</PrecompiledHeaderOutputFile>
|
||||
<AssemblerListingLocation>.\x64\Debug\</AssemblerListingLocation>
|
||||
<ObjectFileName>.\x64\Debug\</ObjectFileName>
|
||||
<ProgramDataBaseFileName>.\x64\Debug\</ProgramDataBaseFileName>
|
||||
<BrowseInformation>true</BrowseInformation>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
|
||||
<CompileAs>Default</CompileAs>
|
||||
</ClCompile>
|
||||
<ResourceCompile>
|
||||
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<Culture>0x0409</Culture>
|
||||
</ResourceCompile>
|
||||
<Link>
|
||||
<OutputFile>.\x64\Debug\gennorm2.exe</OutputFile>
|
||||
<SuppressStartupBanner>true</SuppressStartupBanner>
|
||||
<AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<ProgramDatabaseFile>.\x64\Debug\gennorm2.pdb</ProgramDatabaseFile>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<TargetMachine>MachineX64</TargetMachine>
|
||||
<RandomizedBaseAddress>false</RandomizedBaseAddress>
|
||||
<DataExecutionPrevention>false</DataExecutionPrevention>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="extradata.cpp" />
|
||||
<ClCompile Include="gennorm2.cpp" />
|
||||
<ClCompile Include="n2builder.cpp" />
|
||||
<ClCompile Include="norms.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="extradata.h" />
|
||||
<ClInclude Include="n2builder.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\common\common.vcxproj">
|
||||
<Project>{73c0a65b-d1f2-4de1-b3a6-15dad2c23f3d}</Project>
|
||||
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\toolutil\toolutil.vcxproj">
|
||||
<Project>{6b231032-3cb5-4eed-9210-810d666a23a0}</Project>
|
||||
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
|
||||
</ProjectReference>
|
||||
<ClInclude Include="norms.h" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright (C) 2016 and later: Unicode, Inc. and others.
|
||||
// © 2016 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
/*
|
||||
*******************************************************************************
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
*
|
||||
*******************************************************************************
|
||||
* file name: n2builder.h
|
||||
* encoding: US-ASCII
|
||||
* encoding: UTF-8
|
||||
* tab size: 8 (not used)
|
||||
* indentation:4
|
||||
*
|
||||
|
|
@ -24,20 +24,16 @@
|
|||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/umutablecptrie.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "normalizer2impl.h" // for IX_COUNT
|
||||
#include "toolutil.h"
|
||||
#include "utrie2.h"
|
||||
#include "norms.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
extern UBool beVerbose, haveCopyright;
|
||||
|
||||
struct Norm;
|
||||
|
||||
class BuilderReorderingBuffer;
|
||||
class ExtraDataWriter;
|
||||
|
||||
class Normalizer2DataBuilder {
|
||||
public:
|
||||
Normalizer2DataBuilder(UErrorCode &errorCode);
|
||||
|
|
@ -67,44 +63,43 @@ public:
|
|||
|
||||
void writeBinaryFile(const char *filename);
|
||||
void writeCSourceFile(const char *filename);
|
||||
void writeDataFile(const char *filename, bool writeRemoved) const;
|
||||
|
||||
static void computeDiff(const Normalizer2DataBuilder &b1,
|
||||
const Normalizer2DataBuilder &b2,
|
||||
Normalizer2DataBuilder &diff);
|
||||
|
||||
private:
|
||||
friend class CompositionBuilder;
|
||||
friend class Decomposer;
|
||||
friend class ExtraDataWriter;
|
||||
friend class Norm16Writer;
|
||||
|
||||
// No copy constructor nor assignment operator.
|
||||
Normalizer2DataBuilder(const Normalizer2DataBuilder &other);
|
||||
Normalizer2DataBuilder &operator=(const Normalizer2DataBuilder &other);
|
||||
Normalizer2DataBuilder(const Normalizer2DataBuilder &other) = delete;
|
||||
Normalizer2DataBuilder &operator=(const Normalizer2DataBuilder &other) = delete;
|
||||
|
||||
Norm *allocNorm();
|
||||
Norm *getNorm(UChar32 c);
|
||||
Norm *createNorm(UChar32 c);
|
||||
Norm *checkNormForMapping(Norm *p, UChar32 c); // check for permitted overrides
|
||||
|
||||
const Norm &getNormRef(UChar32 c) const;
|
||||
uint8_t getCC(UChar32 c) const;
|
||||
UBool combinesWithCCBetween(const Norm &norm, uint8_t lowCC, uint8_t highCC) const;
|
||||
UChar32 combine(const Norm &norm, UChar32 trail) const;
|
||||
/**
|
||||
* A starter character with a mapping does not have a composition boundary after it
|
||||
* if the character itself combines-forward (which is tested by the caller of this function),
|
||||
* or it is deleted (mapped to the empty string),
|
||||
* or its mapping contains no starter,
|
||||
* or the last starter combines-forward.
|
||||
*/
|
||||
UBool mappingHasCompBoundaryAfter(const BuilderReorderingBuffer &buffer,
|
||||
Norm::MappingType mappingType) const;
|
||||
/** Returns TRUE if the mapping by itself recomposes, that is, it is not comp-normalized. */
|
||||
UBool mappingRecomposes(const BuilderReorderingBuffer &buffer) const;
|
||||
void postProcess(Norm &norm);
|
||||
|
||||
void addComposition(UChar32 start, UChar32 end, uint32_t value);
|
||||
UBool decompose(UChar32 start, UChar32 end, uint32_t value);
|
||||
void reorder(Norm *p, BuilderReorderingBuffer &buffer);
|
||||
UBool hasNoCompBoundaryAfter(BuilderReorderingBuffer &buffer);
|
||||
void setHangulData();
|
||||
int32_t writeMapping(UChar32 c, const Norm *p, UnicodeString &dataString);
|
||||
void writeCompositions(UChar32 c, const Norm *p, UnicodeString &dataString);
|
||||
void writeExtraData(UChar32 c, uint32_t value, ExtraDataWriter &writer);
|
||||
int32_t getCenterNoNoDelta() {
|
||||
return indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]-Normalizer2Impl::MAX_DELTA-1;
|
||||
void setSmallFCD(UChar32 c);
|
||||
int32_t getMinNoNoDelta() const {
|
||||
return indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]-
|
||||
((2*Normalizer2Impl::MAX_DELTA+1)<<Normalizer2Impl::DELTA_SHIFT);
|
||||
}
|
||||
void writeNorm16(UChar32 start, UChar32 end, uint32_t value);
|
||||
void processData();
|
||||
void writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm);
|
||||
void setHangulData(UMutableCPTrie *norm16Trie);
|
||||
LocalUCPTriePointer processData();
|
||||
|
||||
UTrie2 *normTrie;
|
||||
UToolMemory *normMem;
|
||||
Norm *norms;
|
||||
Norms norms;
|
||||
|
||||
int32_t phase;
|
||||
OverrideHandling overrideHandling;
|
||||
|
|
@ -112,7 +107,7 @@ private:
|
|||
Optimization optimization;
|
||||
|
||||
int32_t indexes[Normalizer2Impl::IX_COUNT];
|
||||
UTrie2 *norm16Trie;
|
||||
uint8_t *norm16TrieBytes;
|
||||
int32_t norm16TrieLength;
|
||||
UnicodeString extraData;
|
||||
uint8_t smallFCD[0x100];
|
||||
|
|
|
|||
324
intl/icu/source/tools/gennorm2/norms.cpp
Normal file
324
intl/icu/source/tools/gennorm2/norms.cpp
Normal file
|
|
@ -0,0 +1,324 @@
|
|||
// © 2017 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
// norms.cpp
|
||||
// created: 2017jun04 Markus W. Scherer
|
||||
// (pulled out of n2builder.cpp)
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/umutablecptrie.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/utf16.h"
|
||||
#include "normalizer2impl.h"
|
||||
#include "norms.h"
|
||||
#include "toolutil.h"
|
||||
#include "uvectr32.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
void BuilderReorderingBuffer::append(UChar32 c, uint8_t cc) {
|
||||
if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
|
||||
if(cc==0) {
|
||||
fLastStarterIndex=fLength;
|
||||
}
|
||||
fArray[fLength++]=(c<<8)|cc;
|
||||
return;
|
||||
}
|
||||
// Let this character bubble back to its canonical order.
|
||||
int32_t i=fLength-1;
|
||||
while(i>fLastStarterIndex && ccAt(i)>cc) {
|
||||
--i;
|
||||
}
|
||||
++i; // after the last starter or prevCC<=cc
|
||||
// Move this and the following characters forward one to make space.
|
||||
for(int32_t j=fLength; i<j; --j) {
|
||||
fArray[j]=fArray[j-1];
|
||||
}
|
||||
fArray[i]=(c<<8)|cc;
|
||||
++fLength;
|
||||
fDidReorder=TRUE;
|
||||
}
|
||||
|
||||
void BuilderReorderingBuffer::toString(UnicodeString &dest) const {
|
||||
dest.remove();
|
||||
for(int32_t i=0; i<fLength; ++i) {
|
||||
dest.append(charAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
UChar32 Norm::combine(UChar32 trail) const {
|
||||
int32_t length;
|
||||
const CompositionPair *pairs=getCompositionPairs(length);
|
||||
for(int32_t i=0; i<length; ++i) {
|
||||
if(trail==pairs[i].trail) {
|
||||
return pairs[i].composite;
|
||||
}
|
||||
if(trail<pairs[i].trail) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return U_SENTINEL;
|
||||
}
|
||||
|
||||
Norms::Norms(UErrorCode &errorCode) {
|
||||
normTrie = umutablecptrie_open(0, 0, &errorCode);
|
||||
normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
|
||||
// Default "inert" Norm struct at index 0. Practically immutable.
|
||||
norms=allocNorm();
|
||||
norms->type=Norm::INERT;
|
||||
}
|
||||
|
||||
Norms::~Norms() {
|
||||
umutablecptrie_close(normTrie);
|
||||
int32_t normsLength=utm_countItems(normMem);
|
||||
for(int32_t i=1; i<normsLength; ++i) {
|
||||
delete norms[i].mapping;
|
||||
delete norms[i].rawMapping;
|
||||
delete norms[i].compositions;
|
||||
}
|
||||
utm_close(normMem);
|
||||
}
|
||||
|
||||
Norm *Norms::allocNorm() {
|
||||
Norm *p=(Norm *)utm_alloc(normMem);
|
||||
norms=(Norm *)utm_getStart(normMem); // in case it got reallocated
|
||||
return p;
|
||||
}
|
||||
|
||||
Norm *Norms::getNorm(UChar32 c) {
|
||||
uint32_t i = umutablecptrie_get(normTrie, c);
|
||||
if(i==0) {
|
||||
return nullptr;
|
||||
}
|
||||
return norms+i;
|
||||
}
|
||||
|
||||
const Norm *Norms::getNorm(UChar32 c) const {
|
||||
uint32_t i = umutablecptrie_get(normTrie, c);
|
||||
if(i==0) {
|
||||
return nullptr;
|
||||
}
|
||||
return norms+i;
|
||||
}
|
||||
|
||||
const Norm &Norms::getNormRef(UChar32 c) const {
|
||||
return norms[umutablecptrie_get(normTrie, c)];
|
||||
}
|
||||
|
||||
Norm *Norms::createNorm(UChar32 c) {
|
||||
uint32_t i=umutablecptrie_get(normTrie, c);
|
||||
if(i!=0) {
|
||||
return norms+i;
|
||||
} else {
|
||||
/* allocate Norm */
|
||||
Norm *p=allocNorm();
|
||||
IcuToolErrorCode errorCode("gennorm2/createNorm()");
|
||||
umutablecptrie_set(normTrie, c, (uint32_t)(p - norms), errorCode);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
void Norms::reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const {
|
||||
int32_t length=mapping.length();
|
||||
U_ASSERT(length<=Normalizer2Impl::MAPPING_LENGTH_MASK);
|
||||
const char16_t *s=mapping.getBuffer();
|
||||
int32_t i=0;
|
||||
UChar32 c;
|
||||
while(i<length) {
|
||||
U16_NEXT(s, i, length, c);
|
||||
buffer.append(c, getCC(c));
|
||||
}
|
||||
if(buffer.didReorder()) {
|
||||
buffer.toString(mapping);
|
||||
}
|
||||
}
|
||||
|
||||
UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const {
|
||||
if((highCC-lowCC)>=2) {
|
||||
int32_t length;
|
||||
const CompositionPair *pairs=norm.getCompositionPairs(length);
|
||||
for(int32_t i=0; i<length; ++i) {
|
||||
uint8_t trailCC=getCC(pairs[i].trail);
|
||||
if(lowCC<trailCC && trailCC<highCC) {
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
}
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
void Norms::enumRanges(Enumerator &e) {
|
||||
UChar32 start = 0, end;
|
||||
uint32_t i;
|
||||
while ((end = umutablecptrie_getRange(normTrie, start, UCPMAP_RANGE_NORMAL, 0,
|
||||
nullptr, nullptr, &i)) >= 0) {
|
||||
if (i > 0) {
|
||||
e.rangeHandler(start, end, norms[i]);
|
||||
}
|
||||
start = end + 1;
|
||||
}
|
||||
}
|
||||
|
||||
Norms::Enumerator::~Enumerator() {}
|
||||
|
||||
void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
|
||||
if(norm.mappingType!=Norm::ROUND_TRIP) { return; }
|
||||
if(start!=end) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: same round-trip mapping for "
|
||||
"more than 1 code point U+%04lX..U+%04lX\n",
|
||||
(long)start, (long)end);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
if(norm.cc!=0) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX has a round-trip mapping and ccc!=0, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
// setRoundTripMapping() ensured that there are exactly two code points.
|
||||
const UnicodeString &m=*norm.mapping;
|
||||
UChar32 lead=m.char32At(0);
|
||||
UChar32 trail=m.char32At(m.length()-1);
|
||||
if(norms.getCC(lead)!=0) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start, (long)lead);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
// Flag for trailing character.
|
||||
norms.createNorm(trail)->combinesBack=TRUE;
|
||||
// Insert (trail, composite) pair into compositions list for the lead character.
|
||||
IcuToolErrorCode errorCode("gennorm2/addComposition()");
|
||||
Norm *leadNorm=norms.createNorm(lead);
|
||||
UVector32 *compositions=leadNorm->compositions;
|
||||
int32_t i;
|
||||
if(compositions==nullptr) {
|
||||
compositions=leadNorm->compositions=new UVector32(errorCode);
|
||||
i=0; // "insert" the first pair at index 0
|
||||
} else {
|
||||
// Insertion sort, and check for duplicate trail characters.
|
||||
int32_t length;
|
||||
const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
|
||||
for(i=0; i<length; ++i) {
|
||||
if(trail==pairs[i].trail) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: same round-trip mapping for "
|
||||
"more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
|
||||
(long)start, (long)lead, (long)trail);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
if(trail<pairs[i].trail) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
compositions->insertElementAt(trail, 2*i, errorCode);
|
||||
compositions->insertElementAt(start, 2*i+1, errorCode);
|
||||
}
|
||||
|
||||
void Decomposer::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
|
||||
if(!norm.hasMapping()) { return; }
|
||||
const UnicodeString &m=*norm.mapping;
|
||||
UnicodeString *decomposed=nullptr;
|
||||
const UChar *s=toUCharPtr(m.getBuffer());
|
||||
int32_t length=m.length();
|
||||
int32_t prev, i=0;
|
||||
UChar32 c;
|
||||
while(i<length) {
|
||||
prev=i;
|
||||
U16_NEXT(s, i, length, c);
|
||||
if(start<=c && c<=end) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
|
||||
(long)c);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
const Norm &cNorm=norms.getNormRef(c);
|
||||
if(cNorm.hasMapping()) {
|
||||
if(norm.mappingType==Norm::ROUND_TRIP) {
|
||||
if(prev==0) {
|
||||
if(cNorm.mappingType!=Norm::ROUND_TRIP) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX's round-trip mapping's starter "
|
||||
"U+%04lX one-way-decomposes, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start, (long)c);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
uint8_t myTrailCC=norms.getCC(m.char32At(i));
|
||||
UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
|
||||
uint8_t cTrailCC=norms.getCC(cTrailChar);
|
||||
if(cTrailCC>myTrailCC) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX's round-trip mapping's starter "
|
||||
"U+%04lX decomposes and the "
|
||||
"inner/earlier tccc=%hu > outer/following tccc=%hu, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start, (long)c,
|
||||
(short)cTrailCC, (short)myTrailCC);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
} else {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX's round-trip mapping's non-starter "
|
||||
"U+%04lX decomposes, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start, (long)c);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
}
|
||||
if(decomposed==nullptr) {
|
||||
decomposed=new UnicodeString(m, 0, prev);
|
||||
}
|
||||
decomposed->append(*cNorm.mapping);
|
||||
} else if(Hangul::isHangul(c)) {
|
||||
UChar buffer[3];
|
||||
int32_t hangulLength=Hangul::decompose(c, buffer);
|
||||
if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
|
||||
fprintf(stderr,
|
||||
"gennorm2 error: "
|
||||
"U+%04lX's round-trip mapping's non-starter "
|
||||
"U+%04lX decomposes, "
|
||||
"not possible in Unicode normalization\n",
|
||||
(long)start, (long)c);
|
||||
exit(U_INVALID_FORMAT_ERROR);
|
||||
}
|
||||
if(decomposed==nullptr) {
|
||||
decomposed=new UnicodeString(m, 0, prev);
|
||||
}
|
||||
decomposed->append(buffer, hangulLength);
|
||||
} else if(decomposed!=nullptr) {
|
||||
decomposed->append(m, prev, i-prev);
|
||||
}
|
||||
}
|
||||
if(decomposed!=nullptr) {
|
||||
if(norm.rawMapping==nullptr) {
|
||||
// Remember the original mapping when decomposing recursively.
|
||||
norm.rawMapping=norm.mapping;
|
||||
} else {
|
||||
delete norm.mapping;
|
||||
}
|
||||
norm.mapping=decomposed;
|
||||
// Not norm.setMappingCP(); because the original mapping
|
||||
// is most likely to be encodable as a delta.
|
||||
didDecompose|=TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif // #if !UCONFIG_NO_NORMALIZATION
|
||||
215
intl/icu/source/tools/gennorm2/norms.h
Normal file
215
intl/icu/source/tools/gennorm2/norms.h
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
// © 2017 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
// norms.h
|
||||
// created: 2017jun04 Markus W. Scherer
|
||||
// (pulled out of n2builder.cpp)
|
||||
|
||||
// Storing & manipulating Normalizer2 builder data.
|
||||
|
||||
#ifndef __NORMS_H__
|
||||
#define __NORMS_H__
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#include "unicode/errorcode.h"
|
||||
#include "unicode/umutablecptrie.h"
|
||||
#include "unicode/uniset.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/utf16.h"
|
||||
#include "normalizer2impl.h"
|
||||
#include "toolutil.h"
|
||||
#include "uvectr32.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class BuilderReorderingBuffer {
|
||||
public:
|
||||
BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(FALSE) {}
|
||||
void reset() {
|
||||
fLength=0;
|
||||
fLastStarterIndex=-1;
|
||||
fDidReorder=FALSE;
|
||||
}
|
||||
int32_t length() const { return fLength; }
|
||||
UBool isEmpty() const { return fLength==0; }
|
||||
int32_t lastStarterIndex() const { return fLastStarterIndex; }
|
||||
UChar32 charAt(int32_t i) const { return fArray[i]>>8; }
|
||||
uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; }
|
||||
UBool didReorder() const { return fDidReorder; }
|
||||
|
||||
void append(UChar32 c, uint8_t cc);
|
||||
void toString(UnicodeString &dest) const;
|
||||
|
||||
private:
|
||||
int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK];
|
||||
int32_t fLength;
|
||||
int32_t fLastStarterIndex;
|
||||
UBool fDidReorder;
|
||||
};
|
||||
|
||||
struct CompositionPair {
|
||||
CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {}
|
||||
UChar32 trail, composite;
|
||||
};
|
||||
|
||||
struct Norm {
|
||||
enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY };
|
||||
|
||||
UBool hasMapping() const { return mappingType>REMOVED; }
|
||||
|
||||
// Requires hasMapping() and well-formed mapping.
|
||||
void setMappingCP() {
|
||||
UChar32 c;
|
||||
if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) {
|
||||
mappingCP=c;
|
||||
} else {
|
||||
mappingCP=U_SENTINEL;
|
||||
}
|
||||
}
|
||||
|
||||
const CompositionPair *getCompositionPairs(int32_t &length) const {
|
||||
if(compositions==nullptr) {
|
||||
length=0;
|
||||
return nullptr;
|
||||
} else {
|
||||
length=compositions->size()/2;
|
||||
return reinterpret_cast<const CompositionPair *>(compositions->getBuffer());
|
||||
}
|
||||
}
|
||||
UChar32 combine(UChar32 trail) const;
|
||||
|
||||
UnicodeString *mapping;
|
||||
UnicodeString *rawMapping; // non-nullptr if the mapping is further decomposed
|
||||
UChar32 mappingCP; // >=0 if mapping to 1 code point
|
||||
int32_t mappingPhase;
|
||||
MappingType mappingType;
|
||||
|
||||
UVector32 *compositions; // (trail, composite) pairs
|
||||
uint8_t cc, leadCC, trailCC;
|
||||
UBool combinesBack;
|
||||
UBool hasCompBoundaryBefore, hasCompBoundaryAfter;
|
||||
|
||||
/**
|
||||
* Overall type of normalization properties.
|
||||
* Set after most processing is done.
|
||||
*
|
||||
* Corresponds to the rows in the chart on
|
||||
* http://site.icu-project.org/design/normalization/custom
|
||||
* in numerical (but reverse visual) order.
|
||||
*
|
||||
* YES_NO means composition quick check=yes, decomposition QC=no -- etc.
|
||||
*/
|
||||
enum Type {
|
||||
/** Initial value until most processing is done. */
|
||||
UNKNOWN,
|
||||
/** No mapping, does not combine, ccc=0. */
|
||||
INERT,
|
||||
/** Starter, no mapping, has compositions. */
|
||||
YES_YES_COMBINES_FWD,
|
||||
/** Starter with a round-trip mapping and compositions. */
|
||||
YES_NO_COMBINES_FWD,
|
||||
/** Starter with a round-trip mapping but no compositions. */
|
||||
YES_NO_MAPPING_ONLY,
|
||||
/** Has a one-way mapping which is comp-normalized. */
|
||||
NO_NO_COMP_YES,
|
||||
/** Has a one-way mapping which is not comp-normalized but has a comp boundary before. */
|
||||
NO_NO_COMP_BOUNDARY_BEFORE,
|
||||
/** Has a one-way mapping which does not have a comp boundary before. */
|
||||
NO_NO_COMP_NO_MAYBE_CC,
|
||||
/** Has a one-way mapping to the empty string. */
|
||||
NO_NO_EMPTY,
|
||||
/** Has an algorithmic one-way mapping to a single code point. */
|
||||
NO_NO_DELTA,
|
||||
/**
|
||||
* Combines both backward and forward, has compositions.
|
||||
* Allowed, but not normally used.
|
||||
*/
|
||||
MAYBE_YES_COMBINES_FWD,
|
||||
/** Combines only backward. */
|
||||
MAYBE_YES_SIMPLE,
|
||||
/** Non-zero ccc but does not combine backward. */
|
||||
YES_YES_WITH_CC
|
||||
} type;
|
||||
/** Offset into the type's part of the extra data, or the algorithmic-mapping delta. */
|
||||
int32_t offset;
|
||||
|
||||
/**
|
||||
* Error string set by processing functions that do not have access
|
||||
* to the code point, deferred for readable reporting.
|
||||
*/
|
||||
const char *error;
|
||||
};
|
||||
|
||||
class Norms {
|
||||
public:
|
||||
Norms(UErrorCode &errorCode);
|
||||
~Norms();
|
||||
|
||||
int32_t length() const { return utm_countItems(normMem); }
|
||||
const Norm &getNormRefByIndex(int32_t i) const { return norms[i]; }
|
||||
Norm &getNormRefByIndex(int32_t i) { return norms[i]; }
|
||||
|
||||
Norm *allocNorm();
|
||||
/** Returns an existing Norm unit, or nullptr if c has no data. */
|
||||
Norm *getNorm(UChar32 c);
|
||||
const Norm *getNorm(UChar32 c) const;
|
||||
/** Returns a Norm unit, creating a new one if necessary. */
|
||||
Norm *createNorm(UChar32 c);
|
||||
/** Returns an existing Norm unit, or an immutable empty object if c has no data. */
|
||||
const Norm &getNormRef(UChar32 c) const;
|
||||
uint8_t getCC(UChar32 c) const { return getNormRef(c).cc; }
|
||||
UBool combinesBack(UChar32 c) const {
|
||||
return Hangul::isJamoV(c) || Hangul::isJamoT(c) || getNormRef(c).combinesBack;
|
||||
}
|
||||
|
||||
void reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const;
|
||||
|
||||
// int32_t highCC not uint8_t so that we can pass in 256 as the upper limit.
|
||||
UBool combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const;
|
||||
|
||||
class Enumerator {
|
||||
public:
|
||||
Enumerator(Norms &n) : norms(n) {}
|
||||
virtual ~Enumerator();
|
||||
/** Called for enumerated value!=0. */
|
||||
virtual void rangeHandler(UChar32 start, UChar32 end, Norm &norm) = 0;
|
||||
protected:
|
||||
Norms &norms;
|
||||
};
|
||||
|
||||
void enumRanges(Enumerator &e);
|
||||
|
||||
UnicodeSet ccSet, mappingSet;
|
||||
|
||||
private:
|
||||
Norms(const Norms &other) = delete;
|
||||
Norms &operator=(const Norms &other) = delete;
|
||||
|
||||
UMutableCPTrie *normTrie;
|
||||
UToolMemory *normMem;
|
||||
Norm *norms;
|
||||
};
|
||||
|
||||
class CompositionBuilder : public Norms::Enumerator {
|
||||
public:
|
||||
CompositionBuilder(Norms &n) : Norms::Enumerator(n) {}
|
||||
/** Adds a composition mapping for the first character in a round-trip mapping. */
|
||||
void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE;
|
||||
};
|
||||
|
||||
class Decomposer : public Norms::Enumerator {
|
||||
public:
|
||||
Decomposer(Norms &n) : Norms::Enumerator(n), didDecompose(FALSE) {}
|
||||
/** Decomposes each character of the current mapping. Sets didDecompose if any. */
|
||||
void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE;
|
||||
UBool didDecompose;
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif // #if !UCONFIG_NO_NORMALIZATION
|
||||
|
||||
#endif // __NORMS_H__
|
||||
Loading…
Add table
Add a link
Reference in a new issue