import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

This commit is contained in:
Roy Tam 2018-01-19 03:59:58 +08:00
commit dcd9973243
150858 changed files with 23884658 additions and 0 deletions

View file

@ -0,0 +1,95 @@
## Makefile.in for ICU - tools/gendict
## Copyright (C) 2016 and later: Unicode, Inc. and others.
## License & terms of use: http://www.unicode.org/copyright.html
## Copyright (c) 2002-2012 International Business Machines Corporation and
## others. All Rights Reserved.
## Source directory information
srcdir = @srcdir@
top_srcdir = @top_srcdir@
top_builddir = ../..
include $(top_builddir)/icudefs.mk
## Build directory information
subdir = tools/gendict
TARGET_STUB_NAME = gendict
SECTION = 1
MAN_FILES = $(TARGET_STUB_NAME).$(SECTION)
## Extra files to remove for 'make clean'
CLEANFILES = *~ $(DEPS) $(MAN_FILES)
## Target information
TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT)
CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil
LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M)
OBJECTS = gendict.o
DEPS = $(OBJECTS:.o=.d)
## List of phony targets
.PHONY : all all-local install install-local clean clean-local \
distclean distclean-local dist dist-local check check-local install-man
## Clear suffix list
.SUFFIXES :
## List of standard targets
all: all-local
install: install-local
clean: clean-local
distclean : distclean-local
dist: dist-local
check: all check-local
all-local: $(TARGET) $(MAN_FILES)
install-local: all-local install-man
$(MKINSTALLDIRS) $(DESTDIR)$(bindir)
$(INSTALL) $(TARGET) $(DESTDIR)$(bindir)
install-man: $(MAN_FILES)
$(MKINSTALLDIRS) $(DESTDIR)$(mandir)/man$(SECTION)
$(INSTALL_DATA) $? $(DESTDIR)$(mandir)/man$(SECTION)
dist-local:
clean-local:
test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES)
$(RMV) $(TARGET) $(OBJECTS)
distclean-local: clean-local
$(RMV) Makefile
check-local: all-local
Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
cd $(top_builddir) \
&& CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
$(TARGET) : $(OBJECTS)
$(LINK.cc) $(OUTOPT)$@ $^ $(LIBS)
$(POST_BUILD_STEP)
%.$(SECTION): $(srcdir)/%.$(SECTION).in
cd $(top_builddir) \
&& CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
ifeq (,$(MAKECMDGOALS))
-include $(DEPS)
else
ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),)
-include $(DEPS)
endif
endif

View file

@ -0,0 +1,133 @@
.\" Hey, Emacs! This is -*-nroff-*- you know...
.\"
.\" gendict.1: manual page for the gendict utility
.\"
.\" Copyright (C) 2016 and later: Unicode, Inc. and others.
.\" License & terms of use: http://www.unicode.org/copyright.html
.\" Copyright (C) 2012 International Business Machines Corporation and others
.\"
.TH GENDICT 1 "1 June 2012" "ICU MANPAGE" "ICU @VERSION@ Manual"
.SH NAME
.B gendict
\- Compiles word list into ICU string trie dictionary
.SH SYNOPSIS
.B gendict
[
.BR "\fB\-\-uchars"
|
.BR "\fB\-\-bytes"
.BI "\fB\-\-transform" " transform"
]
[
.BR "\-h\fP, \fB\-?\fP, \fB\-\-help"
]
[
.BR "\-V\fP, \fB\-\-version"
]
[
.BR "\-c\fP, \fB\-\-copyright"
]
[
.BR "\-v\fP, \fB\-\-verbose"
]
[
.BI "\-i\fP, \fB\-\-icudatadir" " directory"
]
.IR " input-file"
.IR " output\-file"
.SH DESCRIPTION
.B gendict
reads the word list from
.I dictionary-file
and creates a string trie dictionary file. Normally this data file has the
.B .dict
extension.
.PP
Words begin at the beginning of a line and are terminated by the first whitespace.
Lines that begin with whitespace are ignored.
.SH OPTIONS
.TP
.BR "\-h\fP, \fB\-?\fP, \fB\-\-help"
Print help about usage and exit.
.TP
.BR "\-V\fP, \fB\-\-version"
Print the version of
.B gendict
and exit.
.TP
.BR "\-c\fP, \fB\-\-copyright"
Embeds the standard ICU copyright into the
.IR output-file .
.TP
.BR "\-v\fP, \fB\-\-verbose"
Display extra informative messages during execution.
.TP
.BI "\-i\fP, \fB\-\-icudatadir" " directory"
Look for any necessary ICU data files in
.IR directory .
For example, the file
.B pnames.icu
must be located when ICU's data is not built as a shared library.
The default ICU data directory is specified by the environment variable
.BR ICU_DATA .
Most configurations of ICU do not require this argument.
.TP
.BR "\fB\-\-uchars"
Set the output trie type to UChar. Mutually exclusive with
.BR --bytes.
.TP
.BR "\fB\-\-bytes"
Set the output trie type to Bytes. Mutually exclusive with
.BR --uchars.
.TP
.BR "\fB\-\-transform"
Set the transform type. Should only be specified with
.BR --bytes.
Currently supported transforms are:
.BR offset-<hex-number>,
which specifies an offset to subtract from all input characters.
It should be noted that the offset transform also maps U+200D
to 0xFF and U+200C to 0xFE, in order to offer compatibility to
languages that require these characters.
A transform must be specified for a bytes trie, and when applied
to the non-value characters in the
.IR input-file
must produce output between 0x00 and 0xFF.
.TP
.BI " input\-file"
The source file to read.
.TP
.BI " output\-file"
The file to write the output dictionary to.
.SH CAVEATS
The
.IR input-file
is assumed to be encoded in UTF-8.
The integers in the
.IR input-file
that are used as values must be made up of ASCII digits. They
may be specified either in hex, by using a 0x prefix, or in
decimal.
Either
.BI --bytes
or
.BI --uchars
must be specified.
.SH ENVIRONMENT
.TP 10
.B ICU_DATA
Specifies the directory containing ICU data. Defaults to
.BR @thepkgicudatadir@/@PACKAGE@/@VERSION@/ .
Some tools in ICU depend on the presence of the trailing slash. It is thus
important to make sure that it is present if
.B ICU_DATA
is set.
.SH AUTHORS
Maxime Serrano
.SH VERSION
1.0
.SH COPYRIGHT
Copyright (C) 2012 International Business Machines Corporation and others
.SH SEE ALSO
.BR http://www.icu-project.org/userguide/boundaryAnalysis.html

View file

@ -0,0 +1,479 @@
// Copyright (C) 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2002-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*
* File gendict.cpp
*/
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "unicode/ucnv.h"
#include "unicode/uniset.h"
#include "unicode/unistr.h"
#include "unicode/uclean.h"
#include "unicode/udata.h"
#include "unicode/putil.h"
#include "unicode/ucharstriebuilder.h"
#include "unicode/bytestriebuilder.h"
#include "unicode/ucharstrie.h"
#include "unicode/bytestrie.h"
#include "unicode/ucnv.h"
#include "unicode/utf16.h"
#include "charstr.h"
#include "dictionarydata.h"
#include "uoptions.h"
#include "unewdata.h"
#include "cmemory.h"
#include "uassert.h"
#include "ucbuf.h"
#include "toolutil.h"
#include "cstring.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "putilimp.h"
UDate startTime;
static int elapsedTime() {
return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0);
}
U_NAMESPACE_USE
static char *progName;
static UOption options[]={
UOPTION_HELP_H, /* 0 */
UOPTION_HELP_QUESTION_MARK, /* 1 */
UOPTION_VERBOSE, /* 2 */
UOPTION_ICUDATADIR, /* 4 */
UOPTION_COPYRIGHT, /* 5 */
{ "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */
{ "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */
{ "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
UOPTION_QUIET, /* 9 */
};
enum arguments {
ARG_HELP = 0,
ARG_QMARK,
ARG_VERBOSE,
ARG_ICUDATADIR,
ARG_COPYRIGHT,
ARG_UCHARS,
ARG_BYTES,
ARG_TRANSFORM,
ARG_QUIET
};
// prints out the standard usage method describing command line arguments,
// then bails out with the desired exit code
static void usageAndDie(UErrorCode retCode) {
fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
fprintf((U_SUCCESS(retCode) ? stdout : stderr),
"\tRead in a word list and write out a string trie dictionary\n"
"options:\n"
"\t-h or -? or --help this usage text\n"
"\t-V or --version show a version message\n"
"\t-c or --copyright include a copyright notice\n"
"\t-v or --verbose turn on verbose output\n"
"\t-q or --quiet do not display warnings and progress\n"
"\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
"\t followed by path, defaults to %s\n"
"\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n"
"\t--bytes output a BytesTrie (mutually exclusive with -u!)\n"
"\t--transform the kind of transform to use (eg --transform offset-40A3,\n"
"\t which specifies an offset transform with constant 0x40A3)\n",
u_getDataDirectory());
exit(retCode);
}
/* UDataInfo cf. udata.h */
static UDataInfo dataInfo = {
sizeof(UDataInfo),
0,
U_IS_BIG_ENDIAN,
U_CHARSET_FAMILY,
U_SIZEOF_UCHAR,
0,
{ 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */
{ 1, 0, 0, 0 }, /* format version */
{ 0, 0, 0, 0 } /* data version */
};
#if !UCONFIG_NO_BREAK_ITERATION
// A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
// may want to put this somewhere in ICU, as it could be useful outside
// of this tool?
class DataDict {
private:
BytesTrieBuilder *bt;
UCharsTrieBuilder *ut;
UChar32 transformConstant;
int32_t transformType;
public:
// constructs a new data dictionary. if there is an error,
// it will be returned in status
// isBytesTrie != 0 will produce a BytesTrieBuilder,
// isBytesTrie == 0 will produce a UCharsTrieBuilder
DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL),
transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
if (isBytesTrie) {
bt = new BytesTrieBuilder(status);
} else {
ut = new UCharsTrieBuilder(status);
}
}
~DataDict() {
delete bt;
delete ut;
}
private:
char transform(UChar32 c, UErrorCode &status) {
if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
if (c == 0x200D) { return (char)0xFF; }
else if (c == 0x200C) { return (char)0xFE; }
int32_t delta = c - transformConstant;
if (delta < 0 || 0xFD < delta) {
fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
(long)c, (long)transformConstant);
exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
}
return (char)delta;
} else { // no such transform type
status = U_INTERNAL_PROGRAM_ERROR;
return (char)c; // it should be noted this transform type will not generally work
}
}
void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
UChar32 c = 0;
int32_t len = word.length();
for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
c = word.char32At(i);
buf.append(transform(c, errorCode), errorCode);
}
}
public:
// sets the desired transformation data.
// should be populated from a command line argument
// so far the only acceptable format is offset-<hex constant>
// eventually others (mask-<hex constant>?) may be enabled
// more complex functions may be more difficult
void setTransform(const char *t) {
if (strncmp(t, "offset-", 7) == 0) {
char *end;
unsigned long base = uprv_strtoul(t + 7, &end, 16);
if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
transformConstant = (UChar32)base;
}
else {
fprintf(stderr, "Invalid transform specified: %s\n", t);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
}
// add a word to the trie
void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
if (bt) {
CharString buf;
transform(word, buf, status);
bt->add(buf.toStringPiece(), value, status);
}
if (ut) { ut->add(word, value, status); }
}
// if we are a bytestrie, give back the StringPiece representing the serialized version of us
StringPiece serializeBytes(UErrorCode &status) {
return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
}
// if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
void serializeUChars(UnicodeString &s, UErrorCode &status) {
ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
}
int32_t getTransform() {
return (int32_t)(transformType | transformConstant);
}
};
#endif
static const UChar LINEFEED_CHARACTER = 0x000A;
static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D;
static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
int32_t lineLength;
const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
if(line == NULL || errorCode.isFailure()) { return FALSE; }
// Strip trailing CR/LF, comments, and spaces.
const UChar *comment = u_memchr(line, 0x23, lineLength); // '#'
if(comment != NULL) {
lineLength = (int32_t)(comment - line);
} else {
while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
}
while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
fileLine.setTo(FALSE, line, lineLength);
return TRUE;
}
//----------------------------------------------------------------------------
//
// main for gendict
//
//----------------------------------------------------------------------------
int main(int argc, char **argv) {
//
// Pick up and check the command line arguments,
// using the standard ICU tool utils option handling.
//
U_MAIN_INIT_ARGS(argc, argv);
progName = argv[0];
argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options);
if(argc<0) {
// Unrecognized option
fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
// -? or -h for help.
usageAndDie(U_ZERO_ERROR);
}
UBool verbose = options[ARG_VERBOSE].doesOccur;
UBool quiet = options[ARG_QUIET].doesOccur;
if (argc < 3) {
fprintf(stderr, "input and output file must both be specified.\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
const char *outFileName = argv[2];
const char *wordFileName = argv[1];
startTime = uprv_getRawUTCtime(); // initialize start timer
if (options[ARG_ICUDATADIR].doesOccur) {
u_setDataDirectory(options[ARG_ICUDATADIR].value);
}
const char *copyright = NULL;
if (options[ARG_COPYRIGHT].doesOccur) {
copyright = U_COPYRIGHT_STRING;
}
if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
fprintf(stderr, "you must specify exactly one type of trie to output!\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
UBool isBytesTrie = options[ARG_BYTES].doesOccur;
if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
}
IcuToolErrorCode status("gendict/main()");
#if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
const char* outDir=NULL;
UNewDataMemory *pData;
char msg[1024];
UErrorCode tempstatus = U_ZERO_ERROR;
/* write message with just the name */ // potential for a buffer overflow here...
sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
fprintf(stderr, "%s\n", msg);
/* write the dummy data file */
pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
udata_writeBlock(pData, msg, strlen(msg));
udata_finish(pData, &tempstatus);
return (int)tempstatus;
#else
// Read in the dictionary source file
if (verbose) { printf("Opening file %s...\n", wordFileName); }
const char *codepage = "UTF-8";
UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status);
if (status.isFailure()) {
fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
exit(status.reset());
}
if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
DataDict dict(isBytesTrie, status);
if (status.isFailure()) {
fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
exit(status.reset());
}
if (options[ARG_TRANSFORM].doesOccur) {
dict.setTransform(options[ARG_TRANSFORM].value);
}
UnicodeString fileLine;
if (verbose) { puts("Adding words to dictionary..."); }
UBool hasValues = FALSE;
UBool hasValuelessContents = FALSE;
int lineCount = 0;
int wordCount = 0;
int minlen = 255;
int maxlen = 0;
UBool isOk = TRUE;
while (readLine(f, fileLine, status)) {
lineCount++;
if (fileLine.isEmpty()) continue;
// Parse word [spaces value].
int32_t keyLen;
for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
if (keyLen == 0) {
fprintf(stderr, "Error: no word on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
int32_t valueStart;
for (valueStart = keyLen;
valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
++valueStart) {}
if (keyLen < valueStart) {
int32_t valueLength = fileLine.length() - valueStart;
if (valueLength > 15) {
fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
char s[16];
fileLine.extract(valueStart, valueLength, s, 16, US_INV);
char *end;
unsigned long value = uprv_strtoul(s, &end, 0);
if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
isOk = FALSE;
continue;
}
dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
hasValues = TRUE;
wordCount++;
if (keyLen < minlen) minlen = keyLen;
if (keyLen > maxlen) maxlen = keyLen;
} else {
dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
hasValuelessContents = TRUE;
wordCount++;
if (keyLen < minlen) minlen = keyLen;
if (keyLen > maxlen) maxlen = keyLen;
}
if (status.isFailure()) {
fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
status.errorName(), lineCount);
exit(status.reset());
}
}
if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); }
if (!isOk && status.isSuccess()) {
status.set(U_ILLEGAL_ARGUMENT_ERROR);
}
if (hasValues && hasValuelessContents) {
fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
}
if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); }
int32_t outDataSize;
const void *outData;
UnicodeString usp;
if (isBytesTrie) {
StringPiece sp = dict.serializeBytes(status);
outDataSize = sp.size();
outData = sp.data();
} else {
dict.serializeUChars(usp, status);
outDataSize = usp.length() * U_SIZEOF_UCHAR;
outData = usp.getBuffer();
}
if (status.isFailure()) {
fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName());
exit(status.reset());
}
if (verbose) { puts("Opening output file..."); }
UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
if (status.isFailure()) {
fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
exit(status.reset());
}
if (verbose) { puts("Writing to output file..."); }
int32_t indexes[DictionaryData::IX_COUNT] = {
DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
};
int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
indexes[DictionaryData::IX_TOTAL_SIZE] = size;
indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
if (hasValues) {
indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
}
indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
udata_writeBlock(pData, indexes, sizeof(indexes));
udata_writeBlock(pData, outData, outDataSize);
size_t bytesWritten = udata_finish(pData, status);
if (status.isFailure()) {
fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
exit(status.reset());
}
if (bytesWritten != (size_t)size) {
fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
exit(U_INTERNAL_PROGRAM_ERROR);
}
if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); }
#ifdef TEST_GENDICT
if (isBytesTrie) {
BytesTrie::Iterator it(outData, outDataSize, status);
while (it.hasNext()) {
it.next(status);
const StringPiece s = it.getString();
int32_t val = it.getValue();
printf("%s -> %i\n", s.data(), val);
}
} else {
UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
while (it.hasNext()) {
it.next(status);
const UnicodeString s = it.getString();
int32_t val = it.getValue();
char tmp[1024];
s.extract(0, s.length(), tmp, 1024);
printf("%s -> %i\n", tmp, val);
}
}
#endif
return 0;
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
}

View file

@ -0,0 +1,262 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{9D4211F7-2C77-439C-82F0-30A4E43BA569}</ProjectGuid>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseOfMfc>false</UseOfMfc>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseOfMfc>false</UseOfMfc>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseOfMfc>false</UseOfMfc>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseOfMfc>false</UseOfMfc>
<CharacterSet>MultiByte</CharacterSet>
<PlatformToolset>v140</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
<Import Project="$(VCTargetsPath)Microsoft.CPP.UpgradeFromVC71.props" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
<OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">.\x86\Release\</OutDir>
<IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">.\x86\Release\</IntDir>
<LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
<OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">.\x86\Debug\</OutDir>
<IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">.\x86\Debug\</IntDir>
<LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
<OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">.\x64\Release\</OutDir>
<IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">.\x64\Release\</IntDir>
<LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
<OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">.\x64\Debug\</OutDir>
<IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">.\x64\Debug\</IntDir>
<LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<CustomBuildStep>
<Command>copy "$(TargetPath)" ..\..\..\bin
</Command>
<Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
</CustomBuildStep>
<Midl>
<TypeLibraryName>.\x86\Release/gendict.tlb</TypeLibraryName>
</Midl>
<ClCompile>
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<StringPooling>true</StringPooling>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<FunctionLevelLinking>true</FunctionLevelLinking>
<DisableLanguageExtensions>true</DisableLanguageExtensions>
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
<PrecompiledHeaderOutputFile>.\x86\Release/gendict.pch</PrecompiledHeaderOutputFile>
<AssemblerListingLocation>.\x86\Release/</AssemblerListingLocation>
<ObjectFileName>.\x86\Release/</ObjectFileName>
<ProgramDataBaseFileName>.\x86\Release/</ProgramDataBaseFileName>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<CompileAs>Default</CompileAs>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<Culture>0x0409</Culture>
</ResourceCompile>
<Link>
<OutputFile>.\x86\Release/gendict.exe</OutputFile>
<SuppressStartupBanner>true</SuppressStartupBanner>
<ProgramDatabaseFile>.\x86\Release/gendict.pdb</ProgramDatabaseFile>
<SubSystem>Console</SubSystem>
<RandomizedBaseAddress>false</RandomizedBaseAddress>
<DataExecutionPrevention>
</DataExecutionPrevention>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<CustomBuildStep>
<Command>copy "$(TargetPath)" ..\..\..\bin
</Command>
<Outputs>..\..\..\bin\$(TargetFileName);%(Outputs)</Outputs>
</CustomBuildStep>
<Midl>
<TypeLibraryName>.\x86\Debug/gendict.tlb</TypeLibraryName>
</Midl>
<ClCompile>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<BufferSecurityCheck>true</BufferSecurityCheck>
<DisableLanguageExtensions>true</DisableLanguageExtensions>
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
<PrecompiledHeaderOutputFile>.\x86\Debug/gendict.pch</PrecompiledHeaderOutputFile>
<AssemblerListingLocation>.\x86\Debug/</AssemblerListingLocation>
<ObjectFileName>.\x86\Debug/</ObjectFileName>
<ProgramDataBaseFileName>.\x86\Debug/</ProgramDataBaseFileName>
<BrowseInformation>true</BrowseInformation>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<DebugInformationFormat>EditAndContinue</DebugInformationFormat>
<CompileAs>Default</CompileAs>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<Culture>0x0409</Culture>
</ResourceCompile>
<Link>
<OutputFile>.\x86\Debug/gendict.exe</OutputFile>
<SuppressStartupBanner>true</SuppressStartupBanner>
<GenerateDebugInformation>true</GenerateDebugInformation>
<ProgramDatabaseFile>.\x86\Debug/gendict.pdb</ProgramDatabaseFile>
<SubSystem>Console</SubSystem>
<RandomizedBaseAddress>false</RandomizedBaseAddress>
<DataExecutionPrevention>
</DataExecutionPrevention>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<CustomBuildStep>
<Command>copy "$(TargetPath)" ..\..\..\bin64
</Command>
<Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
</CustomBuildStep>
<Midl>
<TargetEnvironment>X64</TargetEnvironment>
<TypeLibraryName>.\x64\Release/gendict.tlb</TypeLibraryName>
</Midl>
<ClCompile>
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN64;WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<StringPooling>true</StringPooling>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
<FunctionLevelLinking>true</FunctionLevelLinking>
<DisableLanguageExtensions>true</DisableLanguageExtensions>
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
<PrecompiledHeaderOutputFile>.\x64\Release/gendict.pch</PrecompiledHeaderOutputFile>
<AssemblerListingLocation>.\x64\Release/</AssemblerListingLocation>
<ObjectFileName>.\x64\Release/</ObjectFileName>
<ProgramDataBaseFileName>.\x64\Release/</ProgramDataBaseFileName>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<CompileAs>Default</CompileAs>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<Culture>0x0409</Culture>
</ResourceCompile>
<Link>
<OutputFile>.\x64\Release/gendict.exe</OutputFile>
<SuppressStartupBanner>true</SuppressStartupBanner>
<ProgramDatabaseFile>.\x64\Release/gendict.pdb</ProgramDatabaseFile>
<SubSystem>Console</SubSystem>
<TargetMachine>MachineX64</TargetMachine>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<CustomBuildStep>
<Command>copy "$(TargetPath)" ..\..\..\bin64
</Command>
<Outputs>..\..\..\bin64\$(TargetFileName);%(Outputs)</Outputs>
</CustomBuildStep>
<Midl>
<TargetEnvironment>X64</TargetEnvironment>
<TypeLibraryName>.\x64\Debug/gendict.tlb</TypeLibraryName>
</Midl>
<ClCompile>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>..\..\common;..\toolutil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN64;WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
<BufferSecurityCheck>true</BufferSecurityCheck>
<DisableLanguageExtensions>true</DisableLanguageExtensions>
<TreatWChar_tAsBuiltInType>true</TreatWChar_tAsBuiltInType>
<PrecompiledHeaderOutputFile>.\x64\Debug/gendict.pch</PrecompiledHeaderOutputFile>
<AssemblerListingLocation>.\x64\Debug/</AssemblerListingLocation>
<ObjectFileName>.\x64\Debug/</ObjectFileName>
<ProgramDataBaseFileName>.\x64\Debug/</ProgramDataBaseFileName>
<BrowseInformation>true</BrowseInformation>
<WarningLevel>Level3</WarningLevel>
<SuppressStartupBanner>true</SuppressStartupBanner>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<CompileAs>Default</CompileAs>
</ClCompile>
<ResourceCompile>
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<Culture>0x0409</Culture>
</ResourceCompile>
<Link>
<OutputFile>.\x64\Debug/gendict.exe</OutputFile>
<SuppressStartupBanner>true</SuppressStartupBanner>
<GenerateDebugInformation>true</GenerateDebugInformation>
<ProgramDatabaseFile>.\x64\Debug/gendict.pdb</ProgramDatabaseFile>
<SubSystem>Console</SubSystem>
<TargetMachine>MachineX64</TargetMachine>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="gendict.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\common\common.vcxproj">
<Project>{73c0a65b-d1f2-4de1-b3a6-15dad2c23f3d}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
<ProjectReference Include="..\toolutil\toolutil.vcxproj">
<Project>{6b231032-3cb5-4eed-9210-810d666a23a0}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

View file

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="Source Files">
<UniqueIdentifier>{570fb8ae-ac18-467d-8502-470a241a60d4}</UniqueIdentifier>
<Extensions>cpp;c;cxx;rc;def;r;odl;idl;hpj;bat</Extensions>
</Filter>
<Filter Include="Header Files">
<UniqueIdentifier>{7b2185f2-4ff9-4419-b596-0a21e37414c9}</UniqueIdentifier>
<Extensions>h;hpp;hxx;hm;inl</Extensions>
</Filter>
<Filter Include="Resource Files">
<UniqueIdentifier>{1dc5e7e3-4d1b-4031-a31f-c39b3a3e283a}</UniqueIdentifier>
<Extensions>ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="gendict.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
</Project>