mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-09-22 08:27:31 +09:00
Issue #1971 - Part 2: Update ICU source to 63.2.
This commit is contained in:
Job Bautista
roytam1
parent
8df84683be
commit
1e69214382
3160 changed files with 275815 additions and 234203 deletions
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright (C) 2016 and later: Unicode, Inc. and others.
|
||||
// © 2016 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
/*
|
||||
***************************************************************************
|
||||
|
|
@ -29,25 +29,15 @@
|
|||
#include "unicode/udata.h"
|
||||
#include "unicode/parseerr.h"
|
||||
#include "unicode/schriter.h"
|
||||
#include "unicode/uchriter.h"
|
||||
|
||||
|
||||
struct UTrie;
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
/** @internal */
|
||||
struct RBBIDataHeader;
|
||||
class RuleBasedBreakIteratorTables;
|
||||
class BreakIterator;
|
||||
class RBBIDataWrapper;
|
||||
class UStack;
|
||||
class LanguageBreakEngine;
|
||||
struct RBBIDataHeader;
|
||||
class RBBIDataWrapper;
|
||||
class UnhandledEngine;
|
||||
struct RBBIStateTable;
|
||||
|
||||
|
||||
|
||||
class UStack;
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -65,9 +55,69 @@ class U_COMMON_API RuleBasedBreakIterator /*U_FINAL*/ : public BreakIterator {
|
|||
private:
|
||||
/**
|
||||
* The UText through which this BreakIterator accesses the text
|
||||
* @internal (private)
|
||||
*/
|
||||
UText fText;
|
||||
|
||||
#ifndef U_HIDE_INTERNAL_API
|
||||
public:
|
||||
#endif /* U_HIDE_INTERNAL_API */
|
||||
/**
|
||||
* The rule data for this BreakIterator instance.
|
||||
* Not for general use; Public only for testing purposes.
|
||||
* @internal
|
||||
*/
|
||||
UText *fText;
|
||||
RBBIDataWrapper *fData;
|
||||
private:
|
||||
|
||||
/**
|
||||
* The current position of the iterator. Pinned, 0 < fPosition <= text.length.
|
||||
* Never has the value UBRK_DONE (-1).
|
||||
*/
|
||||
int32_t fPosition;
|
||||
|
||||
/**
|
||||
* TODO:
|
||||
*/
|
||||
int32_t fRuleStatusIndex;
|
||||
|
||||
/**
|
||||
* Cache of previously determined boundary positions.
|
||||
*/
|
||||
class BreakCache;
|
||||
BreakCache *fBreakCache;
|
||||
|
||||
/**
|
||||
* Cache of boundary positions within a region of text that has been
|
||||
* sub-divided by dictionary based breaking.
|
||||
*/
|
||||
class DictionaryCache;
|
||||
DictionaryCache *fDictionaryCache;
|
||||
|
||||
/**
|
||||
*
|
||||
* If present, UStack of LanguageBreakEngine objects that might handle
|
||||
* dictionary characters. Searched from top to bottom to find an object to
|
||||
* handle a given character.
|
||||
* @internal (private)
|
||||
*/
|
||||
UStack *fLanguageBreakEngines;
|
||||
|
||||
/**
|
||||
*
|
||||
* If present, the special LanguageBreakEngine used for handling
|
||||
* characters that are in the dictionary set, but not handled by any
|
||||
* LangugageBreakEngine.
|
||||
* @internal (private)
|
||||
*/
|
||||
UnhandledEngine *fUnhandledBreakEngine;
|
||||
|
||||
/**
|
||||
* Counter for the number of characters encountered with the "dictionary"
|
||||
* flag set.
|
||||
* @internal (private)
|
||||
*/
|
||||
uint32_t fDictionaryCharCount;
|
||||
|
||||
/**
|
||||
* A character iterator that refers to the same text as the UText, above.
|
||||
|
|
@ -81,87 +131,12 @@ private:
|
|||
* a characterIterator that wraps that data. Needed only for the
|
||||
* implementation of getText(), a backwards compatibility issue.
|
||||
*/
|
||||
StringCharacterIterator *fSCharIter;
|
||||
StringCharacterIterator fSCharIter;
|
||||
|
||||
/**
|
||||
* When the input text is provided by a UText, this
|
||||
* dummy CharacterIterator over an empty string will
|
||||
* be returned from getText()
|
||||
*/
|
||||
UCharCharacterIterator *fDCharIter;
|
||||
|
||||
/**
|
||||
* The rule data for this BreakIterator instance
|
||||
* @internal
|
||||
*/
|
||||
RBBIDataWrapper *fData;
|
||||
|
||||
/** Index of the Rule {tag} values for the most recent match.
|
||||
* @internal
|
||||
*/
|
||||
int32_t fLastRuleStatusIndex;
|
||||
|
||||
/**
|
||||
* Rule tag value valid flag.
|
||||
* Some iterator operations don't intrinsically set the correct tag value.
|
||||
* This flag lets us lazily compute the value if we are ever asked for it.
|
||||
* @internal
|
||||
*/
|
||||
UBool fLastStatusIndexValid;
|
||||
|
||||
/**
|
||||
* Counter for the number of characters encountered with the "dictionary"
|
||||
* flag set.
|
||||
* @internal
|
||||
*/
|
||||
uint32_t fDictionaryCharCount;
|
||||
|
||||
/**
|
||||
* When a range of characters is divided up using the dictionary, the break
|
||||
* positions that are discovered are stored here, preventing us from having
|
||||
* to use either the dictionary or the state table again until the iterator
|
||||
* leaves this range of text. Has the most impact for line breaking.
|
||||
* @internal
|
||||
*/
|
||||
int32_t* fCachedBreakPositions;
|
||||
|
||||
/**
|
||||
* The number of elements in fCachedBreakPositions
|
||||
* @internal
|
||||
*/
|
||||
int32_t fNumCachedBreakPositions;
|
||||
|
||||
/**
|
||||
* if fCachedBreakPositions is not null, this indicates which item in the
|
||||
* cache the current iteration position refers to
|
||||
* @internal
|
||||
*/
|
||||
int32_t fPositionInCache;
|
||||
|
||||
/**
|
||||
*
|
||||
* If present, UStack of LanguageBreakEngine objects that might handle
|
||||
* dictionary characters. Searched from top to bottom to find an object to
|
||||
* handle a given character.
|
||||
* @internal
|
||||
*/
|
||||
UStack *fLanguageBreakEngines;
|
||||
|
||||
/**
|
||||
*
|
||||
* If present, the special LanguageBreakEngine used for handling
|
||||
* characters that are in the dictionary set, but not handled by any
|
||||
* LangugageBreakEngine.
|
||||
* @internal
|
||||
*/
|
||||
UnhandledEngine *fUnhandledBreakEngine;
|
||||
|
||||
/**
|
||||
*
|
||||
* The type of the break iterator, or -1 if it has not been set.
|
||||
* @internal
|
||||
*/
|
||||
int32_t fBreakType;
|
||||
* True when iteration has run off the end, and iterator functions should return UBRK_DONE.
|
||||
*/
|
||||
UBool fDone;
|
||||
|
||||
//=======================================================================
|
||||
// constructors
|
||||
|
|
@ -175,17 +150,15 @@ private:
|
|||
*
|
||||
* The break iterator adopts the memory, and will
|
||||
* free it when done.
|
||||
* @internal
|
||||
* @internal (private)
|
||||
*/
|
||||
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
|
||||
|
||||
|
||||
/** @internal */
|
||||
friend class RBBIRuleBuilder;
|
||||
/** @internal */
|
||||
friend class BreakIterator;
|
||||
|
||||
|
||||
|
||||
public:
|
||||
|
||||
/** Default constructor. Creates an empty shell of an iterator, with no
|
||||
|
|
@ -215,17 +188,17 @@ public:
|
|||
UErrorCode &status);
|
||||
|
||||
/**
|
||||
* Contruct a RuleBasedBreakIterator from a set of precompiled binary rules.
|
||||
* Construct a RuleBasedBreakIterator from a set of precompiled binary rules.
|
||||
* Binary rules are obtained from RulesBasedBreakIterator::getBinaryRules().
|
||||
* Construction of a break iterator in this way is substantially faster than
|
||||
* constuction from source rules.
|
||||
* construction from source rules.
|
||||
*
|
||||
* Ownership of the storage containing the compiled rules remains with the
|
||||
* caller of this function. The compiled rules must not be modified or
|
||||
* deleted during the life of the break iterator.
|
||||
*
|
||||
* The compiled rules are not compatible across different major versions of ICU.
|
||||
* The compiled rules are comaptible only between machines with the same
|
||||
* The compiled rules are compatible only between machines with the same
|
||||
* byte ordering (little or big endian) and the same base character set family
|
||||
* (ASCII or EBCDIC).
|
||||
*
|
||||
|
|
@ -294,7 +267,7 @@ public:
|
|||
* behavior, and iterating over the same text, as this one.
|
||||
* Differs from the copy constructor in that it is polymorphic, and
|
||||
* will correctly clone (copy) a derived class.
|
||||
* clone() is thread safe. Multiple threads may simultaeneously
|
||||
* clone() is thread safe. Multiple threads may simultaneously
|
||||
* clone the same source break iterator.
|
||||
* @return a newly-constructed RuleBasedBreakIterator
|
||||
* @stable ICU 2.0
|
||||
|
|
@ -459,7 +432,7 @@ public:
|
|||
virtual int32_t preceding(int32_t offset);
|
||||
|
||||
/**
|
||||
* Returns true if the specfied position is a boundary position. As a side
|
||||
* Returns true if the specified position is a boundary position. As a side
|
||||
* effect, leaves the iterator pointing to the first boundary position at
|
||||
* or after "offset".
|
||||
* @param offset the offset to check.
|
||||
|
|
@ -469,7 +442,10 @@ public:
|
|||
virtual UBool isBoundary(int32_t offset);
|
||||
|
||||
/**
|
||||
* Returns the current iteration position.
|
||||
* Returns the current iteration position. Note that UBRK_DONE is never
|
||||
* returned from this function; if iteration has run to the end of a
|
||||
* string, current() will return the length of the string while
|
||||
* next() will return UBRK_DONE).
|
||||
* @return The current iteration position.
|
||||
* @stable ICU 2.0
|
||||
*/
|
||||
|
|
@ -477,8 +453,8 @@ public:
|
|||
|
||||
|
||||
/**
|
||||
* Return the status tag from the break rule that determined the most recently
|
||||
* returned break position. For break rules that do not specify a
|
||||
* Return the status tag from the break rule that determined the boundary at
|
||||
* the current iteration position. For break rules that do not specify a
|
||||
* status, a default value of 0 is returned. If more than one break rule
|
||||
* would cause a boundary to be located at some position in the text,
|
||||
* the numerically largest of the applicable status values is returned.
|
||||
|
|
@ -495,15 +471,14 @@ public:
|
|||
* position from <code>next()</code>, <code>previous()</code>, or
|
||||
* any other break iterator functions that returns a boundary position.
|
||||
* <p>
|
||||
* Note that <code>getRuleStatus()</code> returns the value corresponding to
|
||||
* <code>current()</code> index even after <code>next()</code> has returned DONE.
|
||||
* <p>
|
||||
* When creating custom break rules, one is free to define whatever
|
||||
* status values may be convenient for the application.
|
||||
* <p>
|
||||
* Note: this function is not thread safe. It should not have been
|
||||
* declared const, and the const remains only for compatibility
|
||||
* reasons. (The function is logically const, but not bit-wise const).
|
||||
* <p>
|
||||
* @return the status from the break rule that determined the most recently
|
||||
* returned break position.
|
||||
* @return the status from the break rule that determined the boundary
|
||||
* at the current iteration position.
|
||||
*
|
||||
* @see UWordBreak
|
||||
* @stable ICU 2.2
|
||||
|
|
@ -511,8 +486,8 @@ public:
|
|||
virtual int32_t getRuleStatus() const;
|
||||
|
||||
/**
|
||||
* Get the status (tag) values from the break rule(s) that determined the most
|
||||
* recently returned break position.
|
||||
* Get the status (tag) values from the break rule(s) that determined the boundary
|
||||
* at the current iteration position.
|
||||
* <p>
|
||||
* The returned status value(s) are stored into an array provided by the caller.
|
||||
* The values are stored in sorted (ascending) order.
|
||||
|
|
@ -523,10 +498,10 @@ public:
|
|||
* @param fillInVec an array to be filled in with the status values.
|
||||
* @param capacity the length of the supplied vector. A length of zero causes
|
||||
* the function to return the number of status values, in the
|
||||
* normal way, without attemtping to store any values.
|
||||
* normal way, without attempting to store any values.
|
||||
* @param status receives error codes.
|
||||
* @return The number of rule status values from rules that determined
|
||||
* the most recent boundary returned by the break iterator.
|
||||
* @return The number of rule status values from the rules that determined
|
||||
* the boundary at the current iteration position.
|
||||
* In the event of a U_BUFFER_OVERFLOW_ERROR, the return value
|
||||
* is the total number of status values that were available,
|
||||
* not the reduced number that were actually returned.
|
||||
|
|
@ -566,7 +541,7 @@ public:
|
|||
*
|
||||
* Create a clone (copy) of this break iterator in memory provided
|
||||
* by the caller. The idea is to increase performance by avoiding
|
||||
* a storage allocation. Use of this functoin is NOT RECOMMENDED.
|
||||
* a storage allocation. Use of this function is NOT RECOMMENDED.
|
||||
* Performance gains are minimal, and correct buffer management is
|
||||
* tricky. Use clone() instead.
|
||||
*
|
||||
|
|
@ -579,7 +554,7 @@ public:
|
|||
* storage for the cloned object.
|
||||
*
|
||||
* @param status Error status. U_SAFECLONE_ALLOCATED_WARNING will be
|
||||
* returned if the the provided buffer was too small, and
|
||||
* returned if the provided buffer was too small, and
|
||||
* the clone was therefore put on the heap.
|
||||
*
|
||||
* @return Pointer to the clone object. This may differ from the stackBuffer
|
||||
|
|
@ -602,7 +577,7 @@ public:
|
|||
* The binary data can only be used with the same version of ICU
|
||||
* and on the same platform type (processor endian-ness)
|
||||
*
|
||||
* @param length Returns the length of the binary data. (Out paramter.)
|
||||
* @param length Returns the length of the binary data. (Out parameter.)
|
||||
*
|
||||
* @return A pointer to the binary (compiled) rule data. The storage
|
||||
* belongs to the RulesBasedBreakIterator object, not the
|
||||
|
|
@ -646,75 +621,65 @@ private:
|
|||
/**
|
||||
* Dumps caches and performs other actions associated with a complete change
|
||||
* in text or iteration position.
|
||||
* @internal
|
||||
* @internal (private)
|
||||
*/
|
||||
void reset(void);
|
||||
|
||||
/**
|
||||
* Set the type of the break iterator.
|
||||
* @internal
|
||||
*/
|
||||
void setBreakType(int32_t type);
|
||||
|
||||
/**
|
||||
* Common initialization function, used by constructors and bufferClone.
|
||||
* @internal
|
||||
* @internal (private)
|
||||
*/
|
||||
void init();
|
||||
void init(UErrorCode &status);
|
||||
|
||||
/**
|
||||
* This method backs the iterator back up to a "safe position" in the text.
|
||||
* This is a position that we know, without any context, must be a break position.
|
||||
* The various calling methods then iterate forward from this safe position to
|
||||
* the appropriate position to return. (For more information, see the description
|
||||
* of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
|
||||
* @param statetable state table used of moving backwards
|
||||
* @internal
|
||||
* Iterate backwards from an arbitrary position in the input text using the
|
||||
* synthesized Safe Reverse rules.
|
||||
* This locates a "Safe Position" from which the forward break rules
|
||||
* will operate correctly. A Safe Position is not necessarily a boundary itself.
|
||||
*
|
||||
* @param fromPosition the position in the input text to begin the iteration.
|
||||
* @internal (private)
|
||||
*/
|
||||
int32_t handlePrevious(const RBBIStateTable *statetable);
|
||||
int32_t handleSafePrevious(int32_t fromPosition);
|
||||
|
||||
/**
|
||||
* This method is the actual implementation of the next() method. All iteration
|
||||
* vectors through here. This method initializes the state machine to state 1
|
||||
* and advances through the text character by character until we reach the end
|
||||
* of the text or the state machine transitions to state 0. We update our return
|
||||
* value every time the state machine passes through a possible end state.
|
||||
* @param statetable state table used of moving forwards
|
||||
* @internal
|
||||
* Find a rule-based boundary by running the state machine.
|
||||
* Input
|
||||
* fPosition, the position in the text to begin from.
|
||||
* Output
|
||||
* fPosition: the boundary following the starting position.
|
||||
* fDictionaryCharCount the number of dictionary characters encountered.
|
||||
* If > 0, the segment will be further subdivided
|
||||
* fRuleStatusIndex Info from the state table indicating which rules caused the boundary.
|
||||
*
|
||||
* @internal (private)
|
||||
*/
|
||||
int32_t handleNext(const RBBIStateTable *statetable);
|
||||
|
||||
|
||||
/**
|
||||
* This is the function that actually implements dictionary-based
|
||||
* breaking. Covering at least the range from startPos to endPos,
|
||||
* it checks for dictionary characters, and if it finds them determines
|
||||
* the appropriate object to deal with them. It may cache found breaks in
|
||||
* fCachedBreakPositions as it goes. It may well also look at text outside
|
||||
* the range startPos to endPos.
|
||||
* If going forward, endPos is the normal Unicode break result, and
|
||||
* if goind in reverse, startPos is the normal Unicode break result
|
||||
* @param startPos The start position of a range of text
|
||||
* @param endPos The end position of a range of text
|
||||
* @param reverse The call is for the reverse direction
|
||||
* @internal
|
||||
*/
|
||||
int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
|
||||
int32_t handleNext();
|
||||
|
||||
|
||||
/**
|
||||
* This function returns the appropriate LanguageBreakEngine for a
|
||||
* given character c.
|
||||
* @param c A character in the dictionary set
|
||||
* @internal
|
||||
* @internal (private)
|
||||
*/
|
||||
const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
|
||||
|
||||
public:
|
||||
#ifndef U_HIDE_INTERNAL_API
|
||||
/**
|
||||
* @internal
|
||||
* Debugging function only.
|
||||
* @internal
|
||||
*/
|
||||
void makeRuleStatusValid();
|
||||
void dumpCache();
|
||||
|
||||
/**
|
||||
* Debugging function only.
|
||||
* @internal
|
||||
*/
|
||||
void dumpTables();
|
||||
|
||||
#endif /* U_HIDE_INTERNAL_API */
|
||||
};
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue