/*
******************************************************************************
*
* Copyright (C) 2007-2012, International Business Machines
* Corporation and others. All Rights Reserved.
*
******************************************************************************
* file name: unisetspan.cpp
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2007mar01
* created by: Markus W. Scherer
*/
#include "unicode/utypes.h"
#include "unicode/uniset.h"
#include "unicode/ustring.h"
#include "unicode/utf8.h"
#include "unicode/utf16.h"
#include "cmemory.h"
#include "uvector.h"
#include "unisetspan.h"
U_NAMESPACE_BEGIN
/*
* List of offsets from the current position from where to try matching
* a code point or a string.
* Store offsets rather than indexes to simplify the code and use the same list
* for both increments (in span()) and decrements (in spanBack()).
*
* Assumption: The maximum offset is limited, and the offsets that are stored
* at any one time are relatively dense, that is, there are normally no gaps of
* hundreds or thousands of offset values.
*
* The implementation uses a circular buffer of byte flags,
* each indicating whether the corresponding offset is in the list.
* This avoids inserting into a sorted list of offsets (or absolute indexes) and
* physically moving part of the list.
*
* Note: In principle, the caller should setMaxLength() to the maximum of the
* max string length and U16_LENGTH/U8_LENGTH to account for
* "long" single code points.
* However, this implementation uses at least a staticList with more than
* U8_LENGTH entries anyway.
*
* Note: If maxLength were guaranteed to be no more than 32 or 64,
* the list could be stored as bit flags in a single integer.
* Rather than handling a circular buffer with a start list index,
* the integer would simply be shifted when lower offsets are removed.
* UnicodeSet does not have a limit on the lengths of strings.
*/
class OffsetList { // Only ever stack-allocated, does not need to inherit UMemory.
public:
OffsetList() : list(staticList), capacity(0), length(0), start(0) {}
~OffsetList() {
if(list!=staticList) {
uprv_free(list);
}
}
// Call exactly once if the list is to be used.
void setMaxLength(int32_t maxLength) {
if(maxLength<=(int32_t)sizeof(staticList)) {
capacity=(int32_t)sizeof(staticList);
} else {
UBool *l=(UBool *)uprv_malloc(maxLength);
if(l!=NULL) {
list=l;
capacity=maxLength;
}
}
uprv_memset(list, 0, capacity);
}
void clear() {
uprv_memset(list, 0, capacity);
start=length=0;
}
UBool isEmpty() const {
return (UBool)(length==0);
}
// Reduce all stored offsets by delta, used when the current position
// moves by delta.
// There must not be any offsets lower than delta.
// If there is an offset equal to delta, it is removed.
// delta=[1..maxLength]
void shift(int32_t delta) {
int32_t i=start+delta;
if(i>=capacity) {
i-=capacity;
}
if(list[i]) {
list[i]=FALSE;
--length;
}
start=i;
}
// Add an offset. The list must not contain it yet.
// offset=[1..maxLength]
void addOffset(int32_t offset) {
int32_t i=start+offset;
if(i>=capacity) {
i-=capacity;
}
list[i]=TRUE;
++length;
}
// offset=[1..maxLength]
UBool containsOffset(int32_t offset) const {
int32_t i=start+offset;
if(i>=capacity) {
i-=capacity;
}
return list[i];
}
// Find the lowest stored offset from a non-empty list, remove it,
// and reduce all other offsets by this minimum.
// Returns [1..maxLength].
int32_t popMinimum() {
// Look for the next offset in list[start+1..capacity-1].
int32_t i=start, result;
while(++i<capacity) {
if(list[i]) {
list[i]=FALSE;
--length;
result=i-start;
start=i;
return result;
}
}
// i==capacity
// Wrap around and look for the next offset in list[0..start].
// Since the list is not empty, there will be one.
result=capacity-start;
i=0;
while(!list[i]) {
++i;
}
list[i]=FALSE;
--length;
start=i;
return result+=i;
}
private:
UBool *list;
int32_t capacity;
int32_t length;
int32_t start;
UBool staticList[16];
};
// Get the number of UTF-8 bytes for a UTF-16 (sub)string.
static int32_t
getUTF8Length(const UChar *s, int32_t length) {
UErrorCode errorCode=U_ZERO_ERROR;
int32_t length8=0;
u_strToUTF8(NULL, 0, &length8, s, length, &errorCode);
if(U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR) {
return length8;
} else {
// The string contains an unpaired surrogate.
// Ignore this string.
return 0;
}
}
// Append the UTF-8 version of the string to t and return the appended UTF-8 length.
static int32_t
appendUTF8(const UChar *s, int32_t length, uint8_t *t, int32_t capacity) {
UErrorCode errorCode=U_ZERO_ERROR;
int32_t length8=0;
u_strToUTF8((char *)t, capacity, &length8, s, length, &errorCode);
if(U_SUCCESS(errorCode)) {
return length8;
} else {
// The string contains an unpaired surrogate.
// Ignore this string.
return 0;
}
}
static inline uint8_t
makeSpanLengthByte(int32_t spanLength) {
// 0xfe==UnicodeSetStringSpan::LONG_SPAN
return spanLength<0xfe ? (uint8_t)spanLength : (uint8_t)0xfe;
}
// Construct for all variants of span(), or only for any one variant.
// Initialize as little as possible, for single use.
UnicodeSetStringSpan::UnicodeSetStringSpan(const UnicodeSet &set,
const UVector &setStrings,
uint32_t which)
: spanSet(0, 0x10ffff), pSpanNotSet(NULL), strings(setStrings),
utf8Lengths(NULL), spanLengths(NULL), utf8(NULL),
utf8Length(0),
maxLength16(0), maxLength8(0),
all((UBool)(which==ALL)) {
spanSet.retainAll(set);
if(which&NOT_CONTAINED) {
// Default to the same sets.
// addToSpanNotSet() will create a separate set if necessary.
pSpanNotSet=&spanSet;
}
// Determine if the strings even need to be taken into account at all for span() etc.
// If any string is relevant, then all strings need to be used for
// span(longest match) but only the relevant ones for span(while contained).
// TODO: Possible optimization: Distinguish CONTAINED vs. LONGEST_MATCH
// and do not store UTF-8 strings if !thisRelevant and CONTAINED.
// (Only store irrelevant UTF-8 strings for LONGEST_MATCH where they are relevant after all.)
// Also count the lengths of the UTF-8 versions of the strings for memory allocation.
int32_t stringsLength=strings.size();
int32_t i, spanLength;
UBool someRelevant=FALSE;
for(i=0; i<stringsLength; ++i) {
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
UBool thisRelevant;
spanLength=spanSet.span(s16, length16, USET_SPAN_CONTAINED);
if(spanLength<length16) { // Relevant string.
someRelevant=thisRelevant=TRUE;
} else {
thisRelevant=FALSE;
}
if((which&UTF16) && length16>maxLength16) {
maxLength16=length16;
}
if((which&UTF8) && (thisRelevant || (which&CONTAINED))) {
int32_t length8=getUTF8Length(s16, length16);
utf8Length+=length8;
if(length8>maxLength8) {
maxLength8=length8;
}
}
}
if(!someRelevant) {
maxLength16=maxLength8=0;
return;
}
// Freeze after checking for the need to use strings at all because freezing
// a set takes some time and memory which are wasted if there are no relevant strings.
if(all) {
spanSet.freeze();
}
uint8_t *spanBackLengths;
uint8_t *spanUTF8Lengths;
uint8_t *spanBackUTF8Lengths;
// Allocate a block of meta data.
int32_t allocSize;
if(all) {
// UTF-8 lengths, 4 sets of span lengths, UTF-8 strings.
allocSize=stringsLength*(4+1+1+1+1)+utf8Length;
} else {
allocSize=stringsLength; // One set of span lengths.
if(which&UTF8) {
// UTF-8 lengths and UTF-8 strings.
allocSize+=stringsLength*4+utf8Length;
}
}
if(allocSize<=(int32_t)sizeof(staticLengths)) {
utf8Lengths=staticLengths;
} else {
utf8Lengths=(int32_t *)uprv_malloc(allocSize);
if(utf8Lengths==NULL) {
maxLength16=maxLength8=0; // Prevent usage by making needsStringSpanUTF16/8() return FALSE.
return; // Out of memory.
}
}
if(all) {
// Store span lengths for all span() variants.
spanLengths=(uint8_t *)(utf8Lengths+stringsLength);
spanBackLengths=spanLengths+stringsLength;
spanUTF8Lengths=spanBackLengths+stringsLength;
spanBackUTF8Lengths=spanUTF8Lengths+stringsLength;
utf8=spanBackUTF8Lengths+stringsLength;
} else {
// Store span lengths for only one span() variant.
if(which&UTF8) {
spanLengths=(uint8_t *)(utf8Lengths+stringsLength);
utf8=spanLengths+stringsLength;
} else {
spanLengths=(uint8_t *)utf8Lengths;
}
spanBackLengths=spanUTF8Lengths=spanBackUTF8Lengths=spanLengths;
}
// Set the meta data and pSpanNotSet and write the UTF-8 strings.
int32_t utf8Count=0; // Count UTF-8 bytes written so far.
for(i=0; i<stringsLength; ++i) {
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
spanLength=spanSet.span(s16, length16, USET_SPAN_CONTAINED);
if(spanLength<length16) { // Relevant string.
if(which&UTF16) {
if(which&CONTAINED) {
if(which&FWD) {
spanLengths[i]=makeSpanLengthByte(spanLength);
}
if(which&BACK) {
spanLength=length16-spanSet.spanBack(s16, length16, USET_SPAN_CONTAINED);
spanBackLengths[i]=makeSpanLengthByte(spanLength);
}
} else /* not CONTAINED, not all, but NOT_CONTAINED */ {
spanLengths[i]=spanBackLengths[i]=0; // Only store a relevant/irrelevant flag.
}
}
if(which&UTF8) {
uint8_t *s8=utf8+utf8Count;
int32_t length8=appendUTF8(s16, length16, s8, utf8Length-utf8Count);
utf8Count+=utf8Lengths[i]=length8;
if(length8==0) { // Irrelevant for UTF-8 because not representable in UTF-8.
spanUTF8Lengths[i]=spanBackUTF8Lengths[i]=(uint8_t)ALL_CP_CONTAINED;
} else { // Relevant for UTF-8.
if(which&CONTAINED) {
if(which&FWD) {
spanLength=spanSet.spanUTF8((const char *)s8, length8, USET_SPAN_CONTAINED);
spanUTF8Lengths[i]=makeSpanLengthByte(spanLength);
}
if(which&BACK) {
spanLength=length8-spanSet.spanBackUTF8((const char *)s8, length8, USET_SPAN_CONTAINED);
spanBackUTF8Lengths[i]=makeSpanLengthByte(spanLength);
}
} else /* not CONTAINED, not all, but NOT_CONTAINED */ {
spanUTF8Lengths[i]=spanBackUTF8Lengths[i]=0; // Only store a relevant/irrelevant flag.
}
}
}
if(which&NOT_CONTAINED) {
// Add string start and end code points to the spanNotSet so that
// a span(while not contained) stops before any string.
UChar32 c;
if(which&FWD) {
int32_t len=0;
U16_NEXT(s16, len, length16, c);
addToSpanNotSet(c);
}
if(which&BACK) {
int32_t len=length16;
U16_PREV(s16, 0, len, c);
addToSpanNotSet(c);
}
}
} else { // Irrelevant string.
if(which&UTF8) {
if(which&CONTAINED) { // Only necessary for LONGEST_MATCH.
uint8_t *s8=utf8+utf8Count;
int32_t length8=appendUTF8(s16, length16, s8, utf8Length-utf8Count);
utf8Count+=utf8Lengths[i]=length8;
} else {
utf8Lengths[i]=0;
}
}
if(all) {
spanLengths[i]=spanBackLengths[i]=
spanUTF8Lengths[i]=spanBackUTF8Lengths[i]=
(uint8_t)ALL_CP_CONTAINED;
} else {
// All spanXYZLengths pointers contain the same address.
spanLengths[i]=(uint8_t)ALL_CP_CONTAINED;
}
}
}
// Finish.
if(all) {
pSpanNotSet->freeze();
}
}
// Copy constructor. Assumes which==ALL for a frozen set.
UnicodeSetStringSpan::UnicodeSetStringSpan(const UnicodeSetStringSpan &otherStringSpan,
const UVector &newParentSetStrings)
: spanSet(otherStringSpan.spanSet), pSpanNotSet(NULL), strings(newParentSetStrings),
utf8Lengths(NULL), spanLengths(NULL), utf8(NULL),
utf8Length(otherStringSpan.utf8Length),
maxLength16(otherStringSpan.maxLength16), maxLength8(otherStringSpan.maxLength8),
all(TRUE) {
if(otherStringSpan.pSpanNotSet==&otherStringSpan.spanSet) {
pSpanNotSet=&spanSet;
} else {
pSpanNotSet=(UnicodeSet *)otherStringSpan.pSpanNotSet->clone();
}
// Allocate a block of meta data.
// UTF-8 lengths, 4 sets of span lengths, UTF-8 strings.
int32_t stringsLength=strings.size();
int32_t allocSize=stringsLength*(4+1+1+1+1)+utf8Length;
if(allocSize<=(int32_t)sizeof(staticLengths)) {
utf8Lengths=staticLengths;
} else {
utf8Lengths=(int32_t *)uprv_malloc(allocSize);
if(utf8Lengths==NULL) {
maxLength16=maxLength8=0; // Prevent usage by making needsStringSpanUTF16/8() return FALSE.
return; // Out of memory.
}
}
spanLengths=(uint8_t *)(utf8Lengths+stringsLength);
utf8=spanLengths+stringsLength*4;
uprv_memcpy(utf8Lengths, otherStringSpan.utf8Lengths, allocSize);
}
UnicodeSetStringSpan::~UnicodeSetStringSpan() {
if(pSpanNotSet!=NULL && pSpanNotSet!=&spanSet) {
delete pSpanNotSet;
}
if(utf8Lengths!=NULL && utf8Lengths!=staticLengths) {
uprv_free(utf8Lengths);
}
}
void UnicodeSetStringSpan::addToSpanNotSet(UChar32 c) {
if(pSpanNotSet==NULL || pSpanNotSet==&spanSet) {
if(spanSet.contains(c)) {
return; // Nothing to do.
}
UnicodeSet *newSet=(UnicodeSet *)spanSet.cloneAsThawed();
if(newSet==NULL) {
return; // Out of memory.
} else {
pSpanNotSet=newSet;
}
}
pSpanNotSet->add(c);
}
// Compare strings without any argument checks. Requires length>0.
static inline UBool
matches16(const UChar *s, const UChar *t, int32_t length) {
do {
if(*s++!=*t++) {
return FALSE;
}
} while(--length>0);
return TRUE;
}
static inline UBool
matches8(const uint8_t *s, const uint8_t *t, int32_t length) {
do {
if(*s++!=*t++) {
return FALSE;
}
} while(--length>0);
return TRUE;
}
// Compare 16-bit Unicode strings (which may be malformed UTF-16)
// at code point boundaries.
// That is, each edge of a match must not be in the middle of a surrogate pair.
static inline UBool
matches16CPB(const UChar *s, int32_t start, int32_t limit, const UChar *t, int32_t length) {
s+=start;
limit-=start;
return matches16(s, t, length) &&
!(0<start && U16_IS_LEAD(s[-1]) && U16_IS_TRAIL(s[0])) &&
!(length<limit && U16_IS_LEAD(s[length-1]) && U16_IS_TRAIL(s[length]));
}
// Does the set contain the next code point?
// If so, return its length; otherwise return its negative length.
static inline int32_t
spanOne(const UnicodeSet &set, const UChar *s, int32_t length) {
UChar c=*s, c2;
if(c>=0xd800 && c<=0xdbff && length>=2 && U16_IS_TRAIL(c2=s[1])) {
return set.contains(U16_GET_SUPPLEMENTARY(c, c2)) ? 2 : -2;
}
return set.contains(c) ? 1 : -1;
}
static inline int32_t
spanOneBack(const UnicodeSet &set, const UChar *s, int32_t length) {
UChar c=s[length-1], c2;
if(c>=0xdc00 && c<=0xdfff && length>=2 && U16_IS_LEAD(c2=s[length-2])) {
return set.contains(U16_GET_SUPPLEMENTARY(c2, c)) ? 2 : -2;
}
return set.contains(c) ? 1 : -1;
}
static inline int32_t
spanOneUTF8(const UnicodeSet &set, const uint8_t *s, int32_t length) {
UChar32 c=*s;
if((int8_t)c>=0) {
return set.contains(c) ? 1 : -1;
}
// Take advantage of non-ASCII fastpaths in U8_NEXT_OR_FFFD().
int32_t i=0;
U8_NEXT_OR_FFFD(s, i, length, c);
return set.contains(c) ? i : -i;
}
static inline int32_t
spanOneBackUTF8(const UnicodeSet &set, const uint8_t *s, int32_t length) {
UChar32 c=s[length-1];
if((int8_t)c>=0) {
return set.contains(c) ? 1 : -1;
}
int32_t i=length-1;
c=utf8_prevCharSafeBody(s, 0, &i, c, -3);
length-=i;
return set.contains(c) ? length : -length;
}
/*
* Note: In span() when spanLength==0 (after a string match, or at the beginning
* after an empty code point span) and in spanNot() and spanNotUTF8(),
* string matching could use a binary search
* because all string matches are done from the same start index.
*
* For UTF-8, this would require a comparison function that returns UTF-16 order.
*
* This optimization should not be necessary for normal UnicodeSets because
* most sets have no strings, and most sets with strings have
* very few very short strings.
* For cases with many strings, it might be better to use a different API
* and implementation with a DFA (state machine).
*/
/*
* Algorithm for span(USET_SPAN_CONTAINED)
*
* Theoretical algorithm:
* - Iterate through the string, and at each code point boundary:
* + If the code point there is in the set, then remember to continue after it.
* + If a set string matches at the current position, then remember to continue after it.
* + Either recursively span for each code point or string match,
* or recursively span for all but the shortest one and
* iteratively continue the span with the shortest local match.
* + Remember the longest recursive span (the farthest end point).
* + If there is no match at the current position, neither for the code point there
* nor for any set string, then stop and return the longest recursive span length.
*
* Optimized implementation:
*
* (We assume that most sets will have very few very short strings.
* A span using a string-less set is extremely fast.)
*
* Create and cache a spanSet which contains all of the single code points
* of the original set but none of its strings.
*
* - Start with spanLength=spanSet.span(USET_SPAN_CONTAINED).
* - Loop:
* + Try to match each set string at the end of the spanLength.
* ~ Set strings that start with set-contained code points must be matched
* with a partial overlap because the recursive algorithm would have tried
* to match them at every position.
* ~ Set strings that entirely consist of set-contained code points
* are irrelevant for span(USET_SPAN_CONTAINED) because the
* recursive algorithm would continue after them anyway
* and find the longest recursive match from their end.
* ~ Rather than recursing, note each end point of a set string match.
* + If no set string matched after spanSet.span(), then return
* with where the spanSet.span() ended.
* + If at least one set string matched after spanSet.span(), then
* pop the shortest string match end point and continue
* the loop, trying to match all set strings from there.
* + If at least one more set string matched after a previous string match,
* then test if the code point after the previous string match is also
* contained in the set.
* Continue the loop with the shortest end point of either this code point
* or a matching set string.
* + If no more set string matched after a previous string match,
* then try another spanLength=spanSet.span(USET_SPAN_CONTAINED).
* Stop if spanLength==0, otherwise continue the loop.
*
* By noting each end point of a set string match,
* the function visits each string position at most once and finishes
* in linear time.
*
* The recursive algorithm may visit the same string position many times
* if multiple paths lead to it and finishes in exponential time.
*/
/*
* Algorithm for span(USET_SPAN_SIMPLE)
*
* Theoretical algorithm:
* - Iterate through the string, and at each code point boundary:
* + If the code point there is in the set, then remember to continue after it.
* + If a set string matches at the current position, then remember to continue after it.
* + Continue from the farthest match position and ignore all others.
* + If there is no match at the current position,
* then stop and return the current position.
*
* Optimized implementation:
*
* (Same assumption and spanSet as above.)
*
* - Start with spanLength=spanSet.span(USET_SPAN_CONTAINED).
* - Loop:
* + Try to match each set string at the end of the spanLength.
* ~ Set strings that start with set-contained code points must be matched
* with a partial overlap because the standard algorithm would have tried
* to match them earlier.
* ~ Set strings that entirely consist of set-contained code points
* must be matched with a full overlap because the longest-match algorithm
* would hide set string matches that end earlier.
* Such set strings need not be matched earlier inside the code point span
* because the standard algorithm would then have continued after
* the set string match anyway.
* ~ Remember the longest set string match (farthest end point) from the earliest
* starting point.
* + If no set string matched after spanSet.span(), then return
* with where the spanSet.span() ended.
* + If at least one set string matched, then continue the loop after the
* longest match from the earliest position.
* + If no more set string matched after a previous string match,
* then try another spanLength=spanSet.span(USET_SPAN_CONTAINED).
* Stop if spanLength==0, otherwise continue the loop.
*/
int32_t UnicodeSetStringSpan::span(const UChar *s, int32_t length, USetSpanCondition spanCondition) const {
if(spanCondition==USET_SPAN_NOT_CONTAINED) {
return spanNot(s, length);
}
int32_t spanLength=spanSet.span(s, length, USET_SPAN_CONTAINED);
if(spanLength==length) {
return length;
}
// Consider strings; they may overlap with the span.
OffsetList offsets;
if(spanCondition==USET_SPAN_CONTAINED) {
// Use offset list to try all possibilities.
offsets.setMaxLength(maxLength16);
}
int32_t pos=spanLength, rest=length-pos;
int32_t i, stringsLength=strings.size();
for(;;) {
if(spanCondition==USET_SPAN_CONTAINED) {
for(i=0; i<stringsLength; ++i) {
int32_t overlap=spanLengths[i];
if(overlap==ALL_CP_CONTAINED) {
continue; // Irrelevant string.
}
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
// Try to match this string at pos-overlap..pos.
if(overlap>=LONG_SPAN) {
overlap=length16;
// While contained: No point matching fully inside the code point span.
U16_BACK_1(s16, 0, overlap); // Length of the string minus the last code point.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t inc=length16-overlap; // Keep overlap+inc==length16.
for(;;) {
if(inc>rest) {
break;
}
// Try to match if the increment is not listed already.
if(!offsets.containsOffset(inc) && matches16CPB(s, pos-overlap, length, s16, length16)) {
if(inc==rest) {
return length; // Reached the end of the string.
}
offsets.addOffset(inc);
}
if(overlap==0) {
break;
}
--overlap;
++inc;
}
}
} else /* USET_SPAN_SIMPLE */ {
int32_t maxInc=0, maxOverlap=0;
for(i=0; i<stringsLength; ++i) {
int32_t overlap=spanLengths[i];
// For longest match, we do need to try to match even an all-contained string
// to find the match from the earliest start.
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
// Try to match this string at pos-overlap..pos.
if(overlap>=LONG_SPAN) {
overlap=length16;
// Longest match: Need to match fully inside the code point span
// to find the match from the earliest start.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t inc=length16-overlap; // Keep overlap+inc==length16.
for(;;) {
if(inc>rest || overlap<maxOverlap) {
break;
}
// Try to match if the string is longer or starts earlier.
if( (overlap>maxOverlap || /* redundant overlap==maxOverlap && */ inc>maxInc) &&
matches16CPB(s, pos-overlap, length, s16, length16)
) {
maxInc=inc; // Longest match from earliest start.
maxOverlap=overlap;
break;
}
--overlap;
++inc;
}
}
if(maxInc!=0 || maxOverlap!=0) {
// Longest-match algorithm, and there was a string match.
// Simply continue after it.
pos+=maxInc;
rest-=maxInc;
if(rest==0) {
return length; // Reached the end of the string.
}
spanLength=0; // Match strings from after a string match.
continue;
}
}
// Finished trying to match all strings at pos.
if(spanLength!=0 || pos==0) {
// The position is after an unlimited code point span (spanLength!=0),
// not after a string match.
// The only position where spanLength==0 after a span is pos==0.
// Otherwise, an unlimited code point span is only tried again when no
// strings match, and if such a non-initial span fails we stop.
if(offsets.isEmpty()) {
return pos; // No strings matched after a span.
}
// Match strings from after the next string match.
} else {
// The position is after a string match (or a single code point).
if(offsets.isEmpty()) {
// No more strings matched after a previous string match.
// Try another code point span from after the last string match.
spanLength=spanSet.span(s+pos, rest, USET_SPAN_CONTAINED);
if( spanLength==rest || // Reached the end of the string, or
spanLength==0 // neither strings nor span progressed.
) {
return pos+spanLength;
}
pos+=spanLength;
rest-=spanLength;
continue; // spanLength>0: Match strings from after a span.
} else {
// Try to match only one code point from after a string match if some
// string matched beyond it, so that we try all possible positions
// and don't overshoot.
spanLength=spanOne(spanSet, s+pos, rest);
if(spanLength>0) {
if(spanLength==rest) {
return length; // Reached the end of the string.
}
// Match strings after this code point.
// There cannot be any increments below it because UnicodeSet strings
// contain multiple code points.
pos+=spanLength;
rest-=spanLength;
offsets.shift(spanLength);
spanLength=0;
continue; // Match strings from after a single code point.
}
// Match strings from after the next string match.
}
}
int32_t minOffset=offsets.popMinimum();
pos+=minOffset;
rest-=minOffset;
spanLength=0; // Match strings from after a string match.
}
}
int32_t UnicodeSetStringSpan::spanBack(const UChar *s, int32_t length, USetSpanCondition spanCondition) const {
if(spanCondition==USET_SPAN_NOT_CONTAINED) {
return spanNotBack(s, length);
}
int32_t pos=spanSet.spanBack(s, length, USET_SPAN_CONTAINED);
if(pos==0) {
return 0;
}
int32_t spanLength=length-pos;
// Consider strings; they may overlap with the span.
OffsetList offsets;
if(spanCondition==USET_SPAN_CONTAINED) {
// Use offset list to try all possibilities.
offsets.setMaxLength(maxLength16);
}
int32_t i, stringsLength=strings.size();
uint8_t *spanBackLengths=spanLengths;
if(all) {
spanBackLengths+=stringsLength;
}
for(;;) {
if(spanCondition==USET_SPAN_CONTAINED) {
for(i=0; i<stringsLength; ++i) {
int32_t overlap=spanBackLengths[i];
if(overlap==ALL_CP_CONTAINED) {
continue; // Irrelevant string.
}
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
// Try to match this string at pos-(length16-overlap)..pos-length16.
if(overlap>=LONG_SPAN) {
overlap=length16;
// While contained: No point matching fully inside the code point span.
int32_t len1=0;
U16_FWD_1(s16, len1, overlap);
overlap-=len1; // Length of the string minus the first code point.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t dec=length16-overlap; // Keep dec+overlap==length16.
for(;;) {
if(dec>pos) {
break;
}
// Try to match if the decrement is not listed already.
if(!offsets.containsOffset(dec) && matches16CPB(s, pos-dec, length, s16, length16)) {
if(dec==pos) {
return 0; // Reached the start of the string.
}
offsets.addOffset(dec);
}
if(overlap==0) {
break;
}
--overlap;
++dec;
}
}
} else /* USET_SPAN_SIMPLE */ {
int32_t maxDec=0, maxOverlap=0;
for(i=0; i<stringsLength; ++i) {
int32_t overlap=spanBackLengths[i];
// For longest match, we do need to try to match even an all-contained string
// to find the match from the latest end.
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
// Try to match this string at pos-(length16-overlap)..pos-length16.
if(overlap>=LONG_SPAN) {
overlap=length16;
// Longest match: Need to match fully inside the code point span
// to find the match from the latest end.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t dec=length16-overlap; // Keep dec+overlap==length16.
for(;;) {
if(dec>pos || overlap<maxOverlap) {
break;
}
// Try to match if the string is longer or ends later.
if( (overlap>maxOverlap || /* redundant overlap==maxOverlap && */ dec>maxDec) &&
matches16CPB(s, pos-dec, length, s16, length16)
) {
maxDec=dec; // Longest match from latest end.
maxOverlap=overlap;
break;
}
--overlap;
++dec;
}
}
if(maxDec!=0 || maxOverlap!=0) {
// Longest-match algorithm, and there was a string match.
// Simply continue before it.
pos-=maxDec;
if(pos==0) {
return 0; // Reached the start of the string.
}
spanLength=0; // Match strings from before a string match.
continue;
}
}
// Finished trying to match all strings at pos.
if(spanLength!=0 || pos==length) {
// The position is before an unlimited code point span (spanLength!=0),
// not before a string match.
// The only position where spanLength==0 before a span is pos==length.
// Otherwise, an unlimited code point span is only tried again when no
// strings match, and if such a non-initial span fails we stop.
if(offsets.isEmpty()) {
return pos; // No strings matched before a span.
}
// Match strings from before the next string match.
} else {
// The position is before a string match (or a single code point).
if(offsets.isEmpty()) {
// No more strings matched before a previous string match.
// Try another code point span from before the last string match.
int32_t oldPos=pos;
pos=spanSet.spanBack(s, oldPos, USET_SPAN_CONTAINED);
spanLength=oldPos-pos;
if( pos==0 || // Reached the start of the string, or
spanLength==0 // neither strings nor span progressed.
) {
return pos;
}
continue; // spanLength>0: Match strings from before a span.
} else {
// Try to match only one code point from before a string match if some
// string matched beyond it, so that we try all possible positions
// and don't overshoot.
spanLength=spanOneBack(spanSet, s, pos);
if(spanLength>0) {
if(spanLength==pos) {
return 0; // Reached the start of the string.
}
// Match strings before this code point.
// There cannot be any decrements below it because UnicodeSet strings
// contain multiple code points.
pos-=spanLength;
offsets.shift(spanLength);
spanLength=0;
continue; // Match strings from before a single code point.
}
// Match strings from before the next string match.
}
}
pos-=offsets.popMinimum();
spanLength=0; // Match strings from before a string match.
}
}
int32_t UnicodeSetStringSpan::spanUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const {
if(spanCondition==USET_SPAN_NOT_CONTAINED) {
return spanNotUTF8(s, length);
}
int32_t spanLength=spanSet.spanUTF8((const char *)s, length, USET_SPAN_CONTAINED);
if(spanLength==length) {
return length;
}
// Consider strings; they may overlap with the span.
OffsetList offsets;
if(spanCondition==USET_SPAN_CONTAINED) {
// Use offset list to try all possibilities.
offsets.setMaxLength(maxLength8);
}
int32_t pos=spanLength, rest=length-pos;
int32_t i, stringsLength=strings.size();
uint8_t *spanUTF8Lengths=spanLengths;
if(all) {
spanUTF8Lengths+=2*stringsLength;
}
for(;;) {
const uint8_t *s8=utf8;
int32_t length8;
if(spanCondition==USET_SPAN_CONTAINED) {
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
if(length8==0) {
continue; // String not representable in UTF-8.
}
int32_t overlap=spanUTF8Lengths[i];
if(overlap==ALL_CP_CONTAINED) {
s8+=length8;
continue; // Irrelevant string.
}
// Try to match this string at pos-overlap..pos.
if(overlap>=LONG_SPAN) {
overlap=length8;
// While contained: No point matching fully inside the code point span.
U8_BACK_1(s8, 0, overlap); // Length of the string minus the last code point.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t inc=length8-overlap; // Keep overlap+inc==length8.
for(;;) {
if(inc>rest) {
break;
}
// Try to match if the increment is not listed already.
// Match at code point boundaries. (The UTF-8 strings were converted
// from UTF-16 and are guaranteed to be well-formed.)
if( !U8_IS_TRAIL(s[pos-overlap]) &&
!offsets.containsOffset(inc) &&
matches8(s+pos-overlap, s8, length8)
) {
if(inc==rest) {
return length; // Reached the end of the string.
}
offsets.addOffset(inc);
}
if(overlap==0) {
break;
}
--overlap;
++inc;
}
s8+=length8;
}
} else /* USET_SPAN_SIMPLE */ {
int32_t maxInc=0, maxOverlap=0;
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
if(length8==0) {
continue; // String not representable in UTF-8.
}
int32_t overlap=spanUTF8Lengths[i];
// For longest match, we do need to try to match even an all-contained string
// to find the match from the earliest start.
// Try to match this string at pos-overlap..pos.
if(overlap>=LONG_SPAN) {
overlap=length8;
// Longest match: Need to match fully inside the code point span
// to find the match from the earliest start.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t inc=length8-overlap; // Keep overlap+inc==length8.
for(;;) {
if(inc>rest || overlap<maxOverlap) {
break;
}
// Try to match if the string is longer or starts earlier.
// Match at code point boundaries. (The UTF-8 strings were converted
// from UTF-16 and are guaranteed to be well-formed.)
if( !U8_IS_TRAIL(s[pos-overlap]) &&
(overlap>maxOverlap || /* redundant overlap==maxOverlap && */ inc>maxInc) &&
matches8(s+pos-overlap, s8, length8)
) {
maxInc=inc; // Longest match from earliest start.
maxOverlap=overlap;
break;
}
--overlap;
++inc;
}
s8+=length8;
}
if(maxInc!=0 || maxOverlap!=0) {
// Longest-match algorithm, and there was a string match.
// Simply continue after it.
pos+=maxInc;
rest-=maxInc;
if(rest==0) {
return length; // Reached the end of the string.
}
spanLength=0; // Match strings from after a string match.
continue;
}
}
// Finished trying to match all strings at pos.
if(spanLength!=0 || pos==0) {
// The position is after an unlimited code point span (spanLength!=0),
// not after a string match.
// The only position where spanLength==0 after a span is pos==0.
// Otherwise, an unlimited code point span is only tried again when no
// strings match, and if such a non-initial span fails we stop.
if(offsets.isEmpty()) {
return pos; // No strings matched after a span.
}
// Match strings from after the next string match.
} else {
// The position is after a string match (or a single code point).
if(offsets.isEmpty()) {
// No more strings matched after a previous string match.
// Try another code point span from after the last string match.
spanLength=spanSet.spanUTF8((const char *)s+pos, rest, USET_SPAN_CONTAINED);
if( spanLength==rest || // Reached the end of the string, or
spanLength==0 // neither strings nor span progressed.
) {
return pos+spanLength;
}
pos+=spanLength;
rest-=spanLength;
continue; // spanLength>0: Match strings from after a span.
} else {
// Try to match only one code point from after a string match if some
// string matched beyond it, so that we try all possible positions
// and don't overshoot.
spanLength=spanOneUTF8(spanSet, s+pos, rest);
if(spanLength>0) {
if(spanLength==rest) {
return length; // Reached the end of the string.
}
// Match strings after this code point.
// There cannot be any increments below it because UnicodeSet strings
// contain multiple code points.
pos+=spanLength;
rest-=spanLength;
offsets.shift(spanLength);
spanLength=0;
continue; // Match strings from after a single code point.
}
// Match strings from after the next string match.
}
}
int32_t minOffset=offsets.popMinimum();
pos+=minOffset;
rest-=minOffset;
spanLength=0; // Match strings from after a string match.
}
}
int32_t UnicodeSetStringSpan::spanBackUTF8(const uint8_t *s, int32_t length, USetSpanCondition spanCondition) const {
if(spanCondition==USET_SPAN_NOT_CONTAINED) {
return spanNotBackUTF8(s, length);
}
int32_t pos=spanSet.spanBackUTF8((const char *)s, length, USET_SPAN_CONTAINED);
if(pos==0) {
return 0;
}
int32_t spanLength=length-pos;
// Consider strings; they may overlap with the span.
OffsetList offsets;
if(spanCondition==USET_SPAN_CONTAINED) {
// Use offset list to try all possibilities.
offsets.setMaxLength(maxLength8);
}
int32_t i, stringsLength=strings.size();
uint8_t *spanBackUTF8Lengths=spanLengths;
if(all) {
spanBackUTF8Lengths+=3*stringsLength;
}
for(;;) {
const uint8_t *s8=utf8;
int32_t length8;
if(spanCondition==USET_SPAN_CONTAINED) {
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
if(length8==0) {
continue; // String not representable in UTF-8.
}
int32_t overlap=spanBackUTF8Lengths[i];
if(overlap==ALL_CP_CONTAINED) {
s8+=length8;
continue; // Irrelevant string.
}
// Try to match this string at pos-(length8-overlap)..pos-length8.
if(overlap>=LONG_SPAN) {
overlap=length8;
// While contained: No point matching fully inside the code point span.
int32_t len1=0;
U8_FWD_1(s8, len1, overlap);
overlap-=len1; // Length of the string minus the first code point.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t dec=length8-overlap; // Keep dec+overlap==length8.
for(;;) {
if(dec>pos) {
break;
}
// Try to match if the decrement is not listed already.
// Match at code point boundaries. (The UTF-8 strings were converted
// from UTF-16 and are guaranteed to be well-formed.)
if( !U8_IS_TRAIL(s[pos-dec]) &&
!offsets.containsOffset(dec) &&
matches8(s+pos-dec, s8, length8)
) {
if(dec==pos) {
return 0; // Reached the start of the string.
}
offsets.addOffset(dec);
}
if(overlap==0) {
break;
}
--overlap;
++dec;
}
s8+=length8;
}
} else /* USET_SPAN_SIMPLE */ {
int32_t maxDec=0, maxOverlap=0;
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
if(length8==0) {
continue; // String not representable in UTF-8.
}
int32_t overlap=spanBackUTF8Lengths[i];
// For longest match, we do need to try to match even an all-contained string
// to find the match from the latest end.
// Try to match this string at pos-(length8-overlap)..pos-length8.
if(overlap>=LONG_SPAN) {
overlap=length8;
// Longest match: Need to match fully inside the code point span
// to find the match from the latest end.
}
if(overlap>spanLength) {
overlap=spanLength;
}
int32_t dec=length8-overlap; // Keep dec+overlap==length8.
for(;;) {
if(dec>pos || overlap<maxOverlap) {
break;
}
// Try to match if the string is longer or ends later.
// Match at code point boundaries. (The UTF-8 strings were converted
// from UTF-16 and are guaranteed to be well-formed.)
if( !U8_IS_TRAIL(s[pos-dec]) &&
(overlap>maxOverlap || /* redundant overlap==maxOverlap && */ dec>maxDec) &&
matches8(s+pos-dec, s8, length8)
) {
maxDec=dec; // Longest match from latest end.
maxOverlap=overlap;
break;
}
--overlap;
++dec;
}
s8+=length8;
}
if(maxDec!=0 || maxOverlap!=0) {
// Longest-match algorithm, and there was a string match.
// Simply continue before it.
pos-=maxDec;
if(pos==0) {
return 0; // Reached the start of the string.
}
spanLength=0; // Match strings from before a string match.
continue;
}
}
// Finished trying to match all strings at pos.
if(spanLength!=0 || pos==length) {
// The position is before an unlimited code point span (spanLength!=0),
// not before a string match.
// The only position where spanLength==0 before a span is pos==length.
// Otherwise, an unlimited code point span is only tried again when no
// strings match, and if such a non-initial span fails we stop.
if(offsets.isEmpty()) {
return pos; // No strings matched before a span.
}
// Match strings from before the next string match.
} else {
// The position is before a string match (or a single code point).
if(offsets.isEmpty()) {
// No more strings matched before a previous string match.
// Try another code point span from before the last string match.
int32_t oldPos=pos;
pos=spanSet.spanBackUTF8((const char *)s, oldPos, USET_SPAN_CONTAINED);
spanLength=oldPos-pos;
if( pos==0 || // Reached the start of the string, or
spanLength==0 // neither strings nor span progressed.
) {
return pos;
}
continue; // spanLength>0: Match strings from before a span.
} else {
// Try to match only one code point from before a string match if some
// string matched beyond it, so that we try all possible positions
// and don't overshoot.
spanLength=spanOneBackUTF8(spanSet, s, pos);
if(spanLength>0) {
if(spanLength==pos) {
return 0; // Reached the start of the string.
}
// Match strings before this code point.
// There cannot be any decrements below it because UnicodeSet strings
// contain multiple code points.
pos-=spanLength;
offsets.shift(spanLength);
spanLength=0;
continue; // Match strings from before a single code point.
}
// Match strings from before the next string match.
}
}
pos-=offsets.popMinimum();
spanLength=0; // Match strings from before a string match.
}
}
/*
* Algorithm for spanNot()==span(USET_SPAN_NOT_CONTAINED)
*
* Theoretical algorithm:
* - Iterate through the string, and at each code point boundary:
* + If the code point there is in the set, then return with the current position.
* + If a set string matches at the current position, then return with the current position.
*
* Optimized implementation:
*
* (Same assumption as for span() above.)
*
* Create and cache a spanNotSet which contains all of the single code points
* of the original set but none of its strings.
* For each set string add its initial code point to the spanNotSet.
* (Also add its final code point for spanNotBack().)
*
* - Loop:
* + Do spanLength=spanNotSet.span(USET_SPAN_NOT_CONTAINED).
* + If the current code point is in the original set, then
* return the current position.
* + If any set string matches at the current position, then
* return the current position.
* + If there is no match at the current position, neither for the code point there
* nor for any set string, then skip this code point and continue the loop.
* This happens for set-string-initial code points that were added to spanNotSet
* when there is not actually a match for such a set string.
*/
int32_t UnicodeSetStringSpan::spanNot(const UChar *s, int32_t length) const {
int32_t pos=0, rest=length;
int32_t i, stringsLength=strings.size();
do {
// Span until we find a code point from the set,
// or a code point that starts or ends some string.
i=pSpanNotSet->span(s+pos, rest, USET_SPAN_NOT_CONTAINED);
if(i==rest) {
return length; // Reached the end of the string.
}
pos+=i;
rest-=i;
// Check whether the current code point is in the original set,
// without the string starts and ends.
int32_t cpLength=spanOne(spanSet, s+pos, rest);
if(cpLength>0) {
return pos; // There is a set element at pos.
}
// Try to match the strings at pos.
for(i=0; i<stringsLength; ++i) {
if(spanLengths[i]==ALL_CP_CONTAINED) {
continue; // Irrelevant string.
}
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
if(length16<=rest && matches16CPB(s, pos, length, s16, length16)) {
return pos; // There is a set element at pos.
}
}
// The span(while not contained) ended on a string start/end which is
// not in the original set. Skip this code point and continue.
// cpLength<0
pos-=cpLength;
rest+=cpLength;
} while(rest!=0);
return length; // Reached the end of the string.
}
int32_t UnicodeSetStringSpan::spanNotBack(const UChar *s, int32_t length) const {
int32_t pos=length;
int32_t i, stringsLength=strings.size();
do {
// Span until we find a code point from the set,
// or a code point that starts or ends some string.
pos=pSpanNotSet->spanBack(s, pos, USET_SPAN_NOT_CONTAINED);
if(pos==0) {
return 0; // Reached the start of the string.
}
// Check whether the current code point is in the original set,
// without the string starts and ends.
int32_t cpLength=spanOneBack(spanSet, s, pos);
if(cpLength>0) {
return pos; // There is a set element at pos.
}
// Try to match the strings at pos.
for(i=0; i<stringsLength; ++i) {
// Use spanLengths rather than a spanBackLengths pointer because
// it is easier and we only need to know whether the string is irrelevant
// which is the same in either array.
if(spanLengths[i]==ALL_CP_CONTAINED) {
continue; // Irrelevant string.
}
const UnicodeString &string=*(const UnicodeString *)strings.elementAt(i);
const UChar *s16=string.getBuffer();
int32_t length16=string.length();
if(length16<=pos && matches16CPB(s, pos-length16, length, s16, length16)) {
return pos; // There is a set element at pos.
}
}
// The span(while not contained) ended on a string start/end which is
// not in the original set. Skip this code point and continue.
// cpLength<0
pos+=cpLength;
} while(pos!=0);
return 0; // Reached the start of the string.
}
int32_t UnicodeSetStringSpan::spanNotUTF8(const uint8_t *s, int32_t length) const {
int32_t pos=0, rest=length;
int32_t i, stringsLength=strings.size();
uint8_t *spanUTF8Lengths=spanLengths;
if(all) {
spanUTF8Lengths+=2*stringsLength;
}
do {
// Span until we find a code point from the set,
// or a code point that starts or ends some string.
i=pSpanNotSet->spanUTF8((const char *)s+pos, rest, USET_SPAN_NOT_CONTAINED);
if(i==rest) {
return length; // Reached the end of the string.
}
pos+=i;
rest-=i;
// Check whether the current code point is in the original set,
// without the string starts and ends.
int32_t cpLength=spanOneUTF8(spanSet, s+pos, rest);
if(cpLength>0) {
return pos; // There is a set element at pos.
}
// Try to match the strings at pos.
const uint8_t *s8=utf8;
int32_t length8;
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
// ALL_CP_CONTAINED: Irrelevant string.
if(length8!=0 && spanUTF8Lengths[i]!=ALL_CP_CONTAINED && length8<=rest && matches8(s+pos, s8, length8)) {
return pos; // There is a set element at pos.
}
s8+=length8;
}
// The span(while not contained) ended on a string start/end which is
// not in the original set. Skip this code point and continue.
// cpLength<0
pos-=cpLength;
rest+=cpLength;
} while(rest!=0);
return length; // Reached the end of the string.
}
int32_t UnicodeSetStringSpan::spanNotBackUTF8(const uint8_t *s, int32_t length) const {
int32_t pos=length;
int32_t i, stringsLength=strings.size();
uint8_t *spanBackUTF8Lengths=spanLengths;
if(all) {
spanBackUTF8Lengths+=3*stringsLength;
}
do {
// Span until we find a code point from the set,
// or a code point that starts or ends some string.
pos=pSpanNotSet->spanBackUTF8((const char *)s, pos, USET_SPAN_NOT_CONTAINED);
if(pos==0) {
return 0; // Reached the start of the string.
}
// Check whether the current code point is in the original set,
// without the string starts and ends.
int32_t cpLength=spanOneBackUTF8(spanSet, s, pos);
if(cpLength>0) {
return pos; // There is a set element at pos.
}
// Try to match the strings at pos.
const uint8_t *s8=utf8;
int32_t length8;
for(i=0; i<stringsLength; ++i) {
length8=utf8Lengths[i];
// ALL_CP_CONTAINED: Irrelevant string.
if(length8!=0 && spanBackUTF8Lengths[i]!=ALL_CP_CONTAINED && length8<=pos && matches8(s+pos-length8, s8, length8)) {
return pos; // There is a set element at pos.
}
s8+=length8;
}
// The span(while not contained) ended on a string start/end which is
// not in the original set. Skip this code point and continue.
// cpLength<0
pos+=cpLength;
} while(pos!=0);
return 0; // Reached the start of the string.
}
U_NAMESPACE_END