// © 2017 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
// norms.cpp
// created: 2017jun04 Markus W. Scherer
// (pulled out of n2builder.cpp)
#include "unicode/utypes.h"
#if !UCONFIG_NO_NORMALIZATION
#include <stdio.h>
#include <stdlib.h>
#include "unicode/errorcode.h"
#include "unicode/umutablecptrie.h"
#include "unicode/unistr.h"
#include "unicode/utf16.h"
#include "normalizer2impl.h"
#include "norms.h"
#include "toolutil.h"
#include "uvectr32.h"
U_NAMESPACE_BEGIN
void BuilderReorderingBuffer::append(UChar32 c, uint8_t cc) {
if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
if(cc==0) {
fLastStarterIndex=fLength;
}
fArray[fLength++]=(c<<8)|cc;
return;
}
// Let this character bubble back to its canonical order.
int32_t i=fLength-1;
while(i>fLastStarterIndex && ccAt(i)>cc) {
--i;
}
++i; // after the last starter or prevCC<=cc
// Move this and the following characters forward one to make space.
for(int32_t j=fLength; i<j; --j) {
fArray[j]=fArray[j-1];
}
fArray[i]=(c<<8)|cc;
++fLength;
fDidReorder=TRUE;
}
void BuilderReorderingBuffer::toString(UnicodeString &dest) const {
dest.remove();
for(int32_t i=0; i<fLength; ++i) {
dest.append(charAt(i));
}
}
UChar32 Norm::combine(UChar32 trail) const {
int32_t length;
const CompositionPair *pairs=getCompositionPairs(length);
for(int32_t i=0; i<length; ++i) {
if(trail==pairs[i].trail) {
return pairs[i].composite;
}
if(trail<pairs[i].trail) {
break;
}
}
return U_SENTINEL;
}
Norms::Norms(UErrorCode &errorCode) {
normTrie = umutablecptrie_open(0, 0, &errorCode);
normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
// Default "inert" Norm struct at index 0. Practically immutable.
norms=allocNorm();
norms->type=Norm::INERT;
}
Norms::~Norms() {
umutablecptrie_close(normTrie);
int32_t normsLength=utm_countItems(normMem);
for(int32_t i=1; i<normsLength; ++i) {
delete norms[i].mapping;
delete norms[i].rawMapping;
delete norms[i].compositions;
}
utm_close(normMem);
}
Norm *Norms::allocNorm() {
Norm *p=(Norm *)utm_alloc(normMem);
norms=(Norm *)utm_getStart(normMem); // in case it got reallocated
return p;
}
Norm *Norms::getNorm(UChar32 c) {
uint32_t i = umutablecptrie_get(normTrie, c);
if(i==0) {
return nullptr;
}
return norms+i;
}
const Norm *Norms::getNorm(UChar32 c) const {
uint32_t i = umutablecptrie_get(normTrie, c);
if(i==0) {
return nullptr;
}
return norms+i;
}
const Norm &Norms::getNormRef(UChar32 c) const {
return norms[umutablecptrie_get(normTrie, c)];
}
Norm *Norms::createNorm(UChar32 c) {
uint32_t i=umutablecptrie_get(normTrie, c);
if(i!=0) {
return norms+i;
} else {
/* allocate Norm */
Norm *p=allocNorm();
IcuToolErrorCode errorCode("gennorm2/createNorm()");
umutablecptrie_set(normTrie, c, (uint32_t)(p - norms), errorCode);
return p;
}
}
void Norms::reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const {
int32_t length=mapping.length();
U_ASSERT(length<=Normalizer2Impl::MAPPING_LENGTH_MASK);
const char16_t *s=mapping.getBuffer();
int32_t i=0;
UChar32 c;
while(i<length) {
U16_NEXT(s, i, length, c);
buffer.append(c, getCC(c));
}
if(buffer.didReorder()) {
buffer.toString(mapping);
}
}
UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const {
if((highCC-lowCC)>=2) {
int32_t length;
const CompositionPair *pairs=norm.getCompositionPairs(length);
for(int32_t i=0; i<length; ++i) {
uint8_t trailCC=getCC(pairs[i].trail);
if(lowCC<trailCC && trailCC<highCC) {
return TRUE;
}
}
}
return FALSE;
}
void Norms::enumRanges(Enumerator &e) {
UChar32 start = 0, end;
uint32_t i;
while ((end = umutablecptrie_getRange(normTrie, start, UCPMAP_RANGE_NORMAL, 0,
nullptr, nullptr, &i)) >= 0) {
if (i > 0) {
e.rangeHandler(start, end, norms[i]);
}
start = end + 1;
}
}
Norms::Enumerator::~Enumerator() {}
void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
if(norm.mappingType!=Norm::ROUND_TRIP) { return; }
if(start!=end) {
fprintf(stderr,
"gennorm2 error: same round-trip mapping for "
"more than 1 code point U+%04lX..U+%04lX\n",
(long)start, (long)end);
exit(U_INVALID_FORMAT_ERROR);
}
if(norm.cc!=0) {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX has a round-trip mapping and ccc!=0, "
"not possible in Unicode normalization\n",
(long)start);
exit(U_INVALID_FORMAT_ERROR);
}
// setRoundTripMapping() ensured that there are exactly two code points.
const UnicodeString &m=*norm.mapping;
UChar32 lead=m.char32At(0);
UChar32 trail=m.char32At(m.length()-1);
if(norms.getCC(lead)!=0) {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
"not possible in Unicode normalization\n",
(long)start, (long)lead);
exit(U_INVALID_FORMAT_ERROR);
}
// Flag for trailing character.
norms.createNorm(trail)->combinesBack=TRUE;
// Insert (trail, composite) pair into compositions list for the lead character.
IcuToolErrorCode errorCode("gennorm2/addComposition()");
Norm *leadNorm=norms.createNorm(lead);
UVector32 *compositions=leadNorm->compositions;
int32_t i;
if(compositions==nullptr) {
compositions=leadNorm->compositions=new UVector32(errorCode);
i=0; // "insert" the first pair at index 0
} else {
// Insertion sort, and check for duplicate trail characters.
int32_t length;
const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
for(i=0; i<length; ++i) {
if(trail==pairs[i].trail) {
fprintf(stderr,
"gennorm2 error: same round-trip mapping for "
"more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
(long)start, (long)lead, (long)trail);
exit(U_INVALID_FORMAT_ERROR);
}
if(trail<pairs[i].trail) {
break;
}
}
}
compositions->insertElementAt(trail, 2*i, errorCode);
compositions->insertElementAt(start, 2*i+1, errorCode);
}
void Decomposer::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
if(!norm.hasMapping()) { return; }
const UnicodeString &m=*norm.mapping;
UnicodeString *decomposed=nullptr;
const UChar *s=toUCharPtr(m.getBuffer());
int32_t length=m.length();
int32_t prev, i=0;
UChar32 c;
while(i<length) {
prev=i;
U16_NEXT(s, i, length, c);
if(start<=c && c<=end) {
fprintf(stderr,
"gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
(long)c);
exit(U_INVALID_FORMAT_ERROR);
}
const Norm &cNorm=norms.getNormRef(c);
if(cNorm.hasMapping()) {
if(norm.mappingType==Norm::ROUND_TRIP) {
if(prev==0) {
if(cNorm.mappingType!=Norm::ROUND_TRIP) {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX's round-trip mapping's starter "
"U+%04lX one-way-decomposes, "
"not possible in Unicode normalization\n",
(long)start, (long)c);
exit(U_INVALID_FORMAT_ERROR);
}
uint8_t myTrailCC=norms.getCC(m.char32At(i));
UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
uint8_t cTrailCC=norms.getCC(cTrailChar);
if(cTrailCC>myTrailCC) {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX's round-trip mapping's starter "
"U+%04lX decomposes and the "
"inner/earlier tccc=%hu > outer/following tccc=%hu, "
"not possible in Unicode normalization\n",
(long)start, (long)c,
(short)cTrailCC, (short)myTrailCC);
exit(U_INVALID_FORMAT_ERROR);
}
} else {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX's round-trip mapping's non-starter "
"U+%04lX decomposes, "
"not possible in Unicode normalization\n",
(long)start, (long)c);
exit(U_INVALID_FORMAT_ERROR);
}
}
if(decomposed==nullptr) {
decomposed=new UnicodeString(m, 0, prev);
}
decomposed->append(*cNorm.mapping);
} else if(Hangul::isHangul(c)) {
UChar buffer[3];
int32_t hangulLength=Hangul::decompose(c, buffer);
if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
fprintf(stderr,
"gennorm2 error: "
"U+%04lX's round-trip mapping's non-starter "
"U+%04lX decomposes, "
"not possible in Unicode normalization\n",
(long)start, (long)c);
exit(U_INVALID_FORMAT_ERROR);
}
if(decomposed==nullptr) {
decomposed=new UnicodeString(m, 0, prev);
}
decomposed->append(buffer, hangulLength);
} else if(decomposed!=nullptr) {
decomposed->append(m, prev, i-prev);
}
}
if(decomposed!=nullptr) {
if(norm.rawMapping==nullptr) {
// Remember the original mapping when decomposing recursively.
norm.rawMapping=norm.mapping;
} else {
delete norm.mapping;
}
norm.mapping=decomposed;
// Not norm.setMappingCP(); because the original mapping
// is most likely to be encodable as a delta.
didDecompose|=TRUE;
}
}
U_NAMESPACE_END
#endif // #if !UCONFIG_NO_NORMALIZATION