/*
*******************************************************************************
*
* Copyright (C) 2003-2013, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
* file name: convtest.cpp
* encoding: US-ASCII
* tab size: 8 (not used)
* indentation:4
*
* created on: 2003jul15
* created by: Markus W. Scherer
*
* Test file for data-driven conversion tests.
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_LEGACY_CONVERSION
/*
* Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
* is slightly unnecessary - it removes tests for Unicode charsets
* like UTF-8 that should work.
* However, there is no easy way for the test to detect whether a test case
* is for a Unicode charset, so it would be difficult to only exclude those.
* Also, regular testing of ICU is done with all modules on, therefore
* not testing conversion for a custom configuration like this should be ok.
*/
#include "unicode/ucnv.h"
#include "unicode/unistr.h"
#include "unicode/parsepos.h"
#include "unicode/uniset.h"
#include "unicode/ustring.h"
#include "unicode/ures.h"
#include "convtest.h"
#include "unicode/tstdtmod.h"
#include <string.h>
#include <stdlib.h>
#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
enum {
// characters used in test data for callbacks
SUB_CB='?',
SKIP_CB='0',
STOP_CB='.',
ESC_CB='&'
};
ConversionTest::ConversionTest() {
UErrorCode errorCode=U_ZERO_ERROR;
utf8Cnv=ucnv_open("UTF-8", &errorCode);
ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
if(U_FAILURE(errorCode)) {
errln("unable to open UTF-8 converter");
}
}
ConversionTest::~ConversionTest() {
ucnv_close(utf8Cnv);
}
void
ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
if (exec) logln("TestSuite ConversionTest: ");
switch (index) {
#if !UCONFIG_NO_FILE_IO
case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
#else
case 0:
case 1:
case 2: name="skip"; break;
#endif
case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
default: name=""; break; //needed to end loop
}
}
// test data interface ----------------------------------------------------- ***
void
ConversionTest::TestToUnicode() {
ConversionCase cc;
char charset[100], cbopt[4];
const char *option;
UnicodeString s, unicode;
int32_t offsetsLength;
UConverterToUCallback callback;
TestDataModule *dataModule;
TestData *testData;
const DataMap *testCase;
UErrorCode errorCode;
int32_t i;
errorCode=U_ZERO_ERROR;
dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
if(U_SUCCESS(errorCode)) {
testData=dataModule->createTestData("toUnicode", errorCode);
if(U_SUCCESS(errorCode)) {
for(i=0; testData->nextCase(testCase, errorCode); ++i) {
if(U_FAILURE(errorCode)) {
errln("error retrieving conversion/toUnicode test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
cc.caseNr=i;
s=testCase->getString("charset", errorCode);
s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
cc.charset=charset;
// BEGIN android-added
// To save space, Android does not build full ISO-2022-CN tables.
// We skip the TestGetKeywordValuesForLocale for counting available collations.
if (strlen(charset) >= 8 &&
strncmp(charset+4, "2022-CN", 4) == 0) {
continue;
}
// END android-added
cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
unicode=testCase->getString("unicode", errorCode);
cc.unicode=unicode.getBuffer();
cc.unicodeLength=unicode.length();
offsetsLength=0;
cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
if(offsetsLength==0) {
cc.offsets=NULL;
} else if(offsetsLength!=unicode.length()) {
errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
i, unicode.length(), offsetsLength);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
s=testCase->getString("errorCode", errorCode);
if(s==UNICODE_STRING("invalid", 7)) {
cc.outErrorCode=U_INVALID_CHAR_FOUND;
} else if(s==UNICODE_STRING("illegal", 7)) {
cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
} else if(s==UNICODE_STRING("truncated", 9)) {
cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
} else if(s==UNICODE_STRING("illesc", 6)) {
cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
} else if(s==UNICODE_STRING("unsuppesc", 9)) {
cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
} else {
cc.outErrorCode=U_ZERO_ERROR;
}
s=testCase->getString("callback", errorCode);
s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
cc.cbopt=cbopt;
switch(cbopt[0]) {
case SUB_CB:
callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
break;
case SKIP_CB:
callback=UCNV_TO_U_CALLBACK_SKIP;
break;
case STOP_CB:
callback=UCNV_TO_U_CALLBACK_STOP;
break;
case ESC_CB:
callback=UCNV_TO_U_CALLBACK_ESCAPE;
break;
default:
callback=NULL;
break;
}
option=callback==NULL ? cbopt : cbopt+1;
if(*option==0) {
option=NULL;
}
cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
if(U_FAILURE(errorCode)) {
errln("error parsing conversion/toUnicode test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
} else {
logln("TestToUnicode[%d] %s", i, charset);
ToUnicodeCase(cc, callback, option);
}
}
delete testData;
}
delete dataModule;
}
else {
dataerrln("Could not load test conversion data");
}
}
void
ConversionTest::TestFromUnicode() {
ConversionCase cc;
char charset[100], cbopt[4];
const char *option;
UnicodeString s, unicode, invalidUChars;
int32_t offsetsLength, index;
UConverterFromUCallback callback;
TestDataModule *dataModule;
TestData *testData;
const DataMap *testCase;
const UChar *p;
UErrorCode errorCode;
int32_t i, length;
errorCode=U_ZERO_ERROR;
dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
if(U_SUCCESS(errorCode)) {
testData=dataModule->createTestData("fromUnicode", errorCode);
if(U_SUCCESS(errorCode)) {
for(i=0; testData->nextCase(testCase, errorCode); ++i) {
if(U_FAILURE(errorCode)) {
errln("error retrieving conversion/fromUnicode test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
cc.caseNr=i;
s=testCase->getString("charset", errorCode);
s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
cc.charset=charset;
// BEGIN android-added
// To save space, Android does not build full ISO-2022-CN tables.
// We skip the TestGetKeywordValuesForLocale for counting available collations.
if (strlen(charset) >= 8 &&
strncmp(charset+4, "2022-CN", 4) == 0) {
continue;
}
// END android-added
unicode=testCase->getString("unicode", errorCode);
cc.unicode=unicode.getBuffer();
cc.unicodeLength=unicode.length();
cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
offsetsLength=0;
cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
if(offsetsLength==0) {
cc.offsets=NULL;
} else if(offsetsLength!=cc.bytesLength) {
errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
i, cc.bytesLength, offsetsLength);
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
}
cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
s=testCase->getString("errorCode", errorCode);
if(s==UNICODE_STRING("invalid", 7)) {
cc.outErrorCode=U_INVALID_CHAR_FOUND;
} else if(s==UNICODE_STRING("illegal", 7)) {
cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
} else if(s==UNICODE_STRING("truncated", 9)) {
cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
} else {
cc.outErrorCode=U_ZERO_ERROR;
}
s=testCase->getString("callback", errorCode);
cc.setSub=0; // default: no subchar
if((index=s.indexOf((UChar)0))>0) {
// read NUL-separated subchar first, if any
// copy the subchar from Latin-1 characters
// start after the NUL
p=s.getTerminatedBuffer();
length=index+1;
p+=length;
length=s.length()-length;
if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
} else {
int32_t j;
for(j=0; j<length; ++j) {
cc.subchar[j]=(char)p[j];
}
// NUL-terminate the subchar
cc.subchar[j]=0;
cc.setSub=1;
}
// remove the NUL and subchar from s
s.truncate(index);
} else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
// read a substitution string, separated by an equal sign
p=s.getBuffer()+index+1;
length=s.length()-(index+1);
if(length<0 || length>=LENGTHOF(cc.subString)) {
errorCode=U_ILLEGAL_ARGUMENT_ERROR;
} else {
u_memcpy(cc.subString, p, length);
// NUL-terminate the subString
cc.subString[length]=0;
cc.setSub=-1;
}
// remove the equal sign and subString from s
s.truncate(index);
}
s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
cc.cbopt=cbopt;
switch(cbopt[0]) {
case SUB_CB:
callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
break;
case SKIP_CB:
callback=UCNV_FROM_U_CALLBACK_SKIP;
break;
case STOP_CB:
callback=UCNV_FROM_U_CALLBACK_STOP;
break;
case ESC_CB:
callback=UCNV_FROM_U_CALLBACK_ESCAPE;
break;
default:
callback=NULL;
break;
}
option=callback==NULL ? cbopt : cbopt+1;
if(*option==0) {
option=NULL;
}
invalidUChars=testCase->getString("invalidUChars", errorCode);
cc.invalidUChars=invalidUChars.getBuffer();
cc.invalidLength=invalidUChars.length();
if(U_FAILURE(errorCode)) {
errln("error parsing conversion/fromUnicode test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
} else {
logln("TestFromUnicode[%d] %s", i, charset);
FromUnicodeCase(cc, callback, option);
}
}
delete testData;
}
delete dataModule;
}
else {
dataerrln("Could not load test conversion data");
}
}
static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
void
ConversionTest::TestGetUnicodeSet() {
char charset[100];
UnicodeString s, map, mapnot;
int32_t which;
ParsePosition pos;
UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
UnicodeSet *cnvSetPtr = &cnvSet;
LocalUConverterPointer cnv;
TestDataModule *dataModule;
TestData *testData;
const DataMap *testCase;
UErrorCode errorCode;
int32_t i;
errorCode=U_ZERO_ERROR;
dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
if(U_SUCCESS(errorCode)) {
testData=dataModule->createTestData("getUnicodeSet", errorCode);
if(U_SUCCESS(errorCode)) {
for(i=0; testData->nextCase(testCase, errorCode); ++i) {
if(U_FAILURE(errorCode)) {
errln("error retrieving conversion/getUnicodeSet test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
s=testCase->getString("charset", errorCode);
s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
// BEGIN android-added
// To save space, Android does not build full ISO-2022-CN tables.
// We skip the TestGetKeywordValuesForLocale for counting available collations.
if (strlen(charset) >= 8 &&
strncmp(charset+4, "2022-CN", 4) == 0) {
continue;
}
// END android-added
map=testCase->getString("map", errorCode);
mapnot=testCase->getString("mapnot", errorCode);
which=testCase->getInt28("which", errorCode);
if(U_FAILURE(errorCode)) {
errln("error parsing conversion/getUnicodeSet test case %d - %s",
i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
// test this test case
mapSet.clear();
mapnotSet.clear();
pos.setIndex(0);
mapSet.applyPattern(map, pos, 0, NULL, errorCode);
if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
" error index %d index %d U+%04x",
i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
errorCode=U_ZERO_ERROR;
continue;
}
pos.setIndex(0);
mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
" error index %d index %d U+%04x",
i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
errorCode=U_ZERO_ERROR;
continue;
}
logln("TestGetUnicodeSet[%d] %s", i, charset);
cnv.adoptInstead(cnv_open(charset, errorCode));
if(U_FAILURE(errorCode)) {
errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
charset, i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
if(U_FAILURE(errorCode)) {
errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
charset, i, u_errorName(errorCode));
errorCode=U_ZERO_ERROR;
continue;
}
// are there items that must be in cnvSet but are not?
(diffSet=mapSet).removeAll(cnvSet);
if(!diffSet.isEmpty()) {
diffSet.toPattern(s, TRUE);
if(s.length()>100) {
s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
}
errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
charset, i);
errln(s);
}
// are there items that must not be in cnvSet but are?
(diffSet=mapnotSet).retainAll(cnvSet);
if(!diffSet.isEmpty()) {
diffSet.toPattern(s, TRUE);
if(s.length()>100) {
s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
}
errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
charset, i);
errln(s);
}
}
delete testData;
}
delete dataModule;
}
else {
dataerrln("Could not load test conversion data");
}
}
U_CDECL_BEGIN
static void U_CALLCONV
getUnicodeSetCallback(const void *context,
UConverterFromUnicodeArgs * /*fromUArgs*/,
const UChar* /*codeUnits*/,
int32_t /*length*/,
UChar32 codePoint,
UConverterCallbackReason reason,
UErrorCode *pErrorCode) {
if(reason<=UCNV_IRREGULAR) {
((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point
*pErrorCode=U_ZERO_ERROR; // skip
} // else ignore the reset, close and clone calls.
}
U_CDECL_END
// Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
void
ConversionTest::TestGetUnicodeSet2() {
// Build a string with all code points.
UChar32 cpLimit;
int32_t s0Length;
if(quick) {
cpLimit=s0Length=0x10000; // BMP only
} else {
cpLimit=0x110000;
s0Length=0x10000+0x200000; // BMP + surrogate pairs
}
UChar *s0=new UChar[s0Length];
if(s0==NULL) {
return;
}
UChar *s=s0;
UChar32 c;
UChar c2;
// low BMP
for(c=0; c<=0xd7ff; ++c) {
*s++=(UChar)c;
}
// trail surrogates
for(c=0xdc00; c<=0xdfff; ++c) {
*s++=(UChar)c;
}
// lead surrogates
// (after trails so that there is not even one surrogate pair in between)
for(c=0xd800; c<=0xdbff; ++c) {
*s++=(UChar)c;
}
// high BMP
for(c=0xe000; c<=0xffff; ++c) {
*s++=(UChar)c;
}
// supplementary code points = surrogate pairs
if(cpLimit==0x110000) {
for(c=0xd800; c<=0xdbff; ++c) {
for(c2=0xdc00; c2<=0xdfff; ++c2) {
*s++=(UChar)c;
*s++=c2;
}
}
}
static const char *const cnvNames[]={
"UTF-8",
"UTF-7",
"UTF-16",
"US-ASCII",
"ISO-8859-1",
"windows-1252",
"Shift-JIS",
"ibm-1390", // EBCDIC_STATEFUL table
"ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table
"HZ",
"ISO-2022-JP",
"JIS7",
"ISO-2022-CN",
"ISO-2022-CN-EXT",
"LMBCS"
};
LocalUConverterPointer cnv;
char buffer[1024];
int32_t i;
for(i=0; i<LENGTHOF(cnvNames); ++i) {
UErrorCode errorCode=U_ZERO_ERROR;
cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
if(U_FAILURE(errorCode)) {
errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
continue;
}
UnicodeSet expected;
ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
if(U_FAILURE(errorCode)) {
errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
continue;
}
UConverterUnicodeSet which;
for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
ucnv_setFallback(cnv.getAlias(), TRUE);
}
expected.add(0, cpLimit-1);
s=s0;
UBool flush;
do {
char *t=buffer;
flush=(UBool)(s==s0+s0Length);
ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
if(U_FAILURE(errorCode)) {
if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
errorCode=U_ZERO_ERROR;
continue;
} else {
break; // unexpected error, should not occur
}
}
} while(!flush);
UnicodeSet set;
ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
if(cpLimit<0x110000) {
set.remove(cpLimit, 0x10ffff);
}
if(which==UCNV_ROUNDTRIP_SET) {
// ignore PUA code points because they will be converted even if they
// are fallbacks and when other fallbacks are turned off,
// but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
expected.remove(0xe000, 0xf8ff);
expected.remove(0xf0000, 0xffffd);
expected.remove(0x100000, 0x10fffd);
set.remove(0xe000, 0xf8ff);
set.remove(0xf0000, 0xffffd);
set.remove(0x100000, 0x10fffd);
}
if(set!=expected) {
// First try to see if we have different sets because ucnv_getUnicodeSet()
// added strings: The above conversion method does not tell us what strings might be convertible.
// Remove strings from the set and compare again.
// Unfortunately, there are no good, direct set methods for finding out whether there are strings
// in the set, nor for enumerating or removing just them.
// Intersect all code points with the set. The intersection will not contain strings.
UnicodeSet temp(0, 0x10ffff);
temp.retainAll(set);
set=temp;
}
if(set!=expected) {
UnicodeSet diffSet;
UnicodeString out;
// are there items that must be in the set but are not?
(diffSet=expected).removeAll(set);
if(!diffSet.isEmpty()) {
diffSet.toPattern(out, TRUE);
if(out.length()>100) {
out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
}
errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
cnvNames[i], which);
errln(out);
}
// are there items that must not be in the set but are?
(diffSet=set).removeAll(expected);
if(!diffSet.isEmpty()) {
diffSet.toPattern(out, TRUE);
if(out.length()>100) {
out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
}
errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
cnvNames[i], which);
errln(out);
}
}
}
}
delete [] s0;
}
// open testdata or ICU data converter ------------------------------------- ***
UConverter *
ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
if(name!=NULL && *name=='+') {
// Converter names that start with '+' are ignored in ICU4J tests.
++name;
}
if(name!=NULL && *name=='*') {
/* loadTestData(): set the data directory */
return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
} else {
return ucnv_open(name, &errorCode);
}
}
// output helpers ---------------------------------------------------------- ***
static inline char
hexDigit(uint8_t digit) {
return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
}
static char *
printBytes(const uint8_t *bytes, int32_t length, char *out) {
uint8_t b;
if(length>0) {
b=*bytes++;
--length;
*out++=hexDigit((uint8_t)(b>>4));
*out++=hexDigit((uint8_t)(b&0xf));
}
while(length>0) {
b=*bytes++;
--length;
*out++=' ';
*out++=hexDigit((uint8_t)(b>>4));
*out++=hexDigit((uint8_t)(b&0xf));
}
*out++=0;
return out;
}
static char *
printUnicode(const UChar *unicode, int32_t length, char *out) {
UChar32 c;
int32_t i;
for(i=0; i<length;) {
if(i>0) {
*out++=' ';
}
U16_NEXT(unicode, i, length, c);
// write 4..6 digits
if(c>=0x100000) {
*out++='1';
}
if(c>=0x10000) {
*out++=hexDigit((uint8_t)((c>>16)&0xf));
}
*out++=hexDigit((uint8_t)((c>>12)&0xf));
*out++=hexDigit((uint8_t)((c>>8)&0xf));
*out++=hexDigit((uint8_t)((c>>4)&0xf));
*out++=hexDigit((uint8_t)(c&0xf));
}
*out++=0;
return out;
}
static char *
printOffsets(const int32_t *offsets, int32_t length, char *out) {
int32_t i, o, d;
if(offsets==NULL) {
length=0;
}
for(i=0; i<length; ++i) {
if(i>0) {
*out++=' ';
}
o=offsets[i];
// print all offsets with 2 characters each (-x, -9..99, xx)
if(o<-9) {
*out++='-';
*out++='x';
} else if(o<0) {
*out++='-';
*out++=(char)('0'-o);
} else if(o<=99) {
*out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
*out++=(char)('0'+o%10);
} else /* o>99 */ {
*out++='x';
*out++='x';
}
}
*out++=0;
return out;
}
// toUnicode test worker functions ----------------------------------------- ***
static int32_t
stepToUnicode(ConversionCase &cc, UConverter *cnv,
UChar *result, int32_t resultCapacity,
int32_t *resultOffsets, /* also resultCapacity */
int32_t step,
UErrorCode *pErrorCode) {
const char *source, *sourceLimit, *bytesLimit;
UChar *target, *targetLimit, *resultLimit;
UBool flush;
source=(const char *)cc.bytes;
target=result;
bytesLimit=source+cc.bytesLength;
resultLimit=result+resultCapacity;
if(step>=0) {
// call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
// move only one buffer (in vs. out) at a time to be extra mean
// step==0 performs bulk conversion and generates offsets
// initialize the partial limits for the loop
if(step==0) {
// use the entire buffers
sourceLimit=bytesLimit;
targetLimit=resultLimit;
flush=cc.finalFlush;
} else {
// start with empty partial buffers
sourceLimit=source;
targetLimit=target;
flush=FALSE;
// output offsets only for bulk conversion
resultOffsets=NULL;
}
for(;;) {
// resetting the opposite conversion direction must not affect this one
ucnv_resetFromUnicode(cnv);
// convert
ucnv_toUnicode(cnv,
&target, targetLimit,
&source, sourceLimit,
resultOffsets,
flush, pErrorCode);
// check pointers and errors
if(source>sourceLimit || target>targetLimit) {
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
if(target!=targetLimit) {
// buffer overflow must only be set when the target is filled
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(targetLimit==resultLimit) {
// not just a partial overflow
break;
}
// the partial target is filled, set a new limit, reset the error and continue
targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
*pErrorCode=U_ZERO_ERROR;
} else if(U_FAILURE(*pErrorCode)) {
// some other error occurred, done
break;
} else {
if(source!=sourceLimit) {
// when no error occurs, then the input must be consumed
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
}
if(sourceLimit==bytesLimit) {
// we are done
break;
}
// the partial conversion succeeded, set a new limit and continue
sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
}
}
} else /* step<0 */ {
/*
* step==-1: call only ucnv_getNextUChar()
* otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
* if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
* else give it at most (-step-2)/2 bytes
*/
UChar32 c;
// end the loop by getting an index out of bounds error
for(;;) {
// resetting the opposite conversion direction must not affect this one
ucnv_resetFromUnicode(cnv);
// convert
if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
sourceLimit=source; // use sourceLimit not as a real limit
// but to remember the pre-getNextUChar source pointer
c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
// check pointers and errors
if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
if(source!=bytesLimit) {
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
} else {
*pErrorCode=U_ZERO_ERROR;
}
break;
} else if(U_FAILURE(*pErrorCode)) {
break;
}
// source may not move if c is from previous overflow
if(target==resultLimit) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
if(c<=0xffff) {
*target++=(UChar)c;
} else {
*target++=U16_LEAD(c);
if(target==resultLimit) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
break;
}
*target++=U16_TRAIL(c);
}
// alternate between -n-1 and -n but leave -1 alone
if(step<-1) {
++step;
}
} else /* step is even */ {
// allow only one UChar output
targetLimit=target<resultLimit ? target+1 : resultLimit;
// as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
// and never output offsets
if(step==-2) {
sourceLimit=bytesLimit;
} else {
sourceLimit=source+(-step-2)/2;
if(sourceLimit>bytesLimit) {
sourceLimit=bytesLimit;
}
}
ucnv_toUnicode(cnv,
&target, targetLimit,
&source, sourceLimit,
NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
// check pointers and errors
if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
if(target!=targetLimit) {
// buffer overflow must only be set when the target is filled
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(targetLimit==resultLimit) {
// not just a partial overflow
break;
}
// the partial target is filled, set a new limit and continue
*pErrorCode=U_ZERO_ERROR;
} else if(U_FAILURE(*pErrorCode)) {
// some other error occurred, done
break;
} else {
if(source!=sourceLimit) {
// when no error occurs, then the input must be consumed
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
}
// we are done (flush==TRUE) but we continue, to get the index out of bounds error above
}
--step;
}
}
}
return (int32_t)(target-result);
}
UBool
ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
// open the converter
IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
if(errorCode.isFailure()) {
errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
errorCode.reset();
return FALSE;
}
// set the callback
if(callback!=NULL) {
ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
if(U_FAILURE(errorCode)) {
errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
return FALSE;
}
}
int32_t resultOffsets[256];
UChar result[256];
int32_t resultLength;
UBool ok;
static const struct {
int32_t step;
const char *name;
} steps[]={
{ 0, "bulk" }, // must be first for offsets to be checked
{ 1, "step=1" },
{ 3, "step=3" },
{ 7, "step=7" },
{ -1, "getNext" },
{ -2, "toU(bulk)+getNext" },
{ -3, "getNext+toU(bulk)" },
{ -4, "toU(1)+getNext" },
{ -5, "getNext+toU(1)" },
{ -12, "toU(5)+getNext" },
{ -13, "getNext+toU(5)" },
};
int32_t i, step;
ok=TRUE;
for(i=0; i<LENGTHOF(steps) && ok; ++i) {
step=steps[i].step;
if(step<0 && !cc.finalFlush) {
// skip ucnv_getNextUChar() if !finalFlush because
// ucnv_getNextUChar() always implies flush
continue;
}
if(step!=0) {
// bulk test is first, then offsets are not checked any more
cc.offsets=NULL;
}
else {
memset(resultOffsets, -1, LENGTHOF(resultOffsets));
}
memset(result, -1, LENGTHOF(result));
errorCode.reset();
resultLength=stepToUnicode(cc, cnv.getAlias(),
result, LENGTHOF(result),
step==0 ? resultOffsets : NULL,
step, errorCode);
ok=checkToUnicode(
cc, cnv.getAlias(), steps[i].name,
result, resultLength,
cc.offsets!=NULL ? resultOffsets : NULL,
errorCode);
if(errorCode.isFailure() || !cc.finalFlush) {
// reset if an error occurred or we did not flush
// otherwise do nothing to make sure that flushing resets
ucnv_resetToUnicode(cnv.getAlias());
}
if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
cc.caseNr, cc.charset, resultLength);
}
if (result[resultLength] != (UChar)-1) {
errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
cc.caseNr, cc.charset, resultLength);
}
}
// not a real loop, just a convenience for breaking out of the block
while(ok && cc.finalFlush) {
// test ucnv_toUChars()
memset(result, 0, sizeof(result));
errorCode.reset();
resultLength=ucnv_toUChars(cnv.getAlias(),
result, LENGTHOF(result),
(const char *)cc.bytes, cc.bytesLength,
errorCode);
ok=checkToUnicode(
cc, cnv.getAlias(), "toUChars",
result, resultLength,
NULL,
errorCode);
if(!ok) {
break;
}
// test preflighting
// keep the correct result for simple checking
errorCode.reset();
resultLength=ucnv_toUChars(cnv.getAlias(),
NULL, 0,
(const char *)cc.bytes, cc.bytesLength,
errorCode);
if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
errorCode.reset();
}
ok=checkToUnicode(
cc, cnv.getAlias(), "preflight toUChars",
result, resultLength,
NULL,
errorCode);
break;
}
errorCode.reset(); // all errors have already been reported
return ok;
}
UBool
ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
const UChar *result, int32_t resultLength,
const int32_t *resultOffsets,
UErrorCode resultErrorCode) {
char resultInvalidChars[8];
int8_t resultInvalidLength;
UErrorCode errorCode;
const char *msg;
// reset the message; NULL will mean "ok"
msg=NULL;
errorCode=U_ZERO_ERROR;
resultInvalidLength=sizeof(resultInvalidChars);
ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
if(U_FAILURE(errorCode)) {
errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
return FALSE;
}
// check everything that might have gone wrong
if(cc.unicodeLength!=resultLength) {
msg="wrong result length";
} else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
msg="wrong result string";
} else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
msg="wrong offsets";
} else if(cc.outErrorCode!=resultErrorCode) {
msg="wrong error code";
} else if(cc.invalidLength!=resultInvalidLength) {
msg="wrong length of last invalid input";
} else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
msg="wrong last invalid input";
}
if(msg==NULL) {
return TRUE;
} else {
char buffer[2000]; // one buffer for all strings
char *s, *bytesString, *unicodeString, *resultString,
*offsetsString, *resultOffsetsString,
*invalidCharsString, *resultInvalidCharsString;
bytesString=s=buffer;
s=printBytes(cc.bytes, cc.bytesLength, bytesString);
s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
s=printUnicode(result, resultLength, resultString=s);
s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
if((s-buffer)>(int32_t)sizeof(buffer)) {
errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
exit(1);
}
errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
" bytes <%s>[%d]\n"
" expected <%s>[%d]\n"
" result <%s>[%d]\n"
" offsets <%s>\n"
" result offsets <%s>\n"
" error code expected %s got %s\n"
" invalidChars expected <%s> got <%s>\n",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
bytesString, cc.bytesLength,
unicodeString, cc.unicodeLength,
resultString, resultLength,
offsetsString,
resultOffsetsString,
u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
invalidCharsString, resultInvalidCharsString);
return FALSE;
}
}
// fromUnicode test worker functions --------------------------------------- ***
static int32_t
stepFromUTF8(ConversionCase &cc,
UConverter *utf8Cnv, UConverter *cnv,
char *result, int32_t resultCapacity,
int32_t step,
UErrorCode *pErrorCode) {
const char *source, *sourceLimit, *utf8Limit;
UChar pivotBuffer[32];
UChar *pivotSource, *pivotTarget, *pivotLimit;
char *target, *targetLimit, *resultLimit;
UBool flush;
source=cc.utf8;
pivotSource=pivotTarget=pivotBuffer;
target=result;
utf8Limit=source+cc.utf8Length;
resultLimit=result+resultCapacity;
// call ucnv_convertEx() with in/out buffers no larger than (step) at a time
// move only one buffer (in vs. out) at a time to be extra mean
// step==0 performs bulk conversion
// initialize the partial limits for the loop
if(step==0) {
// use the entire buffers
sourceLimit=utf8Limit;
targetLimit=resultLimit;
flush=cc.finalFlush;
pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
} else {
// start with empty partial buffers
sourceLimit=source;
targetLimit=target;
flush=FALSE;
// empty pivot is not allowed, make it of length step
pivotLimit=pivotBuffer+step;
}
for(;;) {
// resetting the opposite conversion direction must not affect this one
ucnv_resetFromUnicode(utf8Cnv);
ucnv_resetToUnicode(cnv);
// convert
ucnv_convertEx(cnv, utf8Cnv,
&target, targetLimit,
&source, sourceLimit,
pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
FALSE, flush, pErrorCode);
// check pointers and errors
if(source>sourceLimit || target>targetLimit) {
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
if(target!=targetLimit) {
// buffer overflow must only be set when the target is filled
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(targetLimit==resultLimit) {
// not just a partial overflow
break;
}
// the partial target is filled, set a new limit, reset the error and continue
targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
*pErrorCode=U_ZERO_ERROR;
} else if(U_FAILURE(*pErrorCode)) {
if(pivotSource==pivotBuffer) {
// toUnicode error, should not occur
// toUnicode errors are tested in cintltst TestConvertExFromUTF8()
break;
} else {
// fromUnicode error
// some other error occurred, done
break;
}
} else {
if(source!=sourceLimit) {
// when no error occurs, then the input must be consumed
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
}
if(sourceLimit==utf8Limit) {
// we are done
if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
// ucnv_convertEx() warns about not terminating the output
// but ucnv_fromUnicode() does not and so
// checkFromUnicode() does not expect it
*pErrorCode=U_ZERO_ERROR;
}
break;
}
// the partial conversion succeeded, set a new limit and continue
sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
}
}
return (int32_t)(target-result);
}
static int32_t
stepFromUnicode(ConversionCase &cc, UConverter *cnv,
char *result, int32_t resultCapacity,
int32_t *resultOffsets, /* also resultCapacity */
int32_t step,
UErrorCode *pErrorCode) {
const UChar *source, *sourceLimit, *unicodeLimit;
char *target, *targetLimit, *resultLimit;
UBool flush;
source=cc.unicode;
target=result;
unicodeLimit=source+cc.unicodeLength;
resultLimit=result+resultCapacity;
// call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
// move only one buffer (in vs. out) at a time to be extra mean
// step==0 performs bulk conversion and generates offsets
// initialize the partial limits for the loop
if(step==0) {
// use the entire buffers
sourceLimit=unicodeLimit;
targetLimit=resultLimit;
flush=cc.finalFlush;
} else {
// start with empty partial buffers
sourceLimit=source;
targetLimit=target;
flush=FALSE;
// output offsets only for bulk conversion
resultOffsets=NULL;
}
for(;;) {
// resetting the opposite conversion direction must not affect this one
ucnv_resetToUnicode(cnv);
// convert
ucnv_fromUnicode(cnv,
&target, targetLimit,
&source, sourceLimit,
resultOffsets,
flush, pErrorCode);
// check pointers and errors
if(source>sourceLimit || target>targetLimit) {
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
if(target!=targetLimit) {
// buffer overflow must only be set when the target is filled
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
} else if(targetLimit==resultLimit) {
// not just a partial overflow
break;
}
// the partial target is filled, set a new limit, reset the error and continue
targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
*pErrorCode=U_ZERO_ERROR;
} else if(U_FAILURE(*pErrorCode)) {
// some other error occurred, done
break;
} else {
if(source!=sourceLimit) {
// when no error occurs, then the input must be consumed
*pErrorCode=U_INTERNAL_PROGRAM_ERROR;
break;
}
if(sourceLimit==unicodeLimit) {
// we are done
break;
}
// the partial conversion succeeded, set a new limit and continue
sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
}
}
return (int32_t)(target-result);
}
UBool
ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
UConverter *cnv;
UErrorCode errorCode;
// open the converter
errorCode=U_ZERO_ERROR;
cnv=cnv_open(cc.charset, errorCode);
if(U_FAILURE(errorCode)) {
errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
return FALSE;
}
ucnv_resetToUnicode(utf8Cnv);
// set the callback
if(callback!=NULL) {
ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
if(U_FAILURE(errorCode)) {
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
ucnv_close(cnv);
return FALSE;
}
}
// set the fallbacks flag
// TODO change with Jitterbug 2401, then add a similar call for toUnicode too
ucnv_setFallback(cnv, cc.fallbacks);
// set the subchar
int32_t length;
if(cc.setSub>0) {
length=(int32_t)strlen(cc.subchar);
ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
if(U_FAILURE(errorCode)) {
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
ucnv_close(cnv);
return FALSE;
}
} else if(cc.setSub<0) {
ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
if(U_FAILURE(errorCode)) {
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
ucnv_close(cnv);
return FALSE;
}
}
// convert unicode to utf8
char utf8[256];
cc.utf8=utf8;
u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
cc.unicode, cc.unicodeLength,
&errorCode);
if(U_FAILURE(errorCode)) {
// skip UTF-8 testing of a string with an unpaired surrogate,
// or of one that's too long
// toUnicode errors are tested in cintltst TestConvertExFromUTF8()
cc.utf8Length=-1;
}
int32_t resultOffsets[256];
char result[256];
int32_t resultLength;
UBool ok;
static const struct {
int32_t step;
const char *name, *utf8Name;
} steps[]={
{ 0, "bulk", "utf8" }, // must be first for offsets to be checked
{ 1, "step=1", "utf8 step=1" },
{ 3, "step=3", "utf8 step=3" },
{ 7, "step=7", "utf8 step=7" }
};
int32_t i, step;
ok=TRUE;
for(i=0; i<LENGTHOF(steps) && ok; ++i) {
step=steps[i].step;
memset(resultOffsets, -1, LENGTHOF(resultOffsets));
memset(result, -1, LENGTHOF(result));
errorCode=U_ZERO_ERROR;
resultLength=stepFromUnicode(cc, cnv,
result, LENGTHOF(result),
step==0 ? resultOffsets : NULL,
step, &errorCode);
ok=checkFromUnicode(
cc, cnv, steps[i].name,
(uint8_t *)result, resultLength,
cc.offsets!=NULL ? resultOffsets : NULL,
errorCode);
if(U_FAILURE(errorCode) || !cc.finalFlush) {
// reset if an error occurred or we did not flush
// otherwise do nothing to make sure that flushing resets
ucnv_resetFromUnicode(cnv);
}
if (resultOffsets[resultLength] != -1) {
errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
cc.caseNr, cc.charset, resultLength);
}
if (result[resultLength] != (char)-1) {
errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
cc.caseNr, cc.charset, resultLength);
}
// bulk test is first, then offsets are not checked any more
cc.offsets=NULL;
// test direct conversion from UTF-8
if(cc.utf8Length>=0) {
errorCode=U_ZERO_ERROR;
resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
result, LENGTHOF(result),
step, &errorCode);
ok=checkFromUnicode(
cc, cnv, steps[i].utf8Name,
(uint8_t *)result, resultLength,
NULL,
errorCode);
if(U_FAILURE(errorCode) || !cc.finalFlush) {
// reset if an error occurred or we did not flush
// otherwise do nothing to make sure that flushing resets
ucnv_resetToUnicode(utf8Cnv);
ucnv_resetFromUnicode(cnv);
}
}
}
// not a real loop, just a convenience for breaking out of the block
while(ok && cc.finalFlush) {
// test ucnv_fromUChars()
memset(result, 0, sizeof(result));
errorCode=U_ZERO_ERROR;
resultLength=ucnv_fromUChars(cnv,
result, LENGTHOF(result),
cc.unicode, cc.unicodeLength,
&errorCode);
ok=checkFromUnicode(
cc, cnv, "fromUChars",
(uint8_t *)result, resultLength,
NULL,
errorCode);
if(!ok) {
break;
}
// test preflighting
// keep the correct result for simple checking
errorCode=U_ZERO_ERROR;
resultLength=ucnv_fromUChars(cnv,
NULL, 0,
cc.unicode, cc.unicodeLength,
&errorCode);
if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
errorCode=U_ZERO_ERROR;
}
ok=checkFromUnicode(
cc, cnv, "preflight fromUChars",
(uint8_t *)result, resultLength,
NULL,
errorCode);
break;
}
ucnv_close(cnv);
return ok;
}
UBool
ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
const uint8_t *result, int32_t resultLength,
const int32_t *resultOffsets,
UErrorCode resultErrorCode) {
UChar resultInvalidUChars[8];
int8_t resultInvalidLength;
UErrorCode errorCode;
const char *msg;
// reset the message; NULL will mean "ok"
msg=NULL;
errorCode=U_ZERO_ERROR;
resultInvalidLength=LENGTHOF(resultInvalidUChars);
ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
if(U_FAILURE(errorCode)) {
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
return FALSE;
}
// check everything that might have gone wrong
if(cc.bytesLength!=resultLength) {
msg="wrong result length";
} else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
msg="wrong result string";
} else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
msg="wrong offsets";
} else if(cc.outErrorCode!=resultErrorCode) {
msg="wrong error code";
} else if(cc.invalidLength!=resultInvalidLength) {
msg="wrong length of last invalid input";
} else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
msg="wrong last invalid input";
}
if(msg==NULL) {
return TRUE;
} else {
char buffer[2000]; // one buffer for all strings
char *s, *unicodeString, *bytesString, *resultString,
*offsetsString, *resultOffsetsString,
*invalidCharsString, *resultInvalidUCharsString;
unicodeString=s=buffer;
s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
s=printBytes(result, resultLength, resultString=s);
s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
if((s-buffer)>(int32_t)sizeof(buffer)) {
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
exit(1);
}
errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
" unicode <%s>[%d]\n"
" expected <%s>[%d]\n"
" result <%s>[%d]\n"
" offsets <%s>\n"
" result offsets <%s>\n"
" error code expected %s got %s\n"
" invalidChars expected <%s> got <%s>\n",
cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
unicodeString, cc.unicodeLength,
bytesString, cc.bytesLength,
resultString, resultLength,
offsetsString,
resultOffsetsString,
u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
invalidCharsString, resultInvalidUCharsString);
return FALSE;
}
}
#endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */