// © 2016 and later: Unicode, Inc. and others. // License & terms of use: http://www.unicode.org/copyright.html /* ****************************************************************************** * Copyright (C) 1998-2003, 2006, International Business Machines Corporation * * and others. All Rights Reserved. * ****************************************************************************** */ #include <errno.h> #include <stdio.h> #include <string.h> #include "unicode/utypes.h" #include "unicode/uchar.h" #include "unicode/uchriter.h" #include "unicode/brkiter.h" #include "unicode/locid.h" #include "unicode/unistr.h" #include "unicode/uniset.h" #include "unicode/ustring.h" /* * This program takes a Unicode text file containing Thai text with * spaces inserted where the word breaks are. It computes a copy of * the text without spaces and uses a word instance of a Thai BreakIterator * to compute the word breaks. The program reports any differences in the * breaks. * * NOTE: by it's very nature, Thai word breaking is not exact, so it is * exptected that this program will always report some differences. */ /* * This class is a break iterator that counts words and spaces. */ class SpaceBreakIterator { public: // The constructor: // text - pointer to an array of UChars to iterate over // count - the number of UChars in text SpaceBreakIterator(const UChar *text, int32_t count); // the destructor ~SpaceBreakIterator(); // return next break position int32_t next(); // return current word count int32_t getWordCount(); // return current space count int32_t getSpaceCount(); private: // No arg constructor: private so clients can't call it. SpaceBreakIterator(); // The underlying BreakIterator BreakIterator *fBreakIter; // address of the UChar array const UChar *fText; // number of UChars in fText int32_t fTextCount; // current word count int32_t fWordCount; // current space count int32_t fSpaceCount; // UnicodeSet of SA characters UnicodeSet fComplexContext; // true when fBreakIter has returned DONE UBool fDone; }; /* * This is the main class. It compares word breaks and reports the differences. */ class ThaiWordbreakTest { public: // The main constructor: // spaces - pointer to a UChar array for the text with spaces // spaceCount - the number of characters in the spaces array // noSpaces - pointer to a UChar array for the text without spaces // noSpaceCount - the number of characters in the noSpaces array // verbose - report all breaks if true, otherwise just report differences ThaiWordbreakTest(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount, UBool verbose); ~ThaiWordbreakTest(); // returns the number of breaks that are in the spaces array // but aren't found in the noSpaces array int32_t getBreaksNotFound(); // returns the number of breaks which are found in the noSpaces // array but aren't in the spaces array int32_t getInvalidBreaks(); // returns the number of words found in the spaces array int32_t getWordCount(); // reads the input Unicode text file: // fileName - the path name of the file // charCount - set to the number of UChars read from the file // returns - the address of the UChar array containing the characters static const UChar *readFile(char *fileName, int32_t &charCount); // removes spaces form the input UChar array: // spaces - pointer to the input UChar array // count - number of UChars in the spaces array // nonSpaceCount - the number of UChars in the result array // returns - the address of the UChar array with spaces removed static const UChar *crunchSpaces(const UChar *spaces, int32_t count, int32_t &nonSpaceCount); private: // The no arg constructor - private so clients can't call it ThaiWordbreakTest(); // This does the actual comparison: // spaces - the address of the UChar array for the text with spaces // spaceCount - the number of UChars in the spaces array // noSpaces - the address of the UChar array for the text without spaces // noSpaceCount - the number of UChars in the noSpaces array // returns - true if all breaks match, FALSE otherwise UBool compareWordBreaks(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount); // helper method to report a break in the spaces // array that's not found in the noSpaces array void breakNotFound(int32_t br); // helper method to report a break that's found in // the noSpaces array that's not in the spaces array void foundInvalidBreak(int32_t br); // count of breaks in the spaces array that // aren't found in the noSpaces array int32_t fBreaksNotFound; // count of breaks found in the noSpaces array // that aren't in the spaces array int32_t fInvalidBreaks; // number of words found in the spaces array int32_t fWordCount; // report all breaks if true, otherwise just report differences UBool fVerbose; }; /* * The main constructor: it calls compareWordBreaks and reports any differences */ ThaiWordbreakTest::ThaiWordbreakTest(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount, UBool verbose) : fBreaksNotFound(0), fInvalidBreaks(0), fWordCount(0), fVerbose(verbose) { compareWordBreaks(spaces, spaceCount, noSpaces, noSpaceCount); } /* * The no arg constructor */ ThaiWordbreakTest::ThaiWordbreakTest() { // nothing } /* * The destructor */ ThaiWordbreakTest::~ThaiWordbreakTest() { // nothing? } /* * returns the number of breaks in the spaces array * that aren't found in the noSpaces array */ inline int32_t ThaiWordbreakTest::getBreaksNotFound() { return fBreaksNotFound; } /* * Returns the number of breaks found in the noSpaces * array that aren't in the spaces array */ inline int32_t ThaiWordbreakTest::getInvalidBreaks() { return fInvalidBreaks; } /* * Returns the number of words found in the spaces array */ inline int32_t ThaiWordbreakTest::getWordCount() { return fWordCount; } /* * This method does the acutal break comparison and reports the results. * It uses a SpaceBreakIterator to iterate over the text with spaces, * and a word instance of a Thai BreakIterator to iterate over the text * without spaces. */ UBool ThaiWordbreakTest::compareWordBreaks(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount) { UBool result = TRUE; Locale thai("th"); UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(noSpaces, noSpaceCount); UErrorCode status = U_ZERO_ERROR; BreakIterator *breakIter = BreakIterator::createWordInstance(thai, status); breakIter->adoptText(noSpaceIter); SpaceBreakIterator spaceIter(spaces, spaceCount); int32_t nextBreak = 0; int32_t nextSpaceBreak = 0; int32_t iterCount = 0; while (TRUE) { nextSpaceBreak = spaceIter.next(); nextBreak = breakIter->next(); if (nextSpaceBreak == BreakIterator::DONE || nextBreak == BreakIterator::DONE) { if (nextBreak != BreakIterator::DONE) { fprintf(stderr, "break iterator didn't end.\n"); } else if (nextSpaceBreak != BreakIterator::DONE) { fprintf(stderr, "premature break iterator end.\n"); } break; } while (nextSpaceBreak != nextBreak && nextSpaceBreak != BreakIterator::DONE && nextBreak != BreakIterator::DONE) { if (nextSpaceBreak < nextBreak) { breakNotFound(nextSpaceBreak); result = FALSE; nextSpaceBreak = spaceIter.next(); } else if (nextSpaceBreak > nextBreak) { foundInvalidBreak(nextBreak); result = FALSE; nextBreak = breakIter->next(); } } if (fVerbose) { printf("%d %d\n", nextSpaceBreak, nextBreak); } } fWordCount = spaceIter.getWordCount(); delete breakIter; return result; } /* * Report a break that's in the text with spaces but * not found in the text without spaces. */ void ThaiWordbreakTest::breakNotFound(int32_t br) { if (fVerbose) { printf("%d ****\n", br); } else { fprintf(stderr, "break not found: %d\n", br); } fBreaksNotFound += 1; } /* * Report a break that's found in the text without spaces * that isn't in the text with spaces. */ void ThaiWordbreakTest::foundInvalidBreak(int32_t br) { if (fVerbose) { printf("**** %d\n", br); } else { fprintf(stderr, "found invalid break: %d\n", br); } fInvalidBreaks += 1; } /* * Read the text from a file. The text must start with a Unicode Byte * Order Mark (BOM) so that we know what order to read the bytes in. */ const UChar *ThaiWordbreakTest::readFile(char *fileName, int32_t &charCount) { FILE *f; int32_t fileSize; UChar *buffer; char *bufferChars; f = fopen(fileName, "rb"); if( f == NULL ) { fprintf(stderr,"Couldn't open %s reason: %s \n", fileName, strerror(errno)); return 0; } fseek(f, 0, SEEK_END); fileSize = ftell(f); fseek(f, 0, SEEK_SET); bufferChars = new char[fileSize]; if(bufferChars == 0) { fprintf(stderr,"Couldn't get memory for reading %s reason: %s \n", fileName, strerror(errno)); fclose(f); return 0; } fread(bufferChars, sizeof(char), fileSize, f); if( ferror(f) ) { fprintf(stderr,"Couldn't read %s reason: %s \n", fileName, strerror(errno)); fclose(f); delete[] bufferChars; return 0; } fclose(f); UnicodeString myText(bufferChars, fileSize, "UTF-8"); delete[] bufferChars; charCount = myText.length(); buffer = new UChar[charCount]; if(buffer == 0) { fprintf(stderr,"Couldn't get memory for reading %s reason: %s \n", fileName, strerror(errno)); return 0; } myText.extract(1, myText.length(), buffer); charCount--; // skip the BOM buffer[charCount] = 0; // NULL terminate for easier reading in the debugger return buffer; } /* * Remove spaces from the input UChar array. * * We check explicitly for a Unicode code value of 0x0020 * because Unicode::isSpaceChar returns true for CR, LF, etc. * */ const UChar *ThaiWordbreakTest::crunchSpaces(const UChar *spaces, int32_t count, int32_t &nonSpaceCount) { int32_t i, out, spaceCount; spaceCount = 0; for (i = 0; i < count; i += 1) { if (spaces[i] == 0x0020 /*Unicode::isSpaceChar(spaces[i])*/) { spaceCount += 1; } } nonSpaceCount = count - spaceCount; UChar *noSpaces = new UChar[nonSpaceCount]; if (noSpaces == 0) { fprintf(stderr, "Couldn't allocate memory for the space stripped text.\n"); return 0; } for (out = 0, i = 0; i < count; i += 1) { if (spaces[i] != 0x0020 /*! Unicode::isSpaceChar(spaces[i])*/) { noSpaces[out++] = spaces[i]; } } return noSpaces; } /* * Generate a text file with spaces in it from a file without. */ int generateFile(const UChar *chars, int32_t length) { Locale root(""); UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(chars, length); UErrorCode status = U_ZERO_ERROR; UnicodeSet complexContext(UNICODE_STRING_SIMPLE("[:LineBreak=SA:]"), status); BreakIterator *breakIter = BreakIterator::createWordInstance(root, status); breakIter->adoptText(noSpaceIter); char outbuf[1024]; int32_t strlength; UChar bom = 0xFEFF; printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &bom, 1, &status)); int32_t prevbreak = 0; while (U_SUCCESS(status)) { int32_t nextbreak = breakIter->next(); if (nextbreak == BreakIterator::DONE) { break; } printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &chars[prevbreak], nextbreak-prevbreak, &status)); if (nextbreak > 0 && complexContext.contains(chars[nextbreak-1]) && complexContext.contains(chars[nextbreak])) { printf(" "); } prevbreak = nextbreak; } if (U_FAILURE(status)) { fprintf(stderr, "generate failed: %s\n", u_errorName(status)); return status; } else { return 0; } } /* * The main routine. Read the command line arguments, read the text file, * remove the spaces, do the comparison and report the final results */ int main(int argc, char **argv) { char *fileName = "space.txt"; int arg = 1; UBool verbose = FALSE; UBool generate = FALSE; if (argc >= 2 && strcmp(argv[1], "-generate") == 0) { generate = TRUE; arg += 1; } if (argc >= 2 && strcmp(argv[1], "-verbose") == 0) { verbose = TRUE; arg += 1; } if (arg == argc - 1) { fileName = argv[arg++]; } if (arg != argc) { fprintf(stderr, "Usage: %s [-verbose] [<file>]\n", argv[0]); return 1; } int32_t spaceCount, nonSpaceCount; const UChar *spaces, *noSpaces; spaces = ThaiWordbreakTest::readFile(fileName, spaceCount); if (spaces == 0) { return 1; } if (generate) { return generateFile(spaces, spaceCount); } noSpaces = ThaiWordbreakTest::crunchSpaces(spaces, spaceCount, nonSpaceCount); if (noSpaces == 0) { return 1; } ThaiWordbreakTest test(spaces, spaceCount, noSpaces, nonSpaceCount, verbose); printf("word count: %d\n", test.getWordCount()); printf("breaks not found: %d\n", test.getBreaksNotFound()); printf("invalid breaks found: %d\n", test.getInvalidBreaks()); return 0; } /* * The main constructor. Clear all the counts and construct a default * word instance of a BreakIterator. */ SpaceBreakIterator::SpaceBreakIterator(const UChar *text, int32_t count) : fBreakIter(0), fText(text), fTextCount(count), fWordCount(0), fSpaceCount(0), fDone(FALSE) { UCharCharacterIterator *iter = new UCharCharacterIterator(text, count); UErrorCode status = U_ZERO_ERROR; fComplexContext.applyPattern(UNICODE_STRING_SIMPLE("[:LineBreak=SA:]"), status); Locale root(""); fBreakIter = BreakIterator::createWordInstance(root, status); fBreakIter->adoptText(iter); } SpaceBreakIterator::SpaceBreakIterator() { // nothing } /* * The destructor. delete the underlying BreakIterator */ SpaceBreakIterator::~SpaceBreakIterator() { delete fBreakIter; } /* * Return the next break, counting words and spaces. */ int32_t SpaceBreakIterator::next() { if (fDone) { return BreakIterator::DONE; } int32_t nextBreak; do { nextBreak = fBreakIter->next(); if (nextBreak == BreakIterator::DONE) { fDone = TRUE; return BreakIterator::DONE; } } while(nextBreak > 0 && fComplexContext.contains(fText[nextBreak-1]) && fComplexContext.contains(fText[nextBreak])); int32_t result = nextBreak - fSpaceCount; if (nextBreak < fTextCount) { if (fText[nextBreak] == 0x0020 /*Unicode::isSpaceChar(fText[nextBreak])*/) { fSpaceCount += fBreakIter->next() - nextBreak; } } fWordCount += 1; return result; } /* * Returns the current space count */ int32_t SpaceBreakIterator::getSpaceCount() { return fSpaceCount; } /* * Returns the current word count */ int32_t SpaceBreakIterator::getWordCount() { return fWordCount; }