/*---------------------------------------------------------------------------*
* make_cfst.cpp *
* *
* Copyright 2007, 2008 Nuance Communciations, Inc. *
* *
* Licensed under the Apache License, Version 2.0 (the 'License'); *
* you may not use this file except in compliance with the License. *
* *
* You may obtain a copy of the License at *
* http://www.apache.org/licenses/LICENSE-2.0 *
* *
* Unless required by applicable law or agreed to in writing, software *
* distributed under the License is distributed on an 'AS IS' BASIS, *
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
* See the License for the specific language governing permissions and *
* limitations under the License. *
* *
*---------------------------------------------------------------------------*/
#include "ptypes.h"
#include "srec_arb.h"
#include "simapi.h"
#include "fst/lib/fstlib.h"
#include "fst/lib/fst-decl.h"
#include "fst/lib/vector-fst.h"
#include "fst/lib/arcsort.h"
#include "fst/lib/invert.h"
#include "fst/lib/rmepsilon.h"
#define MAX_LINE_LENGTH 256
#define MAX_PRONS_LENGTH 1024
#define EPSILON_LABEL 0
#define MAX_MODELS 1024
#define MAXPHID 8888
#define MAX_PHONEMES 128
using namespace fst;
int usage(const char* prog)
{
printf("usage: %s -phones models/phones.map -models models/models128x.map -cfst models/generic.C -swiarb models/generic.swiarb\n", prog);
return 1;
}
typedef struct Minifst_t
{
char lcontexts[MAX_PHONEMES];
char rcontexts[MAX_PHONEMES];
int modelId;
int stateSt;
int stateEn;
phonemeID phonemeId;
unsigned char phonemeCode;
int lcontext_state[MAX_PHONEMES];
int rcontext_state[MAX_PHONEMES];
} Minifst;
int main(int argc, char **argv)
{
char* cfstFilename = 0;
char* swiarbFilename = 0;
char* phonesMap;
char* modelsMap;
int i;
phonemeID lphonId, cphonId, rphonId;
unsigned char cphon;
modelID modelId, max_modelId = 0;
int stateSt, stateEn;
int stateN, stateNp1;
int rc;
Minifst minifst[MAX_MODELS];
int do_show_text = 1;
int do_until_step = 99;
/* initial memory */
rc = PMemInit();
ASSERT( rc == ESR_SUCCESS);
// A vector FST is a general mutable FST
fst::StdVectorFst myCfst;
// fst::Fst<fst::StdArc> myCfst;
if(argc <= 1)
return usage(argv[0]);
for(i=1; i<argc; i++)
{
if(0) ;
else if(!strcmp(argv[i],"-phones"))
phonesMap = argv[++i];
else if(!strcmp(argv[i],"-models"))
modelsMap = argv[++i];
else if(!strcmp(argv[i],"-cfst"))
cfstFilename = argv[++i];
else if(!strcmp(argv[i],"-step"))
do_until_step = atoi(argv[++i]);
else if(!strcmp(argv[i],"-swiarb"))
swiarbFilename = argv[++i];
else {
return usage(argv[0]);
}
}
printf("loading %s ...\n", swiarbFilename);
CA_Arbdata* ca_arbdata = CA_LoadArbdata(swiarbFilename);
srec_arbdata *allotree = (srec_arbdata*)ca_arbdata;
/*-------------------------------------------------------------------------
*
* /---<---<---<---<---<---<---\
* / \
* / -wb-- -wb- \
* / \ / \ / \
* 0 ---#---> n ----M---> n+1 ---#---> 1
*
*
*
*
*-------------------------------------------------------------------------
*/
// Adds state 0 to the initially empty FST and make it the start state.
stateSt = myCfst.AddState(); // 1st state will be state 0 (returned by AddState)
stateEn = myCfst.AddState();
myCfst.SetStart(stateSt); // arg is state ID
myCfst.SetFinal(stateEn, 0.0); // 1st arg is state ID, 2nd arg weight
myCfst.AddArc(stateEn, fst::StdArc(EPSILON_LABEL, EPSILON_LABEL, 0.0, stateSt));
phonemeID silencePhonId = 0;
modelID silenceModelId = 0;
silenceModelId = (modelID)get_modelid_for_pic(allotree, silencePhonId, silencePhonId, silencePhonId);
// silenceModelId += MODEL_LABEL_OFFSET; #define MODEL_LABEL_OFFSET 128
for(modelId=0; modelId<MAX_MODELS; modelId++) {
minifst[modelId].modelId = MAXmodelID;
minifst[modelId].stateSt = minifst[modelId].stateEn = 0;
minifst[modelId].phonemeId = MAXphonemeID;
minifst[modelId].phonemeCode = 0;
for(i=0;i<MAX_PHONEMES;i++) {
minifst[modelId].lcontexts[i] = minifst[modelId].rcontexts[i] = 0;
minifst[modelId].lcontext_state[i] = minifst[modelId].rcontext_state[i] = 0;
}
}
for(cphonId=0; cphonId<allotree->num_phonemes && cphonId<MAXPHID; cphonId++) {
cphon = allotree->pdata[cphonId].code;
printf("processing phoneme %d of %d %d %c\n", cphonId, allotree->num_phonemes, cphon, cphon);
for(lphonId=0; lphonId<allotree->num_phonemes && lphonId<MAXPHID; lphonId++) {
unsigned char lphon = allotree->pdata[lphonId].code;
for(rphonId=0; rphonId<allotree->num_phonemes && rphonId<MAXPHID; rphonId++) {
unsigned char rphon = allotree->pdata[rphonId].code;
if( 1|| cphon=='a') { //22222
modelId = (modelID)get_modelid_for_pic(allotree, lphonId, cphonId, rphonId);
} else {
modelId = (modelID)get_modelid_for_pic(allotree, 0, cphonId, 0);
}
if(modelId == MAXmodelID) {
printf("error while get_modelid_for_pic( %p, %d, %d, %d)\n",
allotree, lphonId, cphonId, rphonId);
continue;
} else
if(do_show_text) printf("%c %c %c hmm%03d_%c %d %d %d\n", lphon, cphon, rphon, modelId, cphon, lphonId, cphonId, rphonId);
ASSERT(modelId < MAX_MODELS);
minifst[ modelId].phonemeId = cphonId;
minifst[ modelId].phonemeCode = cphon;
minifst[ modelId].modelId = modelId;
minifst[ modelId].lcontexts[lphonId] = 1;
minifst[ modelId].rcontexts[rphonId] = 1;
if(modelId>max_modelId) max_modelId = modelId;
}
}
}
printf("adding model arcs .. max_modelId %d\n",max_modelId);
for(modelId=0; modelId<=max_modelId; modelId++) {
if( minifst[modelId].modelId == MAXmodelID) continue;
cphon = minifst[modelId].phonemeCode;
minifst[modelId].stateSt = (stateN = myCfst.AddState());
minifst[modelId].stateEn = (stateNp1 = myCfst.AddState()); /* n plus 1 */
myCfst.AddArc( stateN, fst::StdArc(cphon,modelId,0.0,stateNp1));
myCfst.AddArc( stateNp1, fst::StdArc(WORD_BOUNDARY,WORD_BOUNDARY,0.0,stateNp1));
if(do_show_text) printf("%d\t\%d\t%c\t\%d\n", stateN,stateNp1,cphon,modelId);
#if 1
for( lphonId=0; lphonId<allotree->num_phonemes; lphonId++) {
minifst[modelId].lcontext_state[lphonId] = myCfst.AddState();
myCfst.AddArc( minifst[modelId].lcontext_state[lphonId],
fst::StdArc(EPSILON_LABEL,EPSILON_LABEL,0.0,
minifst[modelId].stateSt));
}
for( rphonId=0; rphonId<allotree->num_phonemes; rphonId++) {
minifst[modelId].rcontext_state[rphonId] = myCfst.AddState();
myCfst.AddArc( minifst[modelId].stateEn,
fst::StdArc(EPSILON_LABEL,EPSILON_LABEL,0.0,
minifst[modelId].rcontext_state[rphonId]));
}
#endif
}
#if 1
printf("adding cross-connections\n");
for( modelId=0; modelId<=max_modelId; modelId++) {
printf("processing model %d\n", modelId);
if( minifst[modelId].modelId == MAXmodelID) continue;
cphonId = minifst[modelId].phonemeId;
for( modelID mId=0; mId<=max_modelId; mId++) {
if( minifst[mId].modelId != MAXmodelID &&
// minifst[mId].phonemeId == rphonId &&
minifst[modelId].rcontexts[ minifst[mId].phonemeId] == 1 &&
minifst[mId].lcontexts[ cphonId] == 1) {
myCfst.AddArc( minifst[modelId].stateEn,
fst::StdArc(EPSILON_LABEL,EPSILON_LABEL,0.0,
minifst[mId].stateSt));
}
}
}
/* start node connections */
myCfst.AddArc( stateSt,
fst::StdArc(EPSILON_LABEL, EPSILON_LABEL, 0.0,
minifst[silenceModelId].stateSt));
myCfst.AddArc( minifst[silenceModelId].stateEn,
fst::StdArc(EPSILON_LABEL, EPSILON_LABEL, 0.0, stateEn));
#endif
fst::StdVectorFst fst2;
fst::StdVectorFst* ofst = &myCfst;
if(do_until_step>0) {
printf("invert\n");
fst::Invert(&myCfst);
bool FLAGS_connect = true;
if(do_until_step>1) {
printf("rmepsilon\n");
fst::RmEpsilon( &myCfst, FLAGS_connect);
if(do_until_step>2) {
printf("determinize\n");
fst::Determinize(myCfst, &fst2);
ofst = &fst2;
if(do_until_step>3) {
printf("arcsort olabels\n");
fst::ArcSort(&fst2, fst::StdOLabelCompare());
}
}
}
}
#if 0
for(fst::SymbolTableIterator syms_iter( *syms); !syms_iter.Done(); syms_iter.Next() ) {
int value = (int)syms_iter.Value();
const char* key = syms_iter.Symbol();
}
#endif
printf("writing output file %s\n", cfstFilename);
// We can save this FST to a file with:
/* fail compilation if char and LCHAR aren't the same! */
{ char zzz[ 1 - (sizeof(LCHAR)!=sizeof(char))]; zzz[0] = 0; }
ofst->Write((const char*)cfstFilename);
CA_FreeArbdata( ca_arbdata);
PMemShutdown();
// CLEANUP:
return (int)rc;
}