/**
* Compute the matrix Q with the method of Gram Schmidt
* @attribute n, number of rows and columns of A and Q.
* @attribute A, matrix A.
* @attribute Q, matrix Q.
*/
void matrixQ(int n, double **A, double **Q) {
int i, j, tam;
double *vectorA, *vectorE, *vectorU, *vectorAux;
createVector(&vectorA, n);
createVector(&vectorE, n);
createVector(&vectorU, n);
createVector(&vectorAux, n);
for (i = 0; i < n; i++)
vectorA[i] = A[i][0];
vectore(n, vectorA, vectorE);
for (i = 0; i < n; i++)
Q[i][0] = vectorE[i];
tam = 1;
for (j = 1; j < n; j++) {
for (i = 0; i < n; i++)
vectorA[i] = A[i][j];
vectoru(n, tam, Q, vectorA, vectorAux);
vectore(n, vectorAux, vectorE);
for (i = 0; i < n; i++)
Q[i][j] = vectorE[i];
tam++;
}
destroyVector(&vectorA, n);
destroyVector(&vectorE, n);
destroyVector(&vectorU, n);
destroyVector(&vectorAux, n);
}
AUD_Error gmm_free( void **phGmmHandle )
{
AUD_ASSERT( phGmmHandle != NULL && *phGmmHandle != NULL );
AUD_Int32s ret = 0;
GmmModel *pState = (GmmModel*)(*phGmmHandle);
ret = destroyMatrix( &(pState->means) );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &(pState->cvars) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(pState->weights) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(pState->dets) );
if ( ret == -1 )
{
AUDLOG( "this GMM has no determinant information\n" );
}
free( pState );
pState = NULL;
*phGmmHandle = NULL;
return AUD_ERROR_NONE;
}
void run_airline(Airline* self, ipc_t conn) {
mprintf("Dude\n");
ipcConn = conn;
me = self;
register_exit_function(NULL);
//redirect_signals();
Vector* threads = bootstrap_planes(self, conn);
planeThreads = threads;
listen(threads);
// If we got here, it means we recieved an end message
struct Message msg;
msg.type = MessageTypeEnd;
broadcast(threads, msg);
for (size_t i = 0; i < self->numberOfPlanes; i++) {
struct PlaneThread* t = getFromVector(threads, i);
pthread_join(t->thread, NULL);
message_queue_destroy(t->queue);
free(t);
}
destroyVector(threads);
mprintf("Out of run_airline\n");
}
int main(int argc, char *argv[])
{
int current;
unsigned int currentIndex;
int retVal = 0;
struct vector *v = createVector(10);
if (! v) {
fprintf(stderr, "Error creating vector\n");
return 1;
}
while (v->used < MAX_ITEMS && scanf("%d", ¤t) != EOF) {
for (currentIndex = 0; currentIndex < v->used; currentIndex++) {
if (elementInVector(v, currentIndex) > current) {
break;
}
}
insertIntoVector(v, current, currentIndex);
}
for (unsigned int i = 0; i < v->used; i++) {
printf("[%d]: %d\n", i, elementInVector(v, i));
}
cleanup:
destroyVector(v);
return retVal;
}
void destroyCompiler(Compiler *self) {
if (self) {
if (self->lexer) destroyLexer(self->lexer);
if (self->parser) destroyParser(self->parser);
if (self->generatorLLVM) destroyLLVMCodeGenerator(self->generatorLLVM);
if (self->semantic) destroySemanticAnalyzer(self->semantic);
if (self->sourceFiles) destroyVector(self->sourceFiles);
free(self);
verboseModeMessage("Destroyed compiler");
}
}
/**
* Function to calculate the vector u.
* @attribute tam, size of vectora, vectorRes.
* @attribute tammatrix, size of matrixe.
* @attribute matrixe, matrix of vectors e.
* @attribute vectora, vectora.
* @attribute vectorRes, vector u where is come back the result.
*/
void vectoru(int tam, int tammatrix, double **matrixe, double *vectora, double *vectorRes) {
int i, j;
double *vectorRes2, *vectorE;
createVector(&vectorRes2, tam);
createVector(&vectorE, tam);
for (i = 0; i < tam; i++)
vectorE[i] = matrixe[i][0];
projection(tam, vectorE, vectora, vectorRes2);
restVector(tam, vectora, vectorRes2, vectorRes);
for (j = 1; j < tammatrix; j++) {
for (i = 0; i < tam; i++)
vectorE[i] = matrixe[i][j];
projection(tam, vectorE, vectora, vectorRes2);
restVector(tam, vectorRes, vectorRes2, vectorRes);
}
destroyVector(&vectorRes2, tam);
destroyVector(&vectorE, tam);
}
/* Bayesian GMM adaptation */
AUD_Error gmm_adapt( void *hGmmHandle, AUD_Matrix *pAdaptData )
{
AUD_ASSERT( hGmmHandle && pAdaptData );
AUD_ASSERT( pAdaptData->dataType == AUD_DATATYPE_INT32S );
AUD_ASSERT( pAdaptData->rows > 0 );
GmmModel *pState = (GmmModel*)hGmmHandle;
AUD_Int32s ret = 0;
AUD_Int32s i, j, k, iter;
AUD_ASSERT( pAdaptData->cols == pState->width );
AUD_Double *pW = (AUD_Double*)calloc( pState->numMix * sizeof(AUD_Double), 1 );
AUD_ASSERT( pW );
AUD_Matrix mean;
mean.rows = pState->numMix;
mean.cols = pState->width;
mean.dataType = AUD_DATATYPE_DOUBLE;
ret = createMatrix( &mean );
AUD_ASSERT( ret == 0 );
// second order moment
// AUD_Matrix sndOrderMnt;
// sndOrderMnt.rows = pState->numMix;
// sndOrderMnt.cols = pState->width;
// sndOrderMnt.dataType = AUD_DATATYPE_DOUBLE;
// ret = createMatrix( &sndOrderMnt );
// AUD_ASSERT( ret == 0 );
AUD_Vector compProb;
compProb.len = pState->numMix;
compProb.dataType = AUD_DATATYPE_DOUBLE;
ret = createVector( &compProb );
AUD_ASSERT( ret == 0 );
AUD_Double gamma = 16.;
for ( iter = 0; iter < 5; iter++ )
{
ret = cleanMatrix( &mean );
AUD_ASSERT( ret == 0 );
memset( pW, 0, pState->numMix * sizeof(AUD_Double) );
for ( i = 0; i < pAdaptData->rows; i++ )
{
AUD_Double llr = 0.;
AUD_Int32s *pDataRow = pAdaptData->pInt32s + i * pAdaptData->cols;
llr = gmm_llr( hGmmHandle, pAdaptData, i, &compProb );
// AUDLOG( "component probability:\n" );
for ( j = 0; j < mean.rows; j++ )
{
AUD_Double *pMeanRow = mean.pDouble + j * mean.cols;
AUD_Double prob;
// AUD_Double *pSndOrderMntRow = sndOrderMnt.pDouble + j * sndOrderMnt.cols;
prob = exp( compProb.pDouble[j] - llr );
// AUDLOG( "%f, ", prob );
pW[j] += prob;
for ( k = 0; k < mean.cols; k++ )
{
pMeanRow[k] += prob * pDataRow[k];
// pSndOrderMntRow[k] += prob * (AUD_Double)pDataRow[k] * (AUD_Double)pDataRow[k];
// AUD_ASSERT( pSndOrderMntRow[k] > 0 );
}
}
// AUDLOG( "\n" );
}
// adaptation procedure
// AUD_Double sumWeight = 0.;
for ( j = 0; j < pState->numMix; j++ )
{
AUD_Double alpha = 1. / ( pW[j] + gamma );
// AUDLOG( "alpha: %f\n", alpha );
AUD_Double *pMeanRow = mean.pDouble + j * mean.cols;
// AUD_Double *pSndOrderMntRow = sndOrderMnt.pDouble + j * sndOrderMnt.cols;
AUD_Int32s *pGmmMeanRow = pState->means.pInt32s + j * pState->means.cols;
// AUD_Int64s *pGmmVarRow = pState->cvars.pInt64s + j * pState->cvars.cols;
// pState->weights.pDouble[j] =( alpha * pW[j] / pAdaptData->rows + ( 1 - alpha ) * pState->weights.pDouble[j] ) * gamma;
// sumWeight += pState->weights.pDouble[j];
for ( k = 0; k < pState->width; k++ )
{
AUD_Double tmpMean;
// AUD_Double tmpVar;
tmpMean = alpha * pMeanRow[k] + alpha * gamma * pGmmMeanRow[k];
// tmpVar = alpha * pSndOrderMntRow[k] - tmpMean * tmpMean + ( 1 - alpha ) * ( (AUD_Double)pGmmVarRow[k] ) + ( 1 - alpha ) * pGmmMeanRow[k] * pGmmMeanRow[k];
// AUD_ASSERT( tmpVar > 0 );
pGmmMeanRow[k] = (AUD_Int32s)round( tmpMean );
// pGmmVarRow[k] = (AUD_Int64s)tmpVar;
// AUDLOG( "%f: %f : %d : %lld, ", tmpMean, tmpVar, pGmmMeanRow[k], pGmmVarRow[k] );
}
// AUDLOG( "\n" );
}
// re-normalize weightes
// for ( j = 0; j < pState->numMix; j++ )
// {
// pState->weights.pDouble[j] /= sumWeight;
// }
}
free( pW );
pW = NULL;
ret = destroyVector( &compProb );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &mean );
AUD_ASSERT( ret == 0 );
// ret = destroyMatrix( &sndOrderMnt );
// AUD_ASSERT( ret == 0 );
return AUD_ERROR_NONE;
}
AUD_Int32s perf_dtw_mfcc_vad()
{
char inputPath[256] = { 0, };
char keywordPath[256] = { 0, };
AUD_Double threshold, minThreshold, maxThreshold, stepThreshold;
AUD_Bool isRecognized = AUD_FALSE;
AUD_Int32s data;
AUD_Error error = AUD_ERROR_NONE;
setbuf( stdin, NULL );
AUDLOG( "pls give template wav stream's path:\n" );
keywordPath[0] = '\0';
data = scanf( "%s", keywordPath );
AUDLOG( "template path is: %s\n", keywordPath );
setbuf( stdin, NULL );
AUDLOG( "pls give test wav stream's path:\n" );
inputPath[0] = '\0';
data = scanf( "%s", inputPath );
AUDLOG( "test stream path is: %s\n", inputPath );
setbuf( stdin, NULL );
AUDLOG( "pls give lowest test threshold:\n" );
data = scanf( "%lf", &minThreshold );
AUDLOG( "lowest threshold is: %.2f\n", minThreshold );
setbuf( stdin, NULL );
AUDLOG( "pls give max test threshold:\n" );
data = scanf( "%lf", &maxThreshold );
AUDLOG( "max threshold is: %.2f\n", maxThreshold );
setbuf( stdin, NULL );
AUDLOG( "pls give test threshold step:\n" );
data = scanf( "%lf", &stepThreshold );
AUDLOG( "threshold step is: %.2f\n", stepThreshold );
AUDLOG( "\n\n" );
// file & directory operation, linux dependent
entry *pEntry = NULL;
dir *pDir = NULL;
entry *pKeywordEntry = NULL;
dir *pKeywordDir = NULL;
AUD_Int32s i, j;
AUD_Feature keywordFeature;
AUD_Int32s keywordSampleNum = 0;
AUD_Int32s keywordWinNum = 0;
AUD_Int8s keywordFile[256] = { 0, };
AUD_Int8s keywordName[256] = { 0, };
AUD_Int32s keywordID = 0;
AUD_DTWSession dtwSession;
AUD_Double score = 0.;
AUD_Int32s sampleNum;
AUD_Int32s keywordBufLen = 0;
AUD_Int16s *pKeywordBuf = NULL;
AUD_Int32s bufLen = 0;
AUD_Int16s *pBuf = NULL;
AUD_Int32s ret;
// init performance benchmark
void *hBenchmark = NULL;
char logName[256] = { 0, };
snprintf( logName, 256, "dtw-mfcc-vad-%d-%d-%d-%d", WOV_DTW_TYPE, WOV_DISTANCE_METRIC, WOV_DTWTRANSITION_STYLE, WOV_DTWSCORING_METHOD );
ret = benchmark_init( &hBenchmark, (AUD_Int8s*)logName, keywords, sizeof(keywords) / sizeof(keywords[0]) );
AUD_ASSERT( ret == 0 );
for ( threshold = minThreshold; threshold < maxThreshold + stepThreshold; threshold += stepThreshold )
{
int frameStride = 0;
ret = benchmark_newthreshold( hBenchmark, threshold );
AUD_ASSERT( ret == 0 );
AUDLOG( "***********************************************\n" );
AUDLOG( "keyword training start...\n" );
pKeywordDir = openDir( (const char*)keywordPath );
keywordID = 0;
while ( ( pKeywordEntry = scanDir( pKeywordDir ) ) )
{
// train keyword
AUD_Summary fileSummary;
snprintf( (char*)keywordFile, 256, "%s/%s", (const char*)keywordPath, pKeywordEntry->name );
ret = parseWavFromFile( keywordFile, &fileSummary );
if ( ret < 0 )
{
continue;
}
AUD_ASSERT( fileSummary.channelNum == CHANNEL_NUM && fileSummary.bytesPerSample == BYTES_PER_SAMPLE && fileSummary.sampleRate == SAMPLE_RATE );
keywordBufLen = fileSummary.dataChunkBytes + 100;
pKeywordBuf = (AUD_Int16s*)calloc( keywordBufLen, 1 );
AUD_ASSERT( pKeywordBuf );
keywordSampleNum = readWavFromFile( keywordFile, pKeywordBuf, keywordBufLen );
if ( keywordSampleNum < 0 )
{
continue;
}
// front end processing
// pre-emphasis
sig_preemphasis( pKeywordBuf, pKeywordBuf, keywordSampleNum );
for ( j = 0; j * FRAME_STRIDE + FRAME_LEN <= keywordSampleNum; j++ )
{
;
}
frameStride = FRAME_STRIDE;
AUDLOG( "pattern[%d: %s] valid frame number[%d]\n", keywordID, keywordFile, j );
keywordWinNum = j;
keywordFeature.featureMatrix.rows = keywordWinNum - MFCC_DELAY;
keywordFeature.featureMatrix.cols = MFCC_FEATDIM;
keywordFeature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &(keywordFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
keywordFeature.featureNorm.len = keywordWinNum - MFCC_DELAY;
keywordFeature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(keywordFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
void *hTemplateMfccHandle = NULL;
// init mfcc handle
error = mfcc16s32s_init( &hTemplateMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, frameStride, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
// calc MFCC feature
error = mfcc16s32s_calc( hTemplateMfccHandle, pKeywordBuf, keywordSampleNum, &keywordFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
// deinit mfcc handle
error = mfcc16s32s_deinit( &hTemplateMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
strncpy( (char*)keywordName, pKeywordEntry->name, 255 );
ret = benchmark_newtemplate( hBenchmark, keywordName );
AUD_ASSERT( ret == 0 );
keywordID++;
keywordBufLen = 0;
free( pKeywordBuf );
pKeywordBuf = NULL;
AUDLOG( "keyword training finish...\n" );
// recognize
pEntry = NULL;
pDir = NULL;
pDir = openDir( (const char*)inputPath );
while ( ( pEntry = scanDir( pDir ) ) )
{
AUD_Int8s inputStream[256] = { 0, };
void *hMfccHandle = NULL;
AUD_Summary fileSummary;
snprintf( (char*)inputStream, 256, "%s/%s", (const char*)inputPath, pEntry->name );
ret = parseWavFromFile( inputStream, &fileSummary );
if ( ret < 0 )
{
continue;
}
AUD_ASSERT( fileSummary.channelNum == CHANNEL_NUM && fileSummary.bytesPerSample == BYTES_PER_SAMPLE && fileSummary.sampleRate == SAMPLE_RATE );
AUDLOG( "input stream: %s\n", inputStream );
bufLen = fileSummary.dataChunkBytes;
pBuf = (AUD_Int16s*)malloc( bufLen + 100 );
AUD_ASSERT( pBuf );
sampleNum = readWavFromFile( inputStream, pBuf, bufLen );
AUD_ASSERT( sampleNum > 0 );
// VAD
AUD_Int16s *pValidBuf = (AUD_Int16s*)calloc( bufLen, 1 );
AUD_ASSERT( pValidBuf );
AUD_Int16s *pSpeechFlag = (AUD_Int16s*)calloc( ( sampleNum / VAD_FRAME_LEN ) + 1, sizeof( AUD_Int16s ) );
AUD_ASSERT( pSpeechFlag );
void *hVadHandle = NULL;
ret = sig_vadinit( &hVadHandle, pValidBuf, pSpeechFlag, bufLen, -1, -1 );
AUD_ASSERT( ret == 0 );
AUD_Int32s start = -1;
for ( i = 0; i * VAD_FRAME_LEN + FRAME_LEN <= sampleNum; i++ )
{
ret = sig_vadupdate( hVadHandle, pBuf + i * VAD_FRAME_LEN, &start );
if ( ret == 0 )
{
continue;
}
else
{
break;
}
}
if ( ret == 0 )
{
continue;
}
AUD_Int32s validSampleNum = ret;
sig_vaddeinit( &hVadHandle );
#if 1
char vadFile[256] = { 0, };
snprintf( vadFile, 255, "./vaded/%s", pEntry->name );
ret = writeWavToFile( (AUD_Int8s*)vadFile, pValidBuf + start, validSampleNum );
AUD_ASSERT( ret == 0 );
#endif
// de-noise
AUD_Int16s *pCleanBuf = (AUD_Int16s*)calloc( (validSampleNum + start) * BYTES_PER_SAMPLE, 1 );
AUD_ASSERT( pCleanBuf );
AUD_Int32s cleanLen = denoise_mmse( pValidBuf, pCleanBuf, (validSampleNum + start) );
#if 1
char cleanFile[256] = { 0, };
snprintf( cleanFile, 255, "./clean/%s", pEntry->name );
ret = writeWavToFile( (AUD_Int8s*)cleanFile, pCleanBuf, cleanLen );
AUD_ASSERT( ret == 0 );
#endif
// front end processing
// pre-emphasis
sig_preemphasis( pCleanBuf, pCleanBuf, cleanLen );
for ( j = 0; j * FRAME_STRIDE + FRAME_LEN <= cleanLen; j++ )
{
;
}
frameStride = FRAME_LEN;
bufLen = 0;
free( pBuf );
pBuf = NULL;
free( pValidBuf );
pValidBuf = NULL;
free( pSpeechFlag );
pSpeechFlag = NULL;
AUD_Feature inFeature;
inFeature.featureMatrix.rows = j - MFCC_DELAY;
inFeature.featureMatrix.cols = MFCC_FEATDIM;
inFeature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &(inFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
inFeature.featureNorm.len = j - MFCC_DELAY;
inFeature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(inFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
// init mfcc handle
error = mfcc16s32s_init( &hMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, frameStride, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = mfcc16s32s_calc( hMfccHandle, pCleanBuf, cleanLen, &inFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
// init dtw match
dtwSession.dtwType = WOV_DTW_TYPE;
dtwSession.distType = WOV_DISTANCE_METRIC;
dtwSession.transitionType = WOV_DTWTRANSITION_STYLE;
error = dtw_initsession( &dtwSession, &keywordFeature, inFeature.featureMatrix.rows );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = dtw_updatefrmcost( &dtwSession, &inFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = dtw_match( &dtwSession, WOV_DTWSCORING_METHOD, &score, NULL );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = dtw_deinitsession( &dtwSession );
AUD_ASSERT( error == AUD_ERROR_NONE );
// deinit mfcc handle
error = mfcc16s32s_deinit( &hMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
ret = destroyMatrix( &(inFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(inFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
AUDLOG( "score: %.2f\n", score );
isRecognized = AUD_FALSE;
if ( score <= threshold )
{
isRecognized = AUD_TRUE;
}
// insert log entry
ret = benchmark_additem( hBenchmark, (AUD_Int8s*)pEntry->name, score, isRecognized );
AUD_ASSERT( ret == 0 );
cleanLen = 0;
free( pCleanBuf );
pCleanBuf = NULL;
}
closeDir( pDir );
pDir = NULL;
ret = benchmark_finalizetemplate( hBenchmark );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &(keywordFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(keywordFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
}
closeDir( pKeywordDir );
pKeywordDir = NULL;
ret = benchmark_finalizethreshold( hBenchmark );
AUD_ASSERT( ret == 0 );
}
ret = benchmark_free( &hBenchmark );
AUD_ASSERT( ret == 0 );
AUDLOG( "DTW-MFCC VAD Benchmark finished\n" );
return 0;
}
AUD_Int32s perf_dtw_mfcc()
{
char inputPath[256] = { 0, };
char keywordPath[256] = { 0, };
AUD_Double threshold, minThreshold, maxThreshold, stepThreshold;
AUD_Bool isRecognized = AUD_FALSE;
AUD_Int32s data;
AUD_Error error = AUD_ERROR_NONE;
setbuf( stdin, NULL );
AUDLOG( "pls give keyword wav stream's path:\n" );
keywordPath[0] = '\0';
data = scanf( "%s", keywordPath );
AUDLOG( "keyword path is: %s\n", keywordPath );
setbuf( stdin, NULL );
AUDLOG( "pls give test wav stream's path:\n" );
inputPath[0] = '\0';
data = scanf( "%s", inputPath );
AUDLOG( "test stream path is: %s\n", inputPath );
setbuf( stdin, NULL );
AUDLOG( "pls give min test threshold:\n" );
data = scanf( "%lf", &minThreshold );
AUDLOG( "min threshold is: %.2f\n", minThreshold );
setbuf( stdin, NULL );
AUDLOG( "pls give max test threshold:\n" );
data = scanf( "%lf", &maxThreshold );
AUDLOG( "max threshold is: %.2f\n", maxThreshold );
setbuf( stdin, NULL );
AUDLOG( "pls give test threshold step:\n" );
data = scanf( "%lf", &stepThreshold );
AUDLOG( "threshold step is: %.2f\n", stepThreshold );
AUDLOG( "\n\n" );
// file & directory operation, linux dependent
entry *pEntry = NULL;
dir *pDir = NULL;
entry *pKeywordEntry = NULL;
dir *pKeywordDir = NULL;
AUD_Feature keywordFeature;
AUD_Int32s keywordSampleNum = 0;
AUD_Int32s keywordWinNum = 0;
AUD_Int8s keywordFile[256] = { 0, };
AUD_Int8s keywordName[256] = { 0, };
AUD_Int32s keywordID = 0;
AUD_DTWSession dtwSession;
AUD_Double score = 0.;
AUD_Int32s sampleNum;
AUD_Int32s keywordBufLen = 0;
AUD_Int16s *pKeywordBuf = NULL;
AUD_Int32s bufLen = 0;
AUD_Int16s *pBuf = NULL;
AUD_Int32s frameStride = FRAME_STRIDE;
AUD_Int32s j;
AUD_Int32s ret;
// init performance benchmark
void *hBenchmark = NULL;
char logName[256] = { 0, };
snprintf( logName, 256, "dtw-mfcc-%d-%d-%d-%d", WOV_DTW_TYPE, WOV_DISTANCE_METRIC, WOV_DTWTRANSITION_STYLE, WOV_DTWSCORING_METHOD );
ret = benchmark_init( &hBenchmark, (AUD_Int8s*)logName, keywords, sizeof(keywords) / sizeof(keywords[0]) );
AUD_ASSERT( ret == 0 );
for ( threshold = minThreshold; threshold < maxThreshold + stepThreshold; threshold += stepThreshold )
{
ret = benchmark_newthreshold( hBenchmark, threshold );
AUD_ASSERT( ret == 0 );
AUDLOG( "***********************************************\n" );
AUDLOG( "keyword training start...\n" );
pKeywordDir = openDir( (const char*)keywordPath );
keywordID = 0;
while ( ( pKeywordEntry = scanDir( pKeywordDir ) ) )
{
// train keyword
keywordBufLen = SAMPLE_RATE * BYTES_PER_SAMPLE * 10;
pKeywordBuf = (AUD_Int16s*)calloc( keywordBufLen, 1 );
AUD_ASSERT( pKeywordBuf );
snprintf( (char*)keywordFile, 256, "%s/%s", (const char*)keywordPath, pKeywordEntry->name );
keywordSampleNum = readWavFromFile( keywordFile, pKeywordBuf, keywordBufLen );
if ( keywordSampleNum < 0 )
{
continue;
}
// front end processing
// pre-emphasis
sig_preemphasis( pKeywordBuf, pKeywordBuf, keywordSampleNum );
for ( j = 0; j * frameStride + FRAME_LEN <= keywordSampleNum; j++ )
{
;
}
AUDLOG( "pattern[%d: %s] valid frame number[%d]\n", keywordID, keywordFile, j );
keywordWinNum = j;
keywordFeature.featureMatrix.rows = keywordWinNum - MFCC_DELAY;
keywordFeature.featureMatrix.cols = MFCC_FEATDIM;
keywordFeature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &(keywordFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
keywordFeature.featureNorm.len = keywordWinNum - MFCC_DELAY;
keywordFeature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(keywordFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
void *hKeywordMfccHandle = NULL;
// init mfcc handle
error = mfcc16s32s_init( &hKeywordMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, frameStride, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
// calc MFCC feature
error = mfcc16s32s_calc( hKeywordMfccHandle, pKeywordBuf, keywordSampleNum, &keywordFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
// deinit mfcc handle
error = mfcc16s32s_deinit( &hKeywordMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
strncpy( (char*)keywordName, pKeywordEntry->name, 255 );
ret = benchmark_newtemplate( hBenchmark, keywordName );
AUD_ASSERT( ret == 0 );
keywordID++;
keywordBufLen = 0;
free( pKeywordBuf );
pKeywordBuf = NULL;
AUDLOG( "keyword training finish...\n" );
// recognize
pEntry = NULL;
pDir = NULL;
pDir = openDir( (const char*)inputPath );
#if PERF_PROFILE
AUD_Tick t;
#endif
while ( ( pEntry = scanDir( pDir ) ) )
{
AUD_Int8s inputStream[256] = { 0, };
void *hMfccHandle = NULL;
AUD_Summary fileSummary;
snprintf( (char*)inputStream, 256, "%s/%s", (const char*)inputPath, pEntry->name );
ret = parseWavFromFile( inputStream, &fileSummary );
if ( ret < 0 )
{
continue;
}
AUD_ASSERT( fileSummary.channelNum == CHANNEL_NUM && fileSummary.bytesPerSample == BYTES_PER_SAMPLE && fileSummary.sampleRate == SAMPLE_RATE );
AUDLOG( "input stream: %s\n", inputStream );
bufLen = fileSummary.dataChunkBytes;
pBuf = (AUD_Int16s*)calloc( bufLen + 100, 1 );
AUD_ASSERT( pBuf );
#if PERF_PROFILE
t = -time_gettick();
#endif
sampleNum = readWavFromFile( inputStream, pBuf, bufLen );
AUD_ASSERT( sampleNum > 0 );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: read wav files takes %.2f ms\n", t / 1000. );
#endif
// front end processing
// pre-emphasis
#if PERF_PROFILE
t = -time_gettick();
#endif
sig_preemphasis( pBuf, pBuf, sampleNum );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: pre-emphasis takes %.2f ms\n", t / 1000. );
#endif
for ( j = 0; j * frameStride + FRAME_LEN <= sampleNum; j++ )
{
;
}
#if PERF_PROFILE
t = -time_gettick();
#endif
AUD_Feature inFeature;
inFeature.featureMatrix.rows = j - MFCC_DELAY;
inFeature.featureMatrix.cols = MFCC_FEATDIM;
inFeature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &(inFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: create feature matrix takes %.2f ms\n", t / 1000. );
#endif
#if PERF_PROFILE
t = -time_gettick();
#endif
inFeature.featureNorm.len = j - MFCC_DELAY;
inFeature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(inFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: create feature norm takes %.2f ms\n", t / 1000. );
#endif
// init mfcc handle
#if PERF_PROFILE
t = -time_gettick();
#endif
error = mfcc16s32s_init( &hMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, frameStride, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: mfcc init takes %.2f ms\n", t / 1000. );
#endif
#if PERF_PROFILE
t = -time_gettick();
#endif
error = mfcc16s32s_calc( hMfccHandle, pBuf, sampleNum, &inFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: mfcc calc takes %.2f ms\n", t / 1000. );
#endif
// init dtw match
dtwSession.dtwType = WOV_DTW_TYPE;
dtwSession.distType = WOV_DISTANCE_METRIC;
dtwSession.transitionType = WOV_DTWTRANSITION_STYLE;
error = dtw_initsession( &dtwSession, &keywordFeature, inFeature.featureMatrix.rows );
AUD_ASSERT( error == AUD_ERROR_NONE );
#if PERF_PROFILE
t = -time_gettick();
#endif
error = dtw_updatefrmcost( &dtwSession, &inFeature );
AUD_ASSERT( error == AUD_ERROR_NONE );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: dtw update frame cost takes %.2f ms\n", t / 1000. );
#endif
#if PERF_PROFILE
t = -time_gettick();
#endif
error = dtw_match( &dtwSession, WOV_DTWSCORING_METHOD, &score, NULL );
AUD_ASSERT( error == AUD_ERROR_NONE );
#if PERF_PROFILE
t += time_gettick();
AUDLOG( "PERF: dtw match takes %.2f ms\n", t / 1000. );
#endif
error = dtw_deinitsession( &dtwSession );
AUD_ASSERT( error == AUD_ERROR_NONE );
// deinit mfcc handle
error = mfcc16s32s_deinit( &hMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
ret = destroyMatrix( &(inFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(inFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
isRecognized = AUD_FALSE;
if ( score <= threshold )
{
isRecognized = AUD_TRUE;
}
AUDLOG( "score: %.2f\n", score );
// insert log entry
ret = benchmark_additem( hBenchmark, (AUD_Int8s*)pEntry->name, score, isRecognized );
AUD_ASSERT( ret == 0 );
bufLen = 0;
free( pBuf );
pBuf = NULL;
}
closeDir( pDir );
pDir = NULL;
ret = benchmark_finalizetemplate( hBenchmark );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &(keywordFeature.featureMatrix) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(keywordFeature.featureNorm) );
AUD_ASSERT( ret == 0 );
}
closeDir( pKeywordDir );
pKeywordDir = NULL;
ret = benchmark_finalizethreshold( hBenchmark );
AUD_ASSERT( ret == 0 );
}
ret = benchmark_free( &hBenchmark );
AUD_ASSERT( ret == 0 );
AUDLOG( "DTW-MFCC Benchmark finished\n" );
return 0;
}
int main() {
int i;
double **K, **M, **Q, **T, **aux, **InverseM;
double **A, **Q2, **R;
double **L, **U;
double *eigenValues, **eigenVectors;
int aux2;
h = (double) (b - a) / n;
alfa = (double) a2 / a1;
beta = (double) b2 / b1;
if (ua == 0 && ub == 0) {
tam = n - 1;
aux2 = 1;
} else {
tam = n + 1;
aux2 = 0;
}
createVector(&x, tam);
x[0] = a + (h * aux2);
for (i = 1; i < tam; i++)
x[i] = x[i - 1] + h;
createMatrix(&K, tam, tam);
createMatrix(&M, tam, tam);
createMatrix(&InverseM, tam, tam);
createMatrix(&Q, tam, tam);
createMatrix(&T, tam, tam);
createMatrix(&aux, tam, tam);
createMatrix(&A, tam, tam);
createMatrix(&Q2, tam, tam);
createMatrix(&R, tam, tam);
createMatrix(&L, tam, tam);
createMatrix(&U, tam, tam);
createVector(&eigenValues, tam);
createMatrix(&eigenVectors, tam, tam);
funcT = functionT;
funcT2 = functionT2;
funcQ = functionQ;
funcM = functionM;
triDiagonalMatrix(tam, funcT, funcT2, funcT, T);
diagonalMatrix(tam, funcQ, Q);
diagonalMatrix(tam, funcM, M);
//inverseMatrix(tam, M, InverseM);
escaleByMatrix(tam, tam, 1.0/pow(h, 2), T, aux);
addMatrix(tam, tam, aux, Q, K);
copyMatrix(tam,tam,K,A);
//multiplicationMatrix(tam, tam, InverseM, K, A);
/*matrixLU(tam, A, L, U);
printMatrix(tam, tam, A);
printf("\n");
printMatrix(tam, tam, L);
printf("\n");
L[0][0] = 0;
L[0][1] = 0;
L[1][0] = 0;
L[1][1] = 0;
printf("%d\n", isCeroDiagonalInf(L,tam));
printMatrix(tam, tam, U);*/
factorizeUL(tam,A,eigenValues,eigenVectors);
printf("Eigenvalues:\n");
printVector(tam, eigenValues);
printf("\nEigenvectors:\n");
printMatrix(tam,tam, eigenVectors);
destroyMatrix(&K, tam, tam);
destroyMatrix(&M, tam, tam);
destroyMatrix(&InverseM, tam, tam);
destroyMatrix(&Q, tam, tam);
destroyMatrix(&T, tam, tam);
destroyMatrix(&aux, tam, tam);
destroyMatrix(&A, tam, tam);
destroyMatrix(&Q2, tam, tam);
destroyMatrix(&R, tam, tam);
destroyMatrix(&L, tam, tam);
destroyMatrix(&U, tam, tam);
destroyVector(&x, tam);
destroyVector(&eigenValues, tam);
destroyMatrix(&eigenVectors, tam, tam);
return 0;
}
AUD_Int32s wov_adapt_gmm_si()
{
AUD_Error error = AUD_ERROR_NONE;
AUD_Int32s ret = 0;
AUD_Int8s wavPath[256] = { 0, };
AUD_Int32s data;
setbuf( stdout, NULL );
setbuf( stdin, NULL );
AUDLOG( "pls give adapt wav stream's folder path:\n" );
wavPath[0] = '\0';
data = scanf( "%s", wavPath );
AUDLOG( "adapt wav stream's folder path is: %s\n", wavPath );
// step 1: read UBM model from file
void *hUbm = NULL;
FILE *fpUbm = fopen( WOV_UBM_GMMMODEL_FILE, "rb" );
if ( fpUbm == NULL )
{
AUDLOG( "cannot open ubm model file: [%s]\n", WOV_UBM_GMMMODEL_FILE );
return AUD_ERROR_IOFAILED;
}
error = gmm_import( &hUbm, fpUbm );
AUD_ASSERT( error == AUD_ERROR_NONE );
fclose( fpUbm );
fpUbm = NULL;
// AUDLOG( "ubm GMM as:\n" );
// gmm_show( hUbm );
AUD_Int32s i = 0, j = 0;
entry *pEntry = NULL;
dir *pDir = NULL;
AUD_Int32s totalWinNum = 0;
pDir = openDir( (const char*)wavPath );
if ( pDir == NULL )
{
AUDLOG( "cannot open folder: %s\n", wavPath );
return -1;
}
while ( ( pEntry = scanDir( pDir ) ) )
{
AUD_Int8s keywordFile[256] = { 0, };
AUD_Summary fileSummary;
AUD_Int32s sampleNum = 0;
snprintf( (char*)keywordFile, 256, "%s/%s", wavPath, pEntry->name );
// AUDLOG( "%s\n", keywordFile );
ret = parseWavFromFile( keywordFile, &fileSummary );
if ( ret < 0 )
{
continue;
}
AUD_ASSERT( fileSummary.channelNum == CHANNEL_NUM && fileSummary.bytesPerSample == BYTES_PER_SAMPLE && fileSummary.sampleRate == SAMPLE_RATE );
// request memeory for template
sampleNum = fileSummary.dataChunkBytes / fileSummary.bytesPerSample;
for ( j = 0; j * FRAME_STRIDE + FRAME_LEN <= sampleNum; j++ )
{
;
}
j = j - MFCC_DELAY;
totalWinNum += j;
}
closeDir( pDir );
pDir = NULL;
AUD_Matrix featureMatrix;
featureMatrix.rows = totalWinNum;
featureMatrix.cols = MFCC_FEATDIM;
featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &featureMatrix );
AUD_ASSERT( ret == 0 );
AUD_Int32s currentRow = 0;
pDir = openDir( (const char*)wavPath );
while ( ( pEntry = scanDir( pDir ) ) )
{
AUD_Int8s keywordFile[256] = { 0, };
AUD_Summary fileSummary;
AUD_Int32s sampleNum = 0;
void *hMfccHandle = NULL;
snprintf( (char*)keywordFile, 256, "%s/%s", wavPath, pEntry->name );
// AUDLOG( "%s\n", keywordFile );
ret = parseWavFromFile( keywordFile, &fileSummary );
if ( ret < 0 )
{
continue;
}
AUD_ASSERT( fileSummary.channelNum == CHANNEL_NUM && fileSummary.bytesPerSample == BYTES_PER_SAMPLE && fileSummary.sampleRate == SAMPLE_RATE );
AUD_Int32s bufLen = fileSummary.dataChunkBytes;
AUD_Int16s *pBuf = (AUD_Int16s*)calloc( bufLen, 1 );
AUD_ASSERT( pBuf );
sampleNum = readWavFromFile( (AUD_Int8s*)keywordFile, pBuf, bufLen );
AUD_ASSERT( sampleNum > 0 );
// pre-processing
// pre-emphasis
sig_preemphasis( pBuf, pBuf, sampleNum );
// calc framing number
for ( j = 0; j * FRAME_STRIDE + FRAME_LEN <= sampleNum; j++ )
{
;
}
// XXX: select salient frames
AUD_Feature feature;
feature.featureMatrix.rows = j - MFCC_DELAY;
feature.featureMatrix.cols = MFCC_FEATDIM;
feature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
feature.featureMatrix.pInt32s = featureMatrix.pInt32s + currentRow * feature.featureMatrix.cols;
feature.featureNorm.len = j - MFCC_DELAY;
feature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(feature.featureNorm) );
AUD_ASSERT( ret == 0 );
error = mfcc16s32s_init( &hMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, FRAME_STRIDE, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = mfcc16s32s_calc( hMfccHandle, pBuf, sampleNum, &feature );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = mfcc16s32s_deinit( &hMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
free( pBuf );
pBuf = NULL;
bufLen = 0;
ret = destroyVector( &(feature.featureNorm) );
AUD_ASSERT( ret == 0 );
currentRow += feature.featureMatrix.rows;
}
closeDir( pDir );
pDir = NULL;
AUD_Matrix llrMatrix;
llrMatrix.rows = totalWinNum;
llrMatrix.cols = gmm_getmixnum( hUbm );
llrMatrix.dataType = AUD_DATATYPE_DOUBLE;
ret = createMatrix( &llrMatrix );
AUD_ASSERT( ret == 0 );
AUD_Double llr = 0.;
for ( j = 0; j < featureMatrix.rows; j++ )
{
AUD_Vector componentLLR;
componentLLR.len = llrMatrix.cols;
componentLLR.dataType = AUD_DATATYPE_DOUBLE;
componentLLR.pDouble = llrMatrix.pDouble + j * llrMatrix.cols;
llr = gmm_llr( hUbm, &(featureMatrix), j, &componentLLR );
}
AUD_Vector sumLlr;
sumLlr.len = llrMatrix.cols;
sumLlr.dataType = AUD_DATATYPE_DOUBLE;
ret = createVector( &sumLlr );
AUD_ASSERT( ret == 0 );
AUD_Double *pSumLlr = sumLlr.pDouble;
for ( j = 0; j < llrMatrix.cols; j++ )
{
pSumLlr[j] = llrMatrix.pDouble[j];
}
for ( i = 1; i < llrMatrix.rows; i++ )
{
for ( j = 0; j < llrMatrix.cols; j++ )
{
pSumLlr[j] = logadd( pSumLlr[j], *(llrMatrix.pDouble + i * llrMatrix.cols + j) );
}
}
#if 0
AUD_Vector bestIndex;
bestIndex.len = TOP_N;
bestIndex.dataType = AUD_DATATYPE_INT32S;
ret = createVector( &bestIndex );
AUD_ASSERT( ret == 0 );
// get top TOP_N component
ret = sortVector( &sumLlr, &bestIndex );
AUD_ASSERT( ret == 0 );
#else
llr = pSumLlr[0];
for ( j = 1; j < sumLlr.len; j++ )
{
llr = logadd( llr, pSumLlr[j] );
}
// AUDLOG( "llr: %.f\n", llr );
AUD_Vector sortIndex;
sortIndex.len = sumLlr.len;
sortIndex.dataType = AUD_DATATYPE_INT32S;
ret = createVector( &sortIndex );
AUD_ASSERT( ret == 0 );
ret = sortVector( &sumLlr, &sortIndex );
AUD_ASSERT( ret == 0 );
int num = 0;
double val = 0.;
for ( i = 0; i < sortIndex.len; i++ )
{
// ln( 0.001 ) ~= -7.
val = pSumLlr[sortIndex.pInt32s[i]] - llr + 7.;
// AUDLOG( "%f, \n", val );
if ( val < 0 )
{
break;
}
num++;
}
// AUDLOG( "\n" );
AUD_ASSERT( num > 0 );
AUDLOG( "computed component num: %d\n", num );
num = AUD_MAX( num, TOP_N );
AUDLOG( "normalized component num: %d\n", num );
AUD_Vector bestIndex;
bestIndex.len = num;
bestIndex.dataType = AUD_DATATYPE_INT32S;
bestIndex.pInt32s = sortIndex.pInt32s;
#endif
int slash = '/';
char *ptr = strrchr( (char*)wavPath, slash );
ptr++;
// select imposter GMM
void *hImposterGmm = NULL;
AUD_Int8s imposterGmmName[256] = { 0, };
snprintf( (char*)imposterGmmName, 256, "%s-imposter", ptr );
error = gmm_select( &hImposterGmm, hUbm, &bestIndex, 0 | GMM_INVERTSELECT_MASK, imposterGmmName );
AUD_ASSERT( error == AUD_ERROR_NONE );
gmm_show( hImposterGmm );
// export gmm
char imposterFile[256] = { 0 };
snprintf( imposterFile, 256, "%s/%s-imposter.gmm", WOV_IMPOSTER_GMMMODEL_DIR, ptr );
AUDLOG( "Export imposter GMM Model to: %s\n", imposterFile );
FILE *fpImposterGmm = fopen( imposterFile, "wb" );
AUD_ASSERT( fpImposterGmm );
error = gmm_export( hImposterGmm, fpImposterGmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
AUDLOG( "Export imposter GMM Model File Done\n" );
fclose( fpImposterGmm );
fpImposterGmm = NULL;
error = gmm_free( &hImposterGmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
// select keyword GMM
void *hAdaptedGmm = NULL;
AUD_Int8s adaptedGmmName[256] = { 0, };
snprintf( (char*)adaptedGmmName, 256, "%s", ptr );
// AUDLOG( "%s\n", adaptedGmmName );
error = gmm_select( &hAdaptedGmm, hUbm, &bestIndex, 0, adaptedGmmName );
AUD_ASSERT( error == AUD_ERROR_NONE );
ret = destroyVector( &sumLlr );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &llrMatrix );
AUD_ASSERT( ret == 0 );
#if 0
ret = destroyVector( &bestIndex );
AUD_ASSERT( ret == 0 );
#else
ret = destroyVector( &sortIndex );
AUD_ASSERT( ret == 0 );
#endif
#if 1
// adapt GMM
error = gmm_adapt( hAdaptedGmm, &featureMatrix );
AUD_ASSERT( error == AUD_ERROR_NONE );
#endif
gmm_show( hAdaptedGmm );
// export gmm
char modelFile[256] = { 0 };
snprintf( modelFile, 256, "%s/%s.gmm", WOV_KEYWORD_GMMMODEL_DIR, ptr );
AUDLOG( "Export GMM Model to: %s\n", modelFile );
FILE *fpGmm = fopen( modelFile, "wb" );
AUD_ASSERT( fpGmm );
error = gmm_export( hAdaptedGmm, fpGmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
AUDLOG( "Export GMM Model File Done\n" );
fclose( fpGmm );
fpGmm = NULL;
ret = destroyMatrix( &featureMatrix );
AUD_ASSERT( ret == 0 );
error = gmm_free( &hAdaptedGmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = gmm_free( &hUbm );
AUD_ASSERT( error == AUD_ERROR_NONE );
AUDLOG( "keyword model adapt2 done\n" );
return 0;
}
AUD_Int32s train_keyword_hmm( const AUD_Int8s *pKeywordFile, AUD_Int8s *pHmmName )
{
AUD_Error error = AUD_ERROR_NONE;
// step 1: read garbage model from file
void *hGarbageGmm = NULL;
FILE *fpGarbage = fopen( (char*)WOV_UBM_GMMHMMMODEL_FILE, "rb" );
if ( fpGarbage == NULL )
{
AUDLOG( "cannot open gmm model file: [%s]\n", WOV_UBM_GMMHMMMODEL_FILE );
return AUD_ERROR_IOFAILED;
}
error = gmm_import( &hGarbageGmm, fpGarbage );
AUD_ASSERT( error == AUD_ERROR_NONE );
fclose( fpGarbage );
fpGarbage = NULL;
// AUDLOG( "garbage GMM as:\n" );
// gmm_show( hGarbageGmm );
// step 2: read template stream & extract MFCC feature vector
AUD_Int32s sampleNum = 0;
AUD_Int32s bufLen = SAMPLE_RATE * BYTES_PER_SAMPLE * 10;
AUD_Int16s *pBuf = (AUD_Int16s*)calloc( bufLen, 1 );
AUD_ASSERT( pBuf );
AUD_Int32s ret;
// read stream from file
sampleNum = readWavFromFile( (AUD_Int8s*)pKeywordFile, pBuf, bufLen );
AUD_ASSERT( sampleNum > 0 );
AUD_Int32s i = 0, j = 0, k = 0, m = 0;
// front end processing
// pre-emphasis
sig_preemphasis( pBuf, pBuf, sampleNum );
// calc frame number
for ( j = 0; j * FRAME_STRIDE + FRAME_LEN <= sampleNum; j++ )
{
;
}
AUD_Feature feature;
feature.featureMatrix.rows = j - MFCC_DELAY;
feature.featureMatrix.cols = MFCC_FEATDIM;
feature.featureMatrix.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &(feature.featureMatrix) );
AUD_ASSERT( ret == 0 );
feature.featureNorm.len = j - MFCC_DELAY;
feature.featureNorm.dataType = AUD_DATATYPE_INT64S;
ret = createVector( &(feature.featureNorm) );
AUD_ASSERT( ret == 0 );
// init mfcc handle
void *hMfccHandle = NULL;
error = mfcc16s32s_init( &hMfccHandle, FRAME_LEN, WINDOW_TYPE, MFCC_ORDER, FRAME_STRIDE, SAMPLE_RATE, COMPRESS_TYPE );
AUD_ASSERT( error == AUD_ERROR_NONE );
// calc MFCC feature
error = mfcc16s32s_calc( hMfccHandle, pBuf, sampleNum, &feature );
AUD_ASSERT( error == AUD_ERROR_NONE );
free( pBuf );
pBuf = NULL;
// step 3: for each feature vector, get the bestN most likelihood component indices from GMM
AUD_Vector componentLLR;
componentLLR.len = gmm_getmixnum( hGarbageGmm );
componentLLR.dataType = AUD_DATATYPE_DOUBLE;
ret = createVector( &componentLLR );
AUD_ASSERT( ret == 0 );
AUD_Matrix indexTable;
indexTable.rows = feature.featureMatrix.rows ;
indexTable.cols = WOV_KEYWORD_GMMMODEL_ORDER;
indexTable.dataType = AUD_DATATYPE_INT32S;
ret = createMatrix( &indexTable );
AUD_ASSERT( ret == 0 );
AUD_Matrix llrTable;
llrTable.rows = feature.featureMatrix.rows;
llrTable.cols = WOV_KEYWORD_GMMMODEL_ORDER;
llrTable.dataType = AUD_DATATYPE_DOUBLE;
ret = createMatrix( &llrTable );
AUD_ASSERT( ret == 0 );
AUD_Double totalLLR;
for ( i = 0; i < feature.featureMatrix.rows; i++ )
{
totalLLR = gmm_llr( hGarbageGmm, &(feature.featureMatrix), i, &componentLLR );
#if 0
showVector( &componentLLR );
#endif
// sort the bestN likelihood
AUD_Int32s *pIndex = indexTable.pInt32s + i * indexTable.cols;
AUD_Double *pLLR = llrTable.pDouble + i * llrTable.cols;
for ( j = 0; j < WOV_KEYWORD_GMMMODEL_ORDER; j++ )
{
pIndex[j] = -1;
pLLR[j] = 0.;
}
for ( j = 0; j < componentLLR.len; j++ )
{
for ( k = 0; k < WOV_KEYWORD_GMMMODEL_ORDER; k++ )
{
if ( pIndex[k] == -1 )
{
pIndex[k] = j;
pLLR[k] = componentLLR.pDouble[j];
break;
}
else if ( componentLLR.pDouble[j] > pLLR[k] )
{
for ( m = WOV_KEYWORD_GMMMODEL_ORDER - 1; m > k ; m-- )
{
pIndex[m] = pIndex[m - 1];
pLLR[m] = pLLR[m - 1];
}
pIndex[k] = j;
pLLR[k] = componentLLR.pDouble[j];
break;
}
}
}
}
#if 0
AUDLOG( "index table( %s, %s, %d ):\n", __FILE__, __FUNCTION__, __LINE__ );
showMatrix( &indexTable );
AUDLOG( "llr table( %s, %s, %d ):\n", __FILE__, __FUNCTION__, __LINE__ );
showMatrix( &llrTable );
#endif
ret = destroyVector( &componentLLR );
AUD_ASSERT( ret == 0 );
// step 4: cluster GMM
AUD_Int32s *pClusterLabel = (AUD_Int32s*)calloc( sizeof(AUD_Int32s) * feature.featureMatrix.rows, 1 );
AUD_ASSERT( pClusterLabel );
error = gmm_cluster( hGarbageGmm, &indexTable, WOV_GMM_CLUSTER_THRESHOLD, pClusterLabel );
AUD_ASSERT( error == AUD_ERROR_NONE );
AUD_Int32s stateNum = pClusterLabel[feature.featureMatrix.rows - 1];
AUD_ASSERT( stateNum >= 5 );
// step 5: select and build state GMM
void **phKeywordGmms = (void**)calloc( sizeof(void*) * stateNum, 1 );
AUD_ASSERT( phKeywordGmms );
AUD_Vector indexVector;
indexVector.len = WOV_KEYWORD_GMMMODEL_ORDER;
indexVector.dataType = AUD_DATATYPE_INT32S;
ret = createVector( &indexVector );
AUD_ASSERT( ret == 0 );
AUD_Vector llrVector;
llrVector.len = WOV_KEYWORD_GMMMODEL_ORDER;
llrVector.dataType = AUD_DATATYPE_DOUBLE;
ret = createVector( &llrVector );
AUD_ASSERT( ret == 0 );
int start = 0, end = 0;
for ( i = 0; i < stateNum; i++ )
{
for ( j = 0; j < indexVector.len; j++ )
{
indexVector.pInt32s[j] = -1;
llrVector.pInt32s[j] = 1.;
}
for ( j = start; j < feature.featureMatrix.rows; j++ )
{
if ( pClusterLabel[j] != i )
{
break;
}
}
end = j;
for ( k = start * llrTable.cols; k < end * llrTable.cols; k++ )
{
for ( m = 0; m < indexVector.len; m++ )
{
if ( llrTable.pDouble[k] == llrVector.pDouble[m] &&
indexTable.pInt32s[k] == indexVector.pInt32s[m] )
{
break;
}
else if ( indexVector.pInt32s[m] == -1 || llrTable.pDouble[k] > llrVector.pDouble[m] )
{
for ( int n = indexVector.len - 1; n > m ; n-- )
{
indexVector.pInt32s[n] = indexVector.pInt32s[n - 1];
llrVector.pDouble[n] = llrVector.pDouble[n - 1];
}
indexVector.pInt32s[m] = indexTable.pInt32s[k];
llrVector.pDouble[m] = llrTable.pDouble[k];
break;
}
}
}
// AUDLOG( "Final GMM indices for state[%d]:\n", i );
// showVector( &indexVector );
AUD_Int8s gmmName[256] = { 0, };
sprintf( (char*)gmmName, "state%d", i );
error = gmm_select( &phKeywordGmms[i], hGarbageGmm, &indexVector, 0, gmmName );
AUD_ASSERT( error == AUD_ERROR_NONE );
start = end;
}
ret = destroyMatrix( &indexTable );
AUD_ASSERT( ret == 0 );
ret = destroyMatrix( &llrTable );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &indexVector );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &llrVector );
AUD_ASSERT( ret == 0 );
free( pClusterLabel );
pClusterLabel = NULL;
// step 6: generate keyword model by Baum-Welch algorithm
AUD_Vector pi;
pi.len = stateNum;
pi.dataType = AUD_DATATYPE_DOUBLE;
ret = createVector( &pi );
AUD_ASSERT( ret == 0 );
pi.pDouble[0] = 1.0f;
void *hKeywordHmm = NULL;
error = gmmhmm_init( &hKeywordHmm, stateNum, &pi, phKeywordGmms );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = gmmhmm_learn( hKeywordHmm, &feature, 1, 0.001 );
AUD_ASSERT( error == AUD_ERROR_NONE );
// step 8: write model to file
error = gmmhmm_export( hKeywordHmm, pHmmName );
AUD_ASSERT( error == AUD_ERROR_NONE );
// gmmhmm_show( hKeywordHmm );
// clean field
error = mfcc16s32s_deinit( &hMfccHandle );
AUD_ASSERT( error == AUD_ERROR_NONE );
error = gmm_free( &hGarbageGmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
ret = destroyMatrix( &(feature.featureMatrix) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &(feature.featureNorm) );
AUD_ASSERT( ret == 0 );
ret = destroyVector( &pi );
AUD_ASSERT( ret == 0 );
for ( i = 0; i < stateNum; i++ )
{
error = gmm_free( &phKeywordGmms[i] );
AUD_ASSERT( error == AUD_ERROR_NONE );
}
free( phKeywordGmms );
phKeywordGmms = NULL;
error = gmmhmm_free( &hKeywordHmm );
AUD_ASSERT( error == AUD_ERROR_NONE );
return 0;
}
