void TumFeatureExtractor::customStartIO() throw(CMSException)
{
if (cMan == NULL) { SMILE_ERR(1,"componentManager (cMan) is NULL, smileMainThread can not be started!"); }
else {
// main openSMILE initialisation
cMan->createInstances(0); // 0 = do not read config (we already did that ..)
/* connect all the feature senders / receivers, etc. */
// get openSMILE component pointers by name from _cMan
if (asink != NULL) setSmileAMQsink(cMan->getComponentInstance(asink));
if (asrc != NULL) setSmileAMQsource(cMan->getComponentInstance(asrc));
if (emmas != NULL) setSmileEMMAsender(cMan->getComponentInstance(emmas));
if (amqsink != NULL) {
amqsink->setFeatureSender(featureSender,&meta);
} else {
SMILE_WRN(1,"amqsink == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no features will be sent now!).");
}
if (emmasender != NULL) {
emmasender->setEmmaSender(emmaSender,&meta);
} else {
SMILE_WRN(1,"emmasender == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages will be sent!).");
}
// start the smile main thread, and call run
smileThreadCreate( smileMainThread, smileThreadRunner, (void*)cMan );
}
}
TumNonverbalFeatureExtractor::TumNonverbalFeatureExtractor(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) :
ComponentForSmile("TumNonverbalFeatureExtractor",_cMan,_conf,true,false)
{
if (userinfoSenderInfo.topic != NULL) {
userinfoSender = new EmmaSender(userinfoSenderInfo.topic, userinfoSenderInfo.componentName );
senders.push_back(userinfoSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"userinfoSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with user speaking status and gender will be sent!).");
}
if (nonverbalSenderInfo.topic != NULL) {
nonverbalSender = new EmmaSender(nonverbalSenderInfo.topic, nonverbalSenderInfo.componentName );
senders.push_back(nonverbalSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"nonverbalSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with non-linguistic vocalisation will be sent!).");
}
}
int cLibsvmSink::myTick(long long t)
{
if (filehandle == NULL) return 0;
SMILE_DBG(4,"tick # %i, writing to lsvm file (lag=%i):",t,lag);
cVector *vec= reader_->getFrameRel(lag); //new cVector(nValues+1);
if (vec == NULL) return 0;
//else reader->nextFrame();
long vi = vec->tmeta->vIdx;
double tm = vec->tmeta->time;
long idx = 1;
/*
if (prname==1) {
fprintf(filehandle,"'%s',",instanceName);
} else if (prname==2) {
fprintf(filehandle,"'%s_%i',",instanceBase,vi);
}
*/
// classes: TODO:::
if ((nClasses > 0)&&(nInst>0)) { // per instance classes
if (inr >= nInst) {
SMILE_WRN(3,"more instances written to LibSVM file (%i), then there are targets available for (%i)!",inr,nInst);
fprintf(filehandle,"%i ",targetNumAll);
} else {
fprintf(filehandle,"%i ",target[inr++]);
}
} else {
fprintf(filehandle,"%i ",targetNumAll);
}
// if (number) fprintf(filehandle,"%i:%i ",idx++,vi);
if (timestamp) fprintf(filehandle,"%i:%f ",idx++,tm);
// now print the vector:
int i;
fprintf(filehandle,"%i:%e ",idx++,vec->dataF[0]);
for (i=1; i<vec->N; i++) {
fprintf(filehandle,"%i:%e ",idx++,vec->dataF[i]);
}
fprintf(filehandle,"%s",NEWLINE);
// tick success
return 1;
}
TumOpenSMILE::TumOpenSMILE(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) :
ComponentForSmile("TumOpenSMILE",_cMan,_conf,true,false),
callbackRecv(NULL), featureSender(NULL), callbackReceiver(NULL)
{
if (_conf->isSet("semaine.callbackRecv")) callbackRecv = _conf->getStr("semaine.callbackRecv");
if (wordSenderInfo.topic != NULL) {
wordSender = new EmmaSender(wordSenderInfo.topic, wordSenderInfo.componentName );
senders.push_back(wordSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"wordSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with keywords will be sent!).");
}
if (userinfoSenderInfo.topic != NULL) {
userinfoSender = new EmmaSender(userinfoSenderInfo.topic, userinfoSenderInfo.componentName );
senders.push_back(userinfoSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"userinfoSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with user speaking status and gender will be sent!).");
}
if (nonverbalSenderInfo.topic != NULL) {
nonverbalSender = new EmmaSender(nonverbalSenderInfo.topic, nonverbalSenderInfo.componentName );
senders.push_back(nonverbalSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"nonverbalSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with non-linguistic vocalisation will be sent!).");
}
if (emotionSenderInfo.topic != NULL) {
emotionSender = new EmmaSender(emotionSenderInfo.topic, emotionSenderInfo.componentName );
senders.push_back(emotionSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"emotionSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with dimensional affect recognition results will be sent!).");
}
if ((outputVAtopic != NULL)) { //????
// check for multiple read topics separated by topicR
char * tr = strdup(outputVAtopic);
char * tmp = tr;
char * x = NULL;
do {
x = (char *)strchr(tmp, ',');
// separate at ','
if (x!=NULL) {
*x = 0;
}
// remove spaces
while (*tmp==' ') { tmp++; }
size_t l = strlen(tmp);
while (tmp[l-1] == ' ') { tmp[l-1] = 0; l--; }
// push-back a callback receiver
callbackReceiver = new XMLReceiver(tmp);
receivers.push_back(callbackReceiver);
printf("Receiver: '%s' \n",tmp);
// analyse next part of string, if present
if (x != NULL) {
tmp = x+1;
}
} while (x != NULL);
free(tr);
} else {
SMILE_WRN(1,"outputVAtopic == NULL in TumOpenSMILE, please check semaineCfg section in config file (no voice data will be read from speech output for echo compensation).");
}
int period = 10; // ms
if (featureTopic != NULL) {
featureSender = new FeatureSender(featureTopic, "", featureSenderSourceName, period);
waitingTime = period;
senders.push_back(featureSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"featureTopic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no low-level features will be sent!).");
}
// Marc, 21 Dec 08: Deactivated, because it leads to dropped messages when the ASR
// is on a different machine where the clock time is not the same.
//featureSender->setTimeToLive(100); // discard messages after 100 ms
}
// get features and put into packet
int cTumkwsaSink::makeFeaturePacket(FLOAT_DMEM *frame, int Nsrc, int isTurn, APacket **p)
{
int i;
if (Nsrc != 39) {
SMILE_WRN(1,"wrong dimension of ASR input feature vector! expected 39 (13 mfcc + delta + delta) , found %i",Nsrc);
return 0;
}
BufferInfo info;
info.srcPK = MFCC + HASZEROC + HASDELTA + HASACCS + HASZEROM ;
// info.srcFF = 0; //WAVEFILE;
info.srcSampRate = 625;
info.frSize = Nsrc*4;
info.frRate = 100000;
info.tgtPK = MFCC + HASZEROC + HASDELTA + HASACCS + HASZEROM ;
info.tgtSampRate = 100000;
info.tgtVecSize = Nsrc;
info.spDetSil = 0;
info.chPeak = 0;
info.spDetSp = 0;
info.spDetSNR = 0;
info.spDetThresh = 0;
info.curVol = 0;
info.matTranFN = NULL;
AObsData * o = new AObsData(&info, 1);
prevSpeakingIndex = thisSpeakingIndex;
thisSpeakingIndex = isTurn;
if (prevSpeakingIndex==1 && thisSpeakingIndex==0) {
// printf("turn end detected, now waiting 20 frames to compute best word hypothesis\n");
// countdown=20;
countdown=0;
}
if (countdown > 0) {
countdown=countdown-1;
if ((countdown == 0)&&(isTurn!=1))
// printf(" stopping decoder. recognition result will appear soon.\n");
outputtrigger=1;
}
if ((countdown > 0) || (isTurn==1)) {
o->data.vq[0] = 1;
} else {
o->data.vq[0] = 0;
}
// if (prevSpeakingIndex==0.0 && thisSpeakingIndex==1.0) {
// printf("turn start detected.\n");
// }
//if ((features[speakingIndex]==0.0)&&(wasSpeaking==1)) { o->data.vq[0] = 1; wasSpeaking=0; }
//if ((features[speakingIndex]==1.0)) { wasSpeaking=1; }
// o->data.fv[1][0] = (float)nFeaturesSelected;
for (i=0; i<Nsrc; i++) {
o->data.fv[1][i+1] = frame[i];
}
// short speakStatus = o->data.vq[0];
// if (queue->PushObs(o) == 1) { // only generate packet if FIFO is full
// AObsData *tmp = queue->GetObs();
// tmp->data.vq[0] = speakStatus; // use delayed packet, but current speaking status
*p = new APacket(o); //tmp
// TODO: USE SMILE TIMES HERE:::
(*p)->SetStartTime(curTime);
curTime += 100000; //info.tgtSampRate; // HARDC_SAMPR (=10) * 10000; //info.tgtSampRate; //
(*p)->SetEndTime(curTime);
//
return 1;
// }
return 0;
}
int cArffSink::myTick(long long t)
{
if (filehandle == NULL) return 0;
SMILE_DBG(4,"tick # %i, reading value vector (lag=%i):",t,lag);
cVector *vec= reader_->getFrameRel(lag); //new cVector(nValues+1);
if (vec == NULL) return 0;
//else reader->nextFrame();
long vi = vec->tmeta->vIdx;
double tm = vec->tmeta->time;
double len = vec->tmeta->lengthSec;
if (vec->tmeta->metadata.iData[1] == 1234) {
instanceName = vec->tmeta->metadata.text;
if (prname==1) {
fprintf(filehandle,"%s,",instanceName);
}
} else {
if (prname==1) {
fprintf(filehandle,"'%s',",instanceName);
} else if (prname==2) {
fprintf(filehandle,"'%s_%i',",instanceBase,vi);
}
}
if (number) fprintf(filehandle,"%i,",vi);
if (timestamp) fprintf(filehandle,"%f,",tm+frameTimeAdd);
if (frameLength) fprintf(filehandle,"%f,",len);
// now print the vector:
int i;
fprintf(filehandle,"%e",vec->dataF[0]);
for (i=1; i<vec->N; i++) {
fprintf(filehandle,",%e",vec->dataF[i]);
//printf(" (a=%i vi=%i, tm=%fs) %s.%s = %f\n",reader->getCurR(),vi,tm,reader->getLevelName(),vec->name(i),vec->dataF[i]);
}
// classes:
if ((vec->tmeta->metadata.iData[1] == 1234)&&(vec->tmeta->metadata.custom != NULL)&&(vec->tmeta->metadata.customLength > 0)&&(useTargetsFromMetadata)) {
// TODO: check the order of the fields....
fprintf(filehandle, ",%s", (const char *)(vec->tmeta->metadata.custom));
} else {
if (nClasses > 0) {
if (nInst>0) {
if (inr >= nInst) {
SMILE_WRN(3,"more instances writte to ARFF file, then there are targets available for (%i)!",nInst);
if (targetall != NULL) {
for (i=0; i<nClasses; i++) {
if (targetall[i] != NULL)
fprintf(filehandle,",%s",targetall[i]);
else
fprintf(filehandle,",NULL");
}
} else {
for (i=0; i<nClasses; i++) {
fprintf(filehandle,",NULL");
}
}
//inr++;
} else {
for (i=0; i<nClasses; i++) {
fprintf(filehandle,",%s",targetinst[i][inr]);
}
inr++;
}
} else {
if (targetall != NULL) {
for (i=0; i<nClasses; i++) {
if (targetall[i] != NULL)
fprintf(filehandle,",%s",targetall[i]);
else
fprintf(filehandle,",NULL");
}
} else {
for (i=0; i<nClasses; i++) {
fprintf(filehandle,",NULL");
}
}
}
} else {
// dummy class attribute, always 0
if (printDefaultClassDummyAttribute) {
fprintf(filehandle,",0");
}
}
}
fprintf(filehandle,"%s",NEWLINE);
int err = fflush(filehandle);
if (err == EOF) {
SMILE_IERR(1,"error writing to file '%s' (code: %i)",filename,errno);
COMP_ERR("aborting");
return 0;
}
nWritten_++;
// tick success
return 1;
}
TumFeatureExtractor::TumFeatureExtractor(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) :
ComponentForSmile("TumFeatureExtractor",_cMan,_conf,true,false),
callbackRecv(NULL), emmaSender(NULL), featureSender(NULL), callbackReceiver(NULL)
{
if (_conf->isSet_f(myvprint("semaine.callbackRecv[%s]",getName().c_str()))) callbackRecv = _conf->getStr_f(myvprint("semaine.callbackRecv[%s]",getName().c_str()));
if ((topicR != NULL)) {
// check for multiple read topics separated by topicR
char * tr = strdup(topicR);
char * tmp = tr;
char * x = NULL;
do {
x = (char *)strchr(tmp, ',');
// separate at ','
if (x!=NULL) {
*x = 0;
}
// remove spaces
while (*tmp==' ') { tmp++; }
size_t l = strlen(tmp);
while (tmp[l-1] == ' ') { tmp[l-1] = 0; l--; }
// push-back a callback receiver
callbackReceiver = new XMLReceiver(tmp);
receivers.push_back(callbackReceiver);
printf("Receiver: '%s' \n",tmp);
// analyse next part of string, if present
if (x != NULL) {
tmp = x+1;
}
} while (x != NULL);
free(tr);
} else {
SMILE_WRN(1,"topicR == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no voice data will be read from speech output for echo compensation).");
}
int period = 10; // ms
if (topicW != NULL) {
featureSender = new FeatureSender(topicW, "", getName(), period);
// featureSender = new FeatureSender("semaine.data.analysis.features.voice", "", getName(), period);
waitingTime = period;
senders.push_back(featureSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"topicW == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no features will be sent now!).");
}
if (topicEmma != NULL) {
emmaSender = new EmmaSender(topicEmma, getName());
senders.push_back(emmaSender);
} else {
// TODO: use semaine exception here...
SMILE_WRN(1,"topicEmma == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages will be sent!).");
}
// Marc, 21 Dec 08: Deactivated, because it leads to dropped messages when the ASR
// is on a different machine where the clock time is not the same.
//featureSender->setTimeToLive(100); // discard messages after 100 ms
}
