void TumFeatureExtractor::customStartIO() throw(CMSException) { if (cMan == NULL) { SMILE_ERR(1,"componentManager (cMan) is NULL, smileMainThread can not be started!"); } else { // main openSMILE initialisation cMan->createInstances(0); // 0 = do not read config (we already did that ..) /* connect all the feature senders / receivers, etc. */ // get openSMILE component pointers by name from _cMan if (asink != NULL) setSmileAMQsink(cMan->getComponentInstance(asink)); if (asrc != NULL) setSmileAMQsource(cMan->getComponentInstance(asrc)); if (emmas != NULL) setSmileEMMAsender(cMan->getComponentInstance(emmas)); if (amqsink != NULL) { amqsink->setFeatureSender(featureSender,&meta); } else { SMILE_WRN(1,"amqsink == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no features will be sent now!)."); } if (emmasender != NULL) { emmasender->setEmmaSender(emmaSender,&meta); } else { SMILE_WRN(1,"emmasender == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages will be sent!)."); } // start the smile main thread, and call run smileThreadCreate( smileMainThread, smileThreadRunner, (void*)cMan ); } }
TumNonverbalFeatureExtractor::TumNonverbalFeatureExtractor(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) : ComponentForSmile("TumNonverbalFeatureExtractor",_cMan,_conf,true,false) { if (userinfoSenderInfo.topic != NULL) { userinfoSender = new EmmaSender(userinfoSenderInfo.topic, userinfoSenderInfo.componentName ); senders.push_back(userinfoSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"userinfoSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with user speaking status and gender will be sent!)."); } if (nonverbalSenderInfo.topic != NULL) { nonverbalSender = new EmmaSender(nonverbalSenderInfo.topic, nonverbalSenderInfo.componentName ); senders.push_back(nonverbalSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"nonverbalSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with non-linguistic vocalisation will be sent!)."); } }
int cLibsvmSink::myTick(long long t) { if (filehandle == NULL) return 0; SMILE_DBG(4,"tick # %i, writing to lsvm file (lag=%i):",t,lag); cVector *vec= reader_->getFrameRel(lag); //new cVector(nValues+1); if (vec == NULL) return 0; //else reader->nextFrame(); long vi = vec->tmeta->vIdx; double tm = vec->tmeta->time; long idx = 1; /* if (prname==1) { fprintf(filehandle,"'%s',",instanceName); } else if (prname==2) { fprintf(filehandle,"'%s_%i',",instanceBase,vi); } */ // classes: TODO::: if ((nClasses > 0)&&(nInst>0)) { // per instance classes if (inr >= nInst) { SMILE_WRN(3,"more instances written to LibSVM file (%i), then there are targets available for (%i)!",inr,nInst); fprintf(filehandle,"%i ",targetNumAll); } else { fprintf(filehandle,"%i ",target[inr++]); } } else { fprintf(filehandle,"%i ",targetNumAll); } // if (number) fprintf(filehandle,"%i:%i ",idx++,vi); if (timestamp) fprintf(filehandle,"%i:%f ",idx++,tm); // now print the vector: int i; fprintf(filehandle,"%i:%e ",idx++,vec->dataF[0]); for (i=1; i<vec->N; i++) { fprintf(filehandle,"%i:%e ",idx++,vec->dataF[i]); } fprintf(filehandle,"%s",NEWLINE); // tick success return 1; }
TumOpenSMILE::TumOpenSMILE(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) : ComponentForSmile("TumOpenSMILE",_cMan,_conf,true,false), callbackRecv(NULL), featureSender(NULL), callbackReceiver(NULL) { if (_conf->isSet("semaine.callbackRecv")) callbackRecv = _conf->getStr("semaine.callbackRecv"); if (wordSenderInfo.topic != NULL) { wordSender = new EmmaSender(wordSenderInfo.topic, wordSenderInfo.componentName ); senders.push_back(wordSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"wordSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with keywords will be sent!)."); } if (userinfoSenderInfo.topic != NULL) { userinfoSender = new EmmaSender(userinfoSenderInfo.topic, userinfoSenderInfo.componentName ); senders.push_back(userinfoSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"userinfoSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with user speaking status and gender will be sent!)."); } if (nonverbalSenderInfo.topic != NULL) { nonverbalSender = new EmmaSender(nonverbalSenderInfo.topic, nonverbalSenderInfo.componentName ); senders.push_back(nonverbalSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"nonverbalSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with non-linguistic vocalisation will be sent!)."); } if (emotionSenderInfo.topic != NULL) { emotionSender = new EmmaSender(emotionSenderInfo.topic, emotionSenderInfo.componentName ); senders.push_back(emotionSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"emotionSender.topic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages with dimensional affect recognition results will be sent!)."); } if ((outputVAtopic != NULL)) { //???? // check for multiple read topics separated by topicR char * tr = strdup(outputVAtopic); char * tmp = tr; char * x = NULL; do { x = (char *)strchr(tmp, ','); // separate at ',' if (x!=NULL) { *x = 0; } // remove spaces while (*tmp==' ') { tmp++; } size_t l = strlen(tmp); while (tmp[l-1] == ' ') { tmp[l-1] = 0; l--; } // push-back a callback receiver callbackReceiver = new XMLReceiver(tmp); receivers.push_back(callbackReceiver); printf("Receiver: '%s' \n",tmp); // analyse next part of string, if present if (x != NULL) { tmp = x+1; } } while (x != NULL); free(tr); } else { SMILE_WRN(1,"outputVAtopic == NULL in TumOpenSMILE, please check semaineCfg section in config file (no voice data will be read from speech output for echo compensation)."); } int period = 10; // ms if (featureTopic != NULL) { featureSender = new FeatureSender(featureTopic, "", featureSenderSourceName, period); waitingTime = period; senders.push_back(featureSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"featureTopic == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no low-level features will be sent!)."); } // Marc, 21 Dec 08: Deactivated, because it leads to dropped messages when the ASR // is on a different machine where the clock time is not the same. //featureSender->setTimeToLive(100); // discard messages after 100 ms }
// get features and put into packet int cTumkwsaSink::makeFeaturePacket(FLOAT_DMEM *frame, int Nsrc, int isTurn, APacket **p) { int i; if (Nsrc != 39) { SMILE_WRN(1,"wrong dimension of ASR input feature vector! expected 39 (13 mfcc + delta + delta) , found %i",Nsrc); return 0; } BufferInfo info; info.srcPK = MFCC + HASZEROC + HASDELTA + HASACCS + HASZEROM ; // info.srcFF = 0; //WAVEFILE; info.srcSampRate = 625; info.frSize = Nsrc*4; info.frRate = 100000; info.tgtPK = MFCC + HASZEROC + HASDELTA + HASACCS + HASZEROM ; info.tgtSampRate = 100000; info.tgtVecSize = Nsrc; info.spDetSil = 0; info.chPeak = 0; info.spDetSp = 0; info.spDetSNR = 0; info.spDetThresh = 0; info.curVol = 0; info.matTranFN = NULL; AObsData * o = new AObsData(&info, 1); prevSpeakingIndex = thisSpeakingIndex; thisSpeakingIndex = isTurn; if (prevSpeakingIndex==1 && thisSpeakingIndex==0) { // printf("turn end detected, now waiting 20 frames to compute best word hypothesis\n"); // countdown=20; countdown=0; } if (countdown > 0) { countdown=countdown-1; if ((countdown == 0)&&(isTurn!=1)) // printf(" stopping decoder. recognition result will appear soon.\n"); outputtrigger=1; } if ((countdown > 0) || (isTurn==1)) { o->data.vq[0] = 1; } else { o->data.vq[0] = 0; } // if (prevSpeakingIndex==0.0 && thisSpeakingIndex==1.0) { // printf("turn start detected.\n"); // } //if ((features[speakingIndex]==0.0)&&(wasSpeaking==1)) { o->data.vq[0] = 1; wasSpeaking=0; } //if ((features[speakingIndex]==1.0)) { wasSpeaking=1; } // o->data.fv[1][0] = (float)nFeaturesSelected; for (i=0; i<Nsrc; i++) { o->data.fv[1][i+1] = frame[i]; } // short speakStatus = o->data.vq[0]; // if (queue->PushObs(o) == 1) { // only generate packet if FIFO is full // AObsData *tmp = queue->GetObs(); // tmp->data.vq[0] = speakStatus; // use delayed packet, but current speaking status *p = new APacket(o); //tmp // TODO: USE SMILE TIMES HERE::: (*p)->SetStartTime(curTime); curTime += 100000; //info.tgtSampRate; // HARDC_SAMPR (=10) * 10000; //info.tgtSampRate; // (*p)->SetEndTime(curTime); // return 1; // } return 0; }
int cArffSink::myTick(long long t) { if (filehandle == NULL) return 0; SMILE_DBG(4,"tick # %i, reading value vector (lag=%i):",t,lag); cVector *vec= reader_->getFrameRel(lag); //new cVector(nValues+1); if (vec == NULL) return 0; //else reader->nextFrame(); long vi = vec->tmeta->vIdx; double tm = vec->tmeta->time; double len = vec->tmeta->lengthSec; if (vec->tmeta->metadata.iData[1] == 1234) { instanceName = vec->tmeta->metadata.text; if (prname==1) { fprintf(filehandle,"%s,",instanceName); } } else { if (prname==1) { fprintf(filehandle,"'%s',",instanceName); } else if (prname==2) { fprintf(filehandle,"'%s_%i',",instanceBase,vi); } } if (number) fprintf(filehandle,"%i,",vi); if (timestamp) fprintf(filehandle,"%f,",tm+frameTimeAdd); if (frameLength) fprintf(filehandle,"%f,",len); // now print the vector: int i; fprintf(filehandle,"%e",vec->dataF[0]); for (i=1; i<vec->N; i++) { fprintf(filehandle,",%e",vec->dataF[i]); //printf(" (a=%i vi=%i, tm=%fs) %s.%s = %f\n",reader->getCurR(),vi,tm,reader->getLevelName(),vec->name(i),vec->dataF[i]); } // classes: if ((vec->tmeta->metadata.iData[1] == 1234)&&(vec->tmeta->metadata.custom != NULL)&&(vec->tmeta->metadata.customLength > 0)&&(useTargetsFromMetadata)) { // TODO: check the order of the fields.... fprintf(filehandle, ",%s", (const char *)(vec->tmeta->metadata.custom)); } else { if (nClasses > 0) { if (nInst>0) { if (inr >= nInst) { SMILE_WRN(3,"more instances writte to ARFF file, then there are targets available for (%i)!",nInst); if (targetall != NULL) { for (i=0; i<nClasses; i++) { if (targetall[i] != NULL) fprintf(filehandle,",%s",targetall[i]); else fprintf(filehandle,",NULL"); } } else { for (i=0; i<nClasses; i++) { fprintf(filehandle,",NULL"); } } //inr++; } else { for (i=0; i<nClasses; i++) { fprintf(filehandle,",%s",targetinst[i][inr]); } inr++; } } else { if (targetall != NULL) { for (i=0; i<nClasses; i++) { if (targetall[i] != NULL) fprintf(filehandle,",%s",targetall[i]); else fprintf(filehandle,",NULL"); } } else { for (i=0; i<nClasses; i++) { fprintf(filehandle,",NULL"); } } } } else { // dummy class attribute, always 0 if (printDefaultClassDummyAttribute) { fprintf(filehandle,",0"); } } } fprintf(filehandle,"%s",NEWLINE); int err = fflush(filehandle); if (err == EOF) { SMILE_IERR(1,"error writing to file '%s' (code: %i)",filename,errno); COMP_ERR("aborting"); return 0; } nWritten_++; // tick success return 1; }
TumFeatureExtractor::TumFeatureExtractor(cComponentManager *_cMan, cConfigManager *_conf) throw(CMSException) : ComponentForSmile("TumFeatureExtractor",_cMan,_conf,true,false), callbackRecv(NULL), emmaSender(NULL), featureSender(NULL), callbackReceiver(NULL) { if (_conf->isSet_f(myvprint("semaine.callbackRecv[%s]",getName().c_str()))) callbackRecv = _conf->getStr_f(myvprint("semaine.callbackRecv[%s]",getName().c_str())); if ((topicR != NULL)) { // check for multiple read topics separated by topicR char * tr = strdup(topicR); char * tmp = tr; char * x = NULL; do { x = (char *)strchr(tmp, ','); // separate at ',' if (x!=NULL) { *x = 0; } // remove spaces while (*tmp==' ') { tmp++; } size_t l = strlen(tmp); while (tmp[l-1] == ' ') { tmp[l-1] = 0; l--; } // push-back a callback receiver callbackReceiver = new XMLReceiver(tmp); receivers.push_back(callbackReceiver); printf("Receiver: '%s' \n",tmp); // analyse next part of string, if present if (x != NULL) { tmp = x+1; } } while (x != NULL); free(tr); } else { SMILE_WRN(1,"topicR == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no voice data will be read from speech output for echo compensation)."); } int period = 10; // ms if (topicW != NULL) { featureSender = new FeatureSender(topicW, "", getName(), period); // featureSender = new FeatureSender("semaine.data.analysis.features.voice", "", getName(), period); waitingTime = period; senders.push_back(featureSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"topicW == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no features will be sent now!)."); } if (topicEmma != NULL) { emmaSender = new EmmaSender(topicEmma, getName()); senders.push_back(emmaSender); } else { // TODO: use semaine exception here... SMILE_WRN(1,"topicEmma == NULL in TumFeatureExtractor, please check semaineCfg section in config file (no emma XML messages will be sent!)."); } // Marc, 21 Dec 08: Deactivated, because it leads to dropped messages when the ASR // is on a different machine where the clock time is not the same. //featureSender->setTimeToLive(100); // discard messages after 100 ms }