void FactorAnalysisStat::substractChannelStats(){ if (verbose) cout <<"(FactorAnalysisStat) Compute and Substract Channel FA Stats... "<<endl; RealVector <double> UX;UX.setSize(_supervsize); UX.setAllValues(0.0);double* ux=UX.getArray(); // Compute Occupations and Statistics unsigned long loc=0; unsigned long sent=0; XLine *pline; String *pFile; fileList.rewind(); double *super_mean=_super_mean.getArray(); double *N_h=_matN_h.getArray(); double *S_X=_matS_X.getArray(); while((pline=fileList.getLine())!=NULL) { while((pFile=pline->getElement())!=NULL) { this->getUX(UX,*pFile); for(unsigned long k=0;k<_supervsize;k++) ux[k]+=super_mean[k]; for(unsigned long k=0;k<_mixsize;k++) for (unsigned long i=0;i<_vsize;i++) S_X[loc*_supervsize+(k*_vsize+i)]-= N_h[sent*_mixsize+k]*ux[i+k*_vsize]; sent++; } loc++; } };
void FactorAnalysisStat::getXEstimate(){ if (verbose) cout << "(FactorAnalysisStat) Compute X Estimate "<<endl; RealVector <double> AUX; AUX.setSize(_rang); XLine *pline; String *pFile; _matX.setAllValues(0.0); double *X=_matX.getArray(); double *U=_matU.getArray(); double *S_X_h=_matS_X_h.getArray(); double *aux=AUX.getArray(); double *super_invvar=_super_invvar.getArray(); fileList.rewind(); while((pline=fileList.getLine())!=NULL) { while((pFile=pline->getElement())!=NULL) { unsigned long sent=_ndxTable.sessionNb(*pFile); AUX.setAllValues(0.0); for(unsigned long i=0;i<_rang;i++) for(unsigned long k=0;k<_supervsize;k++) aux[i]+= U[k*_rang+i]*super_invvar[k]*S_X_h[sent*_supervsize+k]; double *l_h_inv=_l_h_inv[sent].getArray(); for(unsigned long i=0;i<_rang;i++) for(unsigned long k=0;k<_rang;k++) X[sent*_rang+i]+=l_h_inv[i*_rang+k]*aux[k]; sent++; } } };
//------------------------------------------------------------------------- // Load the NGRAM table, selecting the nbSelected first void NGram::load(const String filename,Config &config){ XList input(filename,config); XLine *linep; // Pointor on the current test line input.getLine(0); unsigned long idx=0; while (((linep=input.getLine()) != NULL)&&(idx<getSize())){ for (unsigned long i=0;i<getOrder();i++){ short int a=linep->getElement(i).toLong(); setSymbol(idx,i,a); } if (linep->getElementCount()==(getOrder()+1)){ unsigned long count=linep->getElement(getOrder()).toLong(); setCount(idx,count);_totalCount+=count;} else setCount(idx,0); idx++; } if (idx!=getSize()){ cout << "WARNING ! Number of ngram in the file["<<idx<<"] < to the number requested ["<<getSize()<<"]"<<endl; setSize(idx); } if (verboseLevel>1){ cout <<"load symbol table from ["<<filename <<"]"<<endl; showTable(); } }
// Information on the quantity of data available for each client // Outputs a list with the selected files for a defined quantity of data int ExtractTargetDataInfo(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); bool fixedLabelSelectedFrame; String labelSelectedFrames; if (config.existsParam("useIdForSelectedFrame")) // the ID of each speaker is used as labelSelectedFrame fixedLabelSelectedFrame=false; else{ // the label is decided by the command line and is unique for the run labelSelectedFrames=config.getParam("labelSelectedFrames"); if (verbose) cout << "Computing on" << labelSelectedFrames << " label" << endl; fixedLabelSelectedFrame=true; } unsigned long maxFrame=config.getParam("maxFrame").toLong(); String outputFilename=config.getParam("outputFilename"); ofstream outputFile(outputFilename.c_str(),ios::out| ios::trunc); try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine * linep; if (verbose) cout << "InfoTarget" << endl; // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String *id=linep->getElement(); // Get the Client ID (id) outputFile<<*id; String currentFile=""; XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) if (verbose) cout << "Info model ["<<*id<<"]"<<endl; if (!fixedLabelSelectedFrame){ // the ID is used as label for selecting the frame labelSelectedFrames=*id; if (debug) cout <<*id<<" is used for label selected frames"<<endl; } // label files reading - It creates, for each file and each label, a cluster of segments - will be integrated witth the featre s - asap SegServer segmentsServer; // Reading the segmentation files for each feature input file LabelServer labelServer; initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file unsigned long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the cluster with in interest audio segments SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments Seg *seg; // Will give the current segment unsigned long frameCount=0; selectedSegments.rewind(); // at the begin of the selected segments list while(((seg=selectedSegments.getSeg())!=NULL) && (frameCount<maxFrame)){ // For each of the selected segments until the amount of data is get frameCount+=seg->length(); cout << seg->sourceName()<<" "<<seg->begin()<<" "<<seg->length()<<" Total time="<<frameCount<<endl; if (seg->sourceName()!=currentFile){ outputFile<<" "<<seg->sourceName(); currentFile=seg->sourceName(); } } // end of the initial Train Iteration loop outputFile<<endl; if (verbose) cout << "Save info client ["<<*id<<"]" << endl; } // end of the the target loop } // fin try catch (Exception& e) { cout << e.toString().c_str() << endl; } return 0; }
FactorAnalysisStat::FactorAnalysisStat(String & featFilename,FeatureServer & fs,Config & config):_ms(config),_ss(config){ // constructor for a single file XList faNdx; XLine featLine; featLine.addElement(featFilename); faNdx.addLine()=featLine; _init(faNdx,fs,config); }
//------------------------------------------------------------------------- // Get a tab with indexes of speakers with maximum likelihood (mdtm and etf only) void getTarClientIdx(Config & config, XList & inputList, unsigned long nbLoc, unsigned long * tarTab) { XLine *linep; unsigned long cpt=0; unsigned long cpttab=0; double minLLK=config.getParam("minLLK").toDouble(); double maxScore=minLLK; unsigned long idxTar=0; bool verbose=config.existsParam("verbose"); while((linep=inputList.getLine())!=NULL) { double score=linep->getElement(6).toDouble(); if (score>=maxScore) { maxScore=score; // this is the maximum score idxTar=cpt; // index is just the line if (verbose) {cout << "giving highest score to " << linep->getElement(1) << " "<<maxScore << endl;} } if (cpt%nbLoc==(nbLoc-1)) // when changing segment { tarTab[cpttab]=idxTar; // idx of the target goes in the tab if (verbose) {cout << linep->getElement(1) << " max score: "<<maxScore <<"idx: "<<idxTar<<"cpt: "<<cpt<< endl;} cpttab++; maxScore=minLLK; //reset maxScore } cpt++; } }
/* * Checks what users match a given vector of ELines and sets their ban exempt flag accordingly. */ void XLineManager::CheckELines() { ContainerIter n = lookup_lines.find("E"); if (n == lookup_lines.end()) return; XLineLookup& ELines = n->second; if (ELines.empty()) return; for (LocalUserList::const_iterator u2 = ServerInstance->Users->local_users.begin(); u2 != ServerInstance->Users->local_users.end(); u2++) { LocalUser* u = *u2; /* This uses safe iteration to ensure that if a line expires here, it doenst trash the iterator */ LookupIter safei; for (LookupIter i = ELines.begin(); i != ELines.end(); ) { safei = i; safei++; XLine *e = i->second; u->exempt = e->Matches(u); i = safei; } } }
EventReturn OnCheckKick(User *u, Channel *c, Anope::string &mask, Anope::string &reason) override { XLine *x = this->sqlines.CheckChannel(c); if (x) { this->sqlines.OnMatch(u, x); reason = x->GetReason(); return EVENT_STOP; } return EVENT_CONTINUE; }
ModResult OnUserRegister(LocalUser* user) override { // Apply lines on user connect XLine *rl = ServerInstance->XLines->MatchesLine("R", user); if (rl) { // Bang. :P rl->Apply(user); return MOD_RES_DENY; } return MOD_RES_PASSTHRU; }
CmdResult CommandAddLine::Handle(User* usr, std::vector<std::string>& params) { XLineFactory* xlf = ServerInstance->XLines->GetFactory(params[0]); const std::string& setter = usr->nick; if (!xlf) { ServerInstance->SNO->WriteToSnoMask('d',"%s sent me an unknown ADDLINE type (%s).",setter.c_str(),params[0].c_str()); return CMD_FAILURE; } XLine* xl = NULL; try { xl = xlf->Generate(ServerInstance->Time(), ConvToInt(params[4]), params[2], params[5], params[1]); } catch (ModuleException &e) { ServerInstance->SNO->WriteToSnoMask('d',"Unable to ADDLINE type %s from %s: %s", params[0].c_str(), setter.c_str(), e.GetReason().c_str()); return CMD_FAILURE; } xl->SetCreateTime(ConvToInt(params[3])); if (ServerInstance->XLines->AddLine(xl, NULL)) { if (xl->duration) { std::string timestr = InspIRCd::TimeString(xl->expiry); ServerInstance->SNO->WriteToSnoMask('X',"%s added %s%s on %s to expire on %s: %s",setter.c_str(),params[0].c_str(),params[0].length() == 1 ? "-line" : "", params[1].c_str(), timestr.c_str(), params[5].c_str()); } else { ServerInstance->SNO->WriteToSnoMask('X',"%s added permanent %s%s on %s: %s",setter.c_str(),params[0].c_str(),params[0].length() == 1 ? "-line" : "", params[1].c_str(),params[5].c_str()); } TreeServer* remoteserver = TreeServer::Get(usr); if (!remoteserver->IsBursting()) { ServerInstance->XLines->ApplyLines(); } return CMD_SUCCESS; } else { delete xl; return CMD_FAILURE; } }
void OnUserPostNick(User *user, const std::string &oldnick) override { if (!IS_LOCAL(user)) return; if (!MatchOnNickChange) return; XLine *rl = ServerInstance->XLines->MatchesLine("R", user); if (rl) { // Bang! :D rl->Apply(user); } }
bool LocalUser::CheckLines(bool doZline) { const char* check[] = { "G" , "K", (doZline) ? "Z" : NULL, NULL }; if (!this->exempt) { for (int n = 0; check[n]; ++n) { XLine *r = ServerInstance->XLines->MatchesLine(check[n], this); if (r) { r->Apply(this); return true; } } } return false; }
// applies lines, removing clients and changing nicks etc as applicable void XLineManager::ApplyLines() { for (std::vector<User*>::const_iterator u2 = ServerInstance->Users->local_users.begin(); u2 != ServerInstance->Users->local_users.end(); u2++) { User* u = (User*)(*u2); // Don't ban people who are exempt. if (u->exempt) continue; for (std::vector<XLine *>::iterator i = pending_lines.begin(); i != pending_lines.end(); i++) { XLine *x = *i; if (x->Matches(u)) x->Apply(u); } } pending_lines.clear(); }
// applies lines, removing clients and changing nicks etc as applicable void XLineManager::ApplyLines() { LocalUserList::reverse_iterator u2 = ServerInstance->Users->local_users.rbegin(); while (u2 != ServerInstance->Users->local_users.rend()) { LocalUser* u = *u2++; // Don't ban people who are exempt. if (u->exempt) continue; for (std::vector<XLine *>::iterator i = pending_lines.begin(); i != pending_lines.end(); i++) { XLine *x = *i; if (x->Matches(u)) x->Apply(u); } } pending_lines.clear(); }
static void ReadXLine(ServerConfig* conf, const std::string& tag, const std::string& key, XLineFactory* make) { insp::flat_set<std::string> configlines; ConfigTagList tags = conf->ConfTags(tag); for(ConfigIter i = tags.first; i != tags.second; ++i) { ConfigTag* ctag = i->second; std::string mask; if (!ctag->readString(key, mask)) throw CoreException("<"+tag+":"+key+"> missing at " + ctag->getTagLocation()); std::string reason = ctag->getString("reason", "<Config>"); XLine* xl = make->Generate(ServerInstance->Time(), 0, "<Config>", reason, mask); xl->from_config = true; configlines.insert(xl->Displayable()); if (!ServerInstance->XLines->AddLine(xl, NULL)) delete xl; } ServerInstance->XLines->ExpireRemovedConfigLines(make->GetType(), configlines); }
//------------------------------------------------------------------------- // Produce a tab with mean and cov by segment without the maximum score(mdtm and etf only) void getSegmentalMeanCovWithoutMax(Config & config, XList & inputList, unsigned long nbLoc, unsigned long * tarTab, double* meanTab, double * covTab) { XLine *linep; unsigned long cpt=0; unsigned long cpttab=0; double minLLK=config.getParam("minLLK").toDouble(); double maxScore=minLLK; double meanAcc=0.0; double covAcc=0.0; unsigned long idxTar=0; bool verbose=config.existsParam("verbose"); while((linep=inputList.getLine())!=NULL) { double score=linep->getElement(6).toDouble(); if (score>=maxScore) { maxScore=score; // this is the maximum score idxTar=cpt; // index is just the line if (verbose) {cout << "giving highest score to " << linep->getElement(1) << " "<<maxScore << endl;} } meanAcc+=score; // Accumulate mean and cov covAcc+=(score*score); if (cpt%nbLoc==(nbLoc-1)) // when changing segment { tarTab[cpttab]=idxTar; meanAcc-=maxScore; //remove max from Stats covAcc-=(maxScore*maxScore); meanTab[cpttab]=meanAcc; covTab[cpttab]=covAcc; if (verbose) {cout << linep->getElement(1) << " max score: "<<maxScore <<"idx: "<<idxTar<<"cpt: "<<cpt<< " meanA: "<<meanAcc<<" covA: "<<covAcc<<endl;} cpttab++; maxScore=minLLK; meanAcc=0.0; covAcc=0.0; } cpt++; } }
void SendAkill(User *u, XLine *x) override { if (x->IsRegex() || x->HasNickOrReal()) { if (!u) { /* * No user (this akill was just added), and contains nick and/or realname. * Find users that match and ban them. */ for (user_map::const_iterator it = UserListByNick.begin(); it != UserListByNick.end(); ++it) if (x->manager->Check(it->second, x)) this->SendAkill(it->second, x); return; } XLine *old = x; if (old->manager->HasEntry("*@" + u->host)) return; /* We can't akill x as it has a nick and/or realname included, so create a new akill for *@host */ XLine *xl = new XLine("*@" + u->host, old->GetBy(), old->GetExpires(), old->GetReason(), old->id); old->manager->AddXLine(xl); x = xl; Log(Config->GetClient("OperServ"), "akill") << "AKILL: Added an akill for " << x->GetMask() << " because " << u->GetMask() << "#" << u->realname << " matches " << old->GetMask(); } /* Calculate the time left before this would expire, capping it at 2 days */ time_t timeleft = x->GetExpires() - Anope::CurTime; if (timeleft > 172800 || !x->GetExpires()) timeleft = 172800; UplinkSocket::Message(Config->GetClient("OperServ")) << "KLINE * " << timeleft << " " << x->GetUser() << " " << x->GetHost() << " :" << x->GetReason(); }
void FactorAnalysisStat::estimateAndInverseLUnThreaded(){ if (verbose) cout << "(FactorAnalysisStat) Inverse L Matrix ... "<<endl; unsigned long mk; DoubleSquareMatrix L(_rang); L.setAllValues(0.0); RealVector <double> AUX; AUX.setSize(_rang); unsigned long sent=0; XLine *pline; String *pFile; fileList.rewind(); double *N_h=_matN_h.getArray(); double *U=_matU.getArray(); double *LV=L.getArray(); double *super_invvar=_super_invvar.getArray(); while((pline=fileList.getLine())!=NULL) { while((pFile=pline->getElement())!=NULL) { L.setAllValues(0.0); AUX.setAllValues(0.0); for(unsigned long i=0;i<_rang;i++){ for(unsigned long j=0;j<=i;j++){ for(unsigned long k=0;k<_supervsize;k++){ mk=k/_vsize; LV[i*_rang+j]+=N_h[sent*_mixsize+mk]*super_invvar[k]*U[k*_rang+i]*U[k*_rang+j]; } } } for(unsigned long i=0;i<_rang;i++) for(unsigned long j=i+1;j<_rang;j++) LV[i*_rang+j]=LV[j*_rang+i]; for(unsigned long i=0;i<_rang;i++) LV[i*_rang+i]+=1.0; L.invert(_l_h_inv[sent]); sent++; } } };
void OnLookupComplete(const Query *record) override { if (!user || user->Quitting()) return; const ResourceRecord &ans_record = record->answers[0]; // Replies should be in 127.0.0.0/8 if (ans_record.rdata.find("127.") != 0) return; sockaddrs sresult; sresult.pton(AF_INET, ans_record.rdata); int result = sresult.sa4.sin_addr.s_addr >> 24; Blacklist::Reply *reply = blacklist.Find(result); if (!blacklist.replies.empty() && !reply) return; if (reply && reply->allow_account && user->Account()) return; Anope::string reason = this->blacklist.reason, addr = user->ip.addr(); reason = reason.replace_all_cs("%n", user->nick); reason = reason.replace_all_cs("%u", user->GetIdent()); reason = reason.replace_all_cs("%g", user->realname); reason = reason.replace_all_cs("%h", user->host); reason = reason.replace_all_cs("%i", addr); reason = reason.replace_all_cs("%r", reply ? reply->reason : ""); reason = reason.replace_all_cs("%N", Config->GetBlock("networkinfo")->Get<Anope::string>("networkname")); ServiceBot *OperServ = Config->GetClient("OperServ"); creator->logger.Category("dnsbl").Bot(OperServ).Log(_("{0} ({1}) appears in {2}"), user->GetMask(), addr, this->blacklist.name); XLine *x = Serialize::New<XLine *>(); x->SetMask("*@" + addr); x->SetBy(OperServ ? OperServ->nick : "m_dnsbl"); x->SetCreated(Anope::CurTime); x->SetExpires(Anope::CurTime + this->blacklist.bantime); x->SetReason(reason); x->SetID(XLineManager::GenerateUID()); if (this->add_to_akill && akills) { akills->AddXLine(x); akills->Send(NULL, x); } else { IRCD->Send<messages::Akill>(nullptr, x); delete x; } }
//------------------------------------------------------------------------- // Produce a tab with mean and cov by segment (mdtm and etf only) void getSegmentalMeanCov(XList & inputList, unsigned long nbLoc, double* meanTab, double * covTab) { XLine *linep; unsigned long cpt=0; unsigned long cpttab=0; double meanAcc=0.0; double covAcc=0.0; while((linep=inputList.getLine())!=NULL) { double score=linep->getElement(6).toDouble(); meanAcc+=score; // Accumulate mean and cov covAcc+=(score*score); if (cpt%nbLoc==(nbLoc-1)) // when changing segment { meanTab[cpttab]=meanAcc; covTab[cpttab]=covAcc; cpttab++; meanAcc=0.0; covAcc=0.0; } cpt++; } }
void FactorAnalysisStat::substractSpeakerStats(){ if (verbose) cout <<"(FactorAnalysisStat) Compute and Substract Speaker FA Stats... " << endl; RealVector <double> AUX1;AUX1.setSize(_supervsize); AUX1.setAllValues(0.0); double *aux1=AUX1.getArray(); // Compute Occupations and Statistics unsigned long loc=0; unsigned long sent=0; XLine *pline; String *pFile; fileList.rewind(); double *N_h=_matN_h.getArray(); double *S_X_h=_matS_X_h.getArray(); while((pline=fileList.getLine())!=NULL) { while((pFile=pline->getElement())!=NULL) { this->getMplusDY(AUX1,*pFile); for(unsigned long k=0;k<_mixsize;k++) for (unsigned long i=0;i<_vsize;i++) S_X_h[sent*_supervsize+(k*_vsize+i)]-= N_h[sent*_mixsize+k]*aux1[i+k*_vsize]; sent++; } loc++; } if (verboseLevel >1) cout << "(FactorAnalysisStat) Done "<<endl; };
void bahamut::senders::Akill::Send(User* u, XLine* x) { if (x->IsRegex() || x->HasNickOrReal()) { if (!u) { /* No user (this akill was just added), and contains nick and/or realname. Find users that match and ban them */ for (user_map::const_iterator it = UserListByNick.begin(); it != UserListByNick.end(); ++it) if (x->GetManager()->Check(it->second, x)) this->Send(it->second, x); return; } XLine *old = x; if (old->GetManager()->HasEntry("*@" + u->host)) return; /* We can't akill x as it has a nick and/or realname included, so create a new akill for *@host */ x = Serialize::New<XLine *>(); x->SetMask("*@" + u->host); x->SetBy(old->GetBy()); x->SetExpires(old->GetExpires()); x->SetReason(old->GetReason()); x->SetID(old->GetID()); old->GetManager()->AddXLine(x); Log(Config->GetClient("OperServ"), "akill") << "AKILL: Added an akill for " << x->GetMask() << " because " << u->GetMask() << "#" << u->realname << " matches " << old->GetMask(); } /* ZLine if we can instead */ if (x->GetUser() == "*") { cidr a(x->GetHost()); if (a.valid()) { IRCD->Send<messages::SZLine>(u, x); return; } } // Calculate the time left before this would expire, capping it at 2 days time_t timeleft = x->GetExpires() - Anope::CurTime; if (timeleft > 172800) timeleft = 172800; Uplink::Send("AKILL", x->GetHost(), x->GetUser(), timeleft, x->GetBy(), Anope::CurTime, x->GetReason()); }
//------------------------------------------------------------------------- int labelNGram(Config& config) { if (config.existsParam("debug"))debug=true; else debug=false; if (config.existsParam("verbose"))verbose=true; else verbose=false; String extOutputLabel=".sym.lbl"; // the extension of the output files if (config.existsParam("saveLabelFileExtension")) extOutputLabel=config.getParam("saveLabelFileExtension"); String pathOutput="./"; // the path of the output files if (config.existsParam("labelOutputPath")) pathOutput=config.getParam("labelOutputPath"); String extSymbol=".sym"; // the extension of the symbol files if (config.existsParam("symbolFileExtension")) extSymbol=config.getParam("symbolFileExtension"); String pathSymbol="./"; if (config.existsParam("symbolPath")) pathSymbol=config.getParam("symbolPath"); String formatSymbol="ascii"; if (config.existsParam("symbolFormat")) pathSymbol=config.getParam("symbolFormat"); String NGramFilename=config.getParam("NGramFilename"); unsigned long NGramOrder=3; if (config.existsParam("NGramOrder")) NGramOrder=config.getParam("NGramOrder").toLong(); unsigned long NGramSelected=16; if (config.existsParam("NGramSelected")) NGramSelected=config.getParam("NGramSelected").toLong(); NGram NGramTable(NGramOrder,NGramSelected); NGramTable.load(NGramFilename,config); // Load the NGRAM table, selecting the NGramSelected first String inputFilename=config.getParam("inputFilename"); String labelSelectedFrames=config.getParam("labelSelectedFrames"); XLine inputFileList; try{ if (inputFilename.endsWith(".lst")){ // input is file containing a list of filenames XList tmp(inputFilename,config); inputFileList=tmp.getAllElements(); } else inputFileList.addElement(inputFilename); // a single filename String *p; while ((p=inputFileList.getElement())){ String& filename=*p; if (verbose) cout <<"labelNGram file["<<filename<<"] Table["<<NGramFilename<<"] Order["<<NGramOrder<<"] Selected["<<NGramSelected<<"]"<<endl; SegServer segServer; LabelServer labelServer; loadClusterFile(filename,segServer,labelServer,config); long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the selected cluster if (codeSelectedFrame==-1){ // No data for this model !!!!!!!!!!!!!! cout << " WARNING - NO DATA with the label["<<labelSelectedFrames<<"] in file ["<<filename<<"]"<<endl; exit(0); } SegCluster& cluster=segServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments ULongVector tabS; unsigned long nbSym=loadSymbol(pathSymbol+filename+extSymbol,formatSymbol,tabS,config); // Read the stream of symbols SegServer segServerOutput; SegCluster& clusterOut=segServerOutput.createCluster(0,labelSelectedFrames,cluster.sourceName()); // computeLabelNGram(NGramTable,cluster,clusterOut,tabS,nbSym); // if (verbose){ cout <<"File["<<filename<<"]" <<endl; cout << "Output the new label file in ["<<pathOutput+filename+extOutputLabel <<"]"<<endl; } outputLabelFile(clusterOut,pathOutput+filename+extOutputLabel,config); } // end file loop } // fin try catch (Exception& e) { cout << e.toString().c_str() << endl; } return 0; }
int saveApost(Config &config) { bool writeAllFeature=true; // Output a vector for all input vectors (selected and not selected vectors) - DEFAULT=on if (config.existsParam("writeAllFeatures")) writeAllFeature=config.getParam("writeAllFeatures").toBool(); // Define if all the feature (selected or not) should be written String modelname = config.getParam("inputModelFilename"); String inputFeatureFileName =config.getParam("inputFeatureFilename"); // input feature - could be a simple feature file or a list of filenames XLine inputFeatureFileNameList; // The (feature) input filename list if (inputFeatureFileName.endsWith(".lst")){ // If the file parameter is the name of a XList file XList inputFileNameXList(inputFeatureFileName,config); // Read the filename list file inputFeatureFileNameList=inputFileNameXList.getAllElements(); // And put the filename in a list if the file is a list of feature filenames } else { // It was a simple feature file and not a filename list inputFeatureFileNameList.addElement(inputFeatureFileName); // add the filename in the list } try{ // read UBM MixtureServer _ms(config); StatServer _ss(config); _ms.loadMixtureGD(config.getParam("inputWorldFilename")); MixtureGD & UBM=_ms.getMixtureGD((unsigned long) 0); MixtureGDStat &acc=_ss.createAndStoreMixtureStat(UBM); unsigned long _vsize=UBM.getVectSize(); unsigned long _mixsize=UBM.getDistribCount(); // Loop over the list of feature files String *file; String labelSelectedFrames; unsigned long codeSelectedFrame; while ((file=inputFeatureFileNameList.getElement())!= NULL){ String & featureFilename=(*file); FeatureServer fs(config,featureFilename); FeatureServer fs_out(config,featureFilename); SegServer segmentsServer; LabelServer labelServer; initializeClusters(featureFilename,segmentsServer,labelServer,config); verifyClusterFile(segmentsServer,fs,config); labelSelectedFrames=config.getParam("labelSelectedFrames"); codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Compute Occupations and Statistics acc.resetOcc(); Seg *seg; selectedSegments.rewind(); String currentSource=""; while((seg=selectedSegments.getSeg())!=NULL){ unsigned long begin=seg->begin()+fs.getFirstFeatureIndexOfASource(seg->sourceName()); // Idx of the first frame of the current file in the feature server if (currentSource!=seg->sourceName()) { currentSource=seg->sourceName(); if (verbose)cout << "Processing speaker["<<currentSource<<"]"<< endl; } fs.seekFeature(begin); Feature f; for (unsigned long idxFrame=0;idxFrame<seg->length();idxFrame++){ fs.readFeature(f); acc.computeAndAccumulateOcc(f); RealVector <double> aPost=acc.getOccVect(); Feature tmpF; for(unsigned long k=0;k<_mixsize;k++) { tmpF[k]=aPost[k]; } fs_out.addFeature(f); } } // Writing apost probabilities to file cout << "Writing to: " << featureFilename << endl; FeatureFileWriter w(featureFilename, config); // build a featurefile writer to output the features (real features) SegServer fakeSegServer; if (writeAllFeature) { // Output all the features- feature count id the same SegServer fakeSegServer; // Create a new fake segment server fakeSegServer.createCluster(0); // Create a new cluster SegCluster& fakeSeg=fakeSegServer.getCluster(0); // Get the cluster fakeSeg.add(fakeSegServer.createSeg(0,fs_out.getFeatureCount(),codeSelectedFrame, labelSelectedFrames,featureFilename)); // Add a segment with all the features outputFeatureFile(config,fs_out,fakeSeg,w); // output all the features - giving the same file length } else outputFeatureFile(config,fs_out,selectedSegments, w); // Output only the selected features - giving a shorter output } } catch (Exception& e){cout << e.toString().c_str() << endl;} return 0; }
// Training of client Speakers // Input: Xlist Format: ID_Client Seg1 Seg2 .. // Output: ALIZE_MixtureServer (binaire) + GMM / Client (binary) int TrainTarget(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); String inputWorldFilename = config.getParam("inputWorldFilename"); String outputSERVERFilename = ""; if (config.existsParam("mixtureServer")) outputSERVERFilename =config.getParam("mixtureServer"); bool initByClient=false; // In this case, the init model is read from the file if (config.existsParam("initByClient")) initByClient=config.getParam("initByClient").toBool(); bool saveEmptyModel=false; if (config.existsParam("saveEmptyModel")) saveEmptyModel=config.getParam("saveEmptyModel").toBool(); // label for selected frames - Only the frames associated with this label, in the label files, will be used bool fixedLabelSelectedFrame=true; String labelSelectedFrames; if (config.existsParam("useIdForSelectedFrame")) // the ID of each speaker is used as labelSelectedFrame ? fixedLabelSelectedFrame=(config.getParam("useIdForSelectedFrame").toBool()==false); if (fixedLabelSelectedFrame) // the label is decided by the command line and is unique for the run labelSelectedFrames=config.getParam("labelSelectedFrames"); bool modelData=false; if (config.existsParam("useModelData")) modelData=config.getParam("useModelData").toBool(); String initModelS=inputWorldFilename; if (modelData) if (config.existsParam("initModel")) initModelS=config.getParam("initModel"); // Use a specific model for Em init bool outputAdaptParam=false; if (config.existsParam("superVector")) outputAdaptParam=true; bool NAP=false; Matrix <double> ChannelMatrix; if (config.existsParam("NAP")) { if (verbose) cout<< "Removing channel effect with NAP from " << config.getParam("NAP") << " of size: ["; NAP=true; // enable NAP ChannelMatrix.load(config.getParam("NAP"),config); //get Channel Matrix from args and load in a Matrix object if (verbose) cout << ChannelMatrix.rows() << "," <<ChannelMatrix.cols() << "]" << endl; } bool saveCompleteServer=false; try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine * linep; inputClientList.getLine(0); MixtureServer ms(config); StatServer ss(config, ms); if (verbose) cout << "TrainTarget - Load world model [" << inputWorldFilename<<"]"<<endl; MixtureGD& world = ms.loadMixtureGD(inputWorldFilename); MixtureGD& initModel =ms.loadMixtureGD(initModelS); if (verbose) cout <<"Use["<<initModelS<<"] for initializing EM"<<endl; // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String *id=linep->getElement(); // Get the Client ID (id) XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) if (verbose) cout << "Train model ["<<*id<<"]"<<endl; if (!fixedLabelSelectedFrame){ // the ID is used as label for selecting the frame labelSelectedFrames=*id; if (verbose) cout <<*id<<" is used for label selected frames"<<endl; } FeatureServer fs(config,featureFileListp); // Reading the features (from several files) SegServer segmentsServer; // Create the segment server for managing the segments/clusters LabelServer labelServer; // Create the lable server, for indexing the segments/clusters initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file verifyClusterFile(segmentsServer,fs,config); // Verify if the segments ending before the end of the feature files... MixtureGD & adaptedMixture = ms.duplicateMixture(world,DUPL_DISTRIB); // Creating final as a copy of the world model MixtureGD & clientMixture= ms.duplicateMixture(world,DUPL_DISTRIB); if (initByClient){ // During trainig data statistic estimation by EM, clientMixture= ms.loadMixtureGD(*id); // the client model is used for initalization adaptedMixture=clientMixture; } long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the cluster with in interest audio segments if (codeSelectedFrame==-1){ // No data for this model !!!!!!!!!!!!!! cout << " WARNING - NO DATA FOR TRAINING ["<<*id<<"]"; if (saveEmptyModel){ cout <<" World model is returned"<<endl; // In this case, the client model is the world model if (verbose) cout << "Save client model ["<<*id<<"]" << endl; adaptedMixture.save(*id, config); // Save the client model } } else{ SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments if (!initByClient) ms.setMixtureId(clientMixture,*id); // Set the client model Id if (modelData) modelBasedadaptModel(config,ss,ms,fs,selectedSegments,world,clientMixture,initModel); // EM algo with MAP criterion else adaptModel(config,ss,ms,fs,selectedSegments,world,clientMixture); // EM algo with MAP criterion if (NAP) { if (verbose) cout << "NAP on SVs" << endl; computeNap(clientMixture,ChannelMatrix); } if (outputAdaptParam) { RealVector<double> v; getSuperVector(v,world,clientMixture,config); String out=config.getParam("saveVectorFilesPath")+*id+config.getParam("vectorFilesExtension"); Matrix <double> vv=(Matrix<double>)v; vv.save(out,config); } if (!outputAdaptParam) { if (verbose) cout << "Save client model ["<<*id<<"]" << endl; clientMixture.save(*id, config); // Save the client model } if (!saveCompleteServer){ long tid=ms.getMixtureIndex(*id); // TO BE SUPPRESSED BY ms.deleteMixtures(tid,tid); // ADDING a delete on a mixture pointor ms.deleteUnusedDistribs(); } } } // end of the the target loop // Save the complete mixture server // TODO } // fin try catch (Exception& e) { cout << e.toString().c_str() << endl; } return 0; }
// Training of client Speakers // The same than TrainTarget but train simultaneoulsy 1 model for each cluster (set of segments with the same label) // found in the input files labels. // One option in order to save the n models as a modification of the world model - save disk space int TrainTargetByLabel(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); String inputWorldFilename = config.getParam("inputWorldFilename"); String outputSERVERFilename = config.getParam("mixtureServer"); // label for selected frames - Only the frames associated with this label, in the label files, will be used //bool fixedLabelSelectedFrame; bool initByClient=false; bool aprioriWorld=true; if (config.existsParam("initByClient")) initByClient=true; if (config.existsParam("aprioriClient")){ aprioriWorld=false; initByClient=true; } bool saveCompleteServer=false; bool outputAdaptParam=false; if (config.existsParam("outputAdaptParam")) outputAdaptParam=config.getParam("outputAdaptParam").toBool(); try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine *linep; inputClientList.getLine(0); MixtureServer ms(config); StatServer ss(config, ms); if (verbose) cout << "TrainTarget - by label opption - Load world model [" << inputWorldFilename<<"]"<<endl; MixtureGD& world = ms.loadMixtureGD(inputWorldFilename); // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String clientId=(*linep->getElement()); // Get the Client ID (clientId) XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) FeatureServer fs(config,featureFileListp); // Reading the features (from several files) if (verbose) cout << "Train label models for client ["<<clientId<<"]"<<endl; MixtureGD &clientGModel=ms.createMixtureGD(); if (initByClient) { if (verbose) cout << "Load client model [" << clientId <<"]"<<endl; clientGModel = ms.loadMixtureGD(clientId); //not necessary to load client model } SegServer segmentsServer; // Create the segment server for managing the segments/clusters LabelServer labelServer; // Create the lable server, for indexing the segments/clusters initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file verifyClusterFile(segmentsServer,fs,config); // Verify if the segments ending before the end of the feature files... for (unsigned long codeSelectedFrame=0;codeSelectedFrame<segmentsServer.getClusterCount();codeSelectedFrame++){ // For each cluster String clientIdByLabel=clientId+"_"+labelServer.getLabel(codeSelectedFrame).getString(); // Build the model name for the client and the label if (verbose) cout << "Train labeldependent model ["<<clientIdByLabel<<"]"<<endl; SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments MixtureGD & clientMixture = ms.duplicateMixture(world,DUPL_DISTRIB); // Creating clientMixture as a copy of the world model ms.setMixtureId(clientMixture,clientIdByLabel); // Set the client model Id if (initByClient) // During trainig data statistic estimation by EM, clientMixture=clientGModel; // the global client model is used for initalization if (aprioriWorld) // EM algo with MAP criterion adaptModel(config,ss,ms,fs,selectedSegments,world,clientMixture); // A priori info is the world model else adaptModel(config,ss,ms,fs,selectedSegments,clientGModel,clientMixture);// A priori info is the client model-by default initByClient is also set if (!outputAdaptParam) { if (verbose) cout << "Save client model ["<<clientIdByLabel<<"]" << endl; clientMixture.save(clientIdByLabel, config); // Save the client model } if (!saveCompleteServer){ long tid=ms.getMixtureIndex(clientIdByLabel); // TO BE SUPPRESSED BY ms.deleteMixtures(tid,tid); // ADDING a delete on a mixture pointor ms.deleteUnusedDistribs(); } } if (!saveCompleteServer){ long tid=ms.getMixtureIndex(clientId); // TO BE SUPPRESSED BY ms.deleteMixtures(tid,tid); // ADDING a delete on a mixture pointor ms.deleteUnusedDistribs(); } // end of the the label loop fr a speaker } // end of the the target loop // Save the complete mixture server // TODO } // fin try catch (Exception& e) { cout << e.toString().c_str() << endl; } return 0; }
//----------------------------------------------------------------------------------------------------------------------------------------------------------- int TrainTargetFA(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); String inputWorldFilename = config.getParam("inputWorldFilename"); String outputSERVERFilename = ""; if (config.existsParam("mixtureServer")) outputSERVERFilename =config.getParam("mixtureServer"); bool initByClient=false; // In this case, the init model is read from the file if (config.existsParam("initByClient")) initByClient=config.getParam("initByClient").toBool(); bool saveEmptyModel=false; if (config.existsParam("saveEmptyModel")) saveEmptyModel=config.getParam("saveEmptyModel").toBool(); // label for selected frames - Only the frames associated with this label, in the label files, will be used bool fixedLabelSelectedFrame=true; String labelSelectedFrames; if (config.existsParam("useIdForSelectedFrame")) // the ID of each speaker is used as labelSelectedFrame ? fixedLabelSelectedFrame=(config.getParam("useIdForSelectedFrame").toBool()==false); if (fixedLabelSelectedFrame) // the label is decided by the command line and is unique for the run labelSelectedFrames=config.getParam("labelSelectedFrames"); bool modelData=false; if (config.existsParam("useModelData")) modelData=config.getParam("useModelData").toBool(); String initModelS=inputWorldFilename; if (modelData) if (config.existsParam("initModel")) initModelS=config.getParam("initModel"); // Use a specific model for Em init bool outputAdaptParam=false; if (config.existsParam("superVectors")) outputAdaptParam=true; Matrix <double> ChannelMatrix; if (verbose) cout<< "EigenMAP and Eigenchannel with [" << config.getParam("initChannelMatrix") << "] of size: ["; ChannelMatrix.load(config.getParam("initChannelMatrix"),config); //get Channel Matrix from args and load in a Matrix object if (verbose) cout << ChannelMatrix.rows() << "," <<ChannelMatrix.cols() << "]" << endl; bool varAdapt=false; if (config.existsParam("FAVarAdapt")) varAdapt=true; bool saveCompleteServer=false; try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine * linep; inputClientList.getLine(0); MixtureServer ms(config); StatServer ss(config, ms); if (verbose) cout << "(TrainTarget) Factor Analysis - Load world model [" << inputWorldFilename<<"]"<<endl; MixtureGD& world = ms.loadMixtureGD(inputWorldFilename); if (verbose) cout <<"(TrainTarget) Use["<<initModelS<<"] for initializing EM"<<endl; // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String *id=linep->getElement(); // Get the Client ID (id) XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) if (verbose) cout << "(TrainTarget) Train model ["<<*id<<"]"<<endl; FeatureServer fs(config,featureFileListp); // Reading the features (from several files) SegServer segmentsServer; // Create the segment server for managing the segments/clusters LabelServer labelServer; // Create the lable server, for indexing the segments/clusters initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file verifyClusterFile(segmentsServer,fs,config); // Verify if the segments ending before the end of the feature files... MixtureGD & adaptedMixture = ms.duplicateMixture(world,DUPL_DISTRIB); // Creating final as a copy of the world model MixtureGD & clientMixture= ms.duplicateMixture(world,DUPL_DISTRIB); long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the cluster with in interest audio segments if (codeSelectedFrame==-1){ // No data for this model !!!!!!!!!!!!!! cout << " WARNING - NO DATA FOR TRAINING ["<<*id<<"]"; if (saveEmptyModel){ cout <<" World model is returned"<<endl; // In this case, the client model is the world model if (verbose) cout << "Save client model ["<<*id<<"]" << endl; adaptedMixture.save(*id, config); // Save the client model } } else{ SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments /// **** Factor Analysis Stuff XList faNdx; faNdx.addLine()=featureFileListp; FactorAnalysisStat FA(faNdx,fs,config); // give all features to FA stats //FA.computeAndAccumulateGeneralFAStats(selectedSegments,fs,config); for(int i=0;i<config.getParam("nbTrainIt").toLong();i++){ if (verbose) cout << "------ Iteration ["<<i<<"] ------"<<endl; FA.computeAndAccumulateGeneralFAStats(selectedSegments,fs,config); /*if (!varAdapt) FA.getTrueSpeakerModel(clientMixture,linep->getElement(1)); else FA.getFactorAnalysisModel(clientMixture,linep->getElement(1)); if (verbose) cout << "LLK for model["<<*id<<"] at it["<<i-1<<"]="<<FA.getLLK(selectedSegments,clientMixture,fs,config) << endl; */ FA.estimateAndInverseL(config); FA.substractSpeakerStats(); FA.getXEstimate(); FA.substractChannelStats(); FA.getYEstimate(); } MixtureGD & sessionMixture= ms.duplicateMixture(world,DUPL_DISTRIB); bool saveSessionModel=false; if (config.existsParam("saveSessionModel")) saveSessionModel=true; if (saveSessionModel) FA.getSessionModel(sessionMixture,linep->getElement(1)); if (!varAdapt) FA.getTrueSpeakerModel(clientMixture,linep->getElement(1)); // basically compute M_s_h=M+Dy_s and get a model else FA.getFactorAnalysisModel(clientMixture,linep->getElement(1)); // get FA variance adapted model if (verbose) cout << "Final LLK for model["<<*id<<"]="<<FA.getLLK(selectedSegments,clientMixture,fs,config) << endl; /// **** End of FA if (!outputAdaptParam) { if (verbose) cout << "Save client model ["<<*id<<"]" << endl; clientMixture.save(*id, config); // Save the client model if (saveSessionModel) { String sessionfile=*id+".session"; if (verbose) cout << "Save session model ["<<sessionfile<<"]" << endl; sessionMixture.save(sessionfile,config); } } if (!saveCompleteServer){ long tid=ms.getMixtureIndex(*id); // TO BE SUPPRESSED BY ms.deleteMixtures(tid,tid); // ADDING a delete on a mixture pointor ms.deleteUnusedDistribs(); } } } } // fin try catch (Exception& e) {cout << e.toString().c_str() << endl;} return 0; }
//----------------------------------------------------------------------------------------------------------------------------------------------------------- int TrainTargetJFA(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); String inputWorldFilename = config.getParam("inputWorldFilename"); String outputSERVERFilename = ""; if (config.existsParam("mixtureServer")) outputSERVERFilename =config.getParam("mixtureServer"); bool initByClient=false; // In this case, the init model is read from the file if (config.existsParam("initByClient")) initByClient=config.getParam("initByClient").toBool(); bool saveEmptyModel=false; if (config.existsParam("saveEmptyModel")) saveEmptyModel=config.getParam("saveEmptyModel").toBool(); // label for selected frames - Only the frames associated with this label, in the label files, will be used bool fixedLabelSelectedFrame=true; String labelSelectedFrames; if (config.existsParam("useIdForSelectedFrame")) // the ID of each speaker is used as labelSelectedFrame ? fixedLabelSelectedFrame=(config.getParam("useIdForSelectedFrame").toBool()==false); if (fixedLabelSelectedFrame) // the label is decided by the command line and is unique for the run labelSelectedFrames=config.getParam("labelSelectedFrames"); bool modelData=false; if (config.existsParam("useModelData")) modelData=config.getParam("useModelData").toBool(); String initModelS=inputWorldFilename; if (modelData) if (config.existsParam("initModel")) initModelS=config.getParam("initModel"); // Use a specific model for Em init bool outputAdaptParam=false; if (config.existsParam("superVectors")) outputAdaptParam=true; try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine * linep; inputClientList.getLine(0); MixtureServer ms(config); StatServer ss(config, ms); if (verbose) cout << "(TrainTarget) Joint Factor Analysis - Load world model [" << inputWorldFilename<<"]"<<endl; MixtureGD& world = ms.loadMixtureGD(inputWorldFilename); if (verbose) cout <<"(TrainTarget) Use["<<initModelS<<"] for initializing EM"<<endl; //LOAD JFA MAtrices Matrix<double> U, V; DoubleVector D; //Initialise EC matrix if(config.existsParam("eigenChannelMatrix")){ String uName = config.getParam("matrixFilesPath") + config.getParam("eigenChannelMatrix") + config.getParam("loadMatrixFilesExtension"); U.load (uName, config); if (verboseLevel >=1) cout << "(TrainTargetJFA) Init EC matrix from "<< config.getParam("eigenChannelMatrix") <<" from EigenChannel Matrix: "<<", rank: ["<<U.rows() << "] sv size: [" << U.cols() <<"]"<<endl; } else{ unsigned long sS = world.getVectSize() * world.getDistribCount(); U.setDimensions(1,sS); U.setAllValues(0.0); if (verboseLevel >=1) cout << "(TrainTargetJFA) Init EC matrix to 0"<<endl; } //Initialise EV matrix if(config.existsParam("eigenVoiceMatrix")){ String vName = config.getParam("matrixFilesPath") + config.getParam("eigenVoiceMatrix") + config.getParam("loadMatrixFilesExtension"); V.load (vName, config); if (verboseLevel >=1) cout << "(TrainTargetJFA) Init EV matrix from "<< config.getParam("eigenVoiceMatrix") <<" from EigenVoice Matrix: "<<", rank: ["<<V.rows() << "] sv size: [" << V.cols() <<"]"<<endl; } else{ unsigned long sS = world.getVectSize() * world.getDistribCount(); V.setDimensions(1,sS); V.setAllValues(0.0); if (verboseLevel >=1) cout << "(TrainTargetJFA) Init EV matrix to 0"<<endl; } //Initialise D matrix if(config.existsParam("DMatrix")){ String dName = config.getParam("matrixFilesPath") + config.getParam("DMatrix") + config.getParam("loadMatrixFilesExtension"); Matrix<double> tmpD(dName, config); if( (tmpD.rows() != 1) || ( tmpD.cols() != world.getVectSize()*world.getDistribCount() ) ){ throw Exception("Incorrect dimension of D Matrix",__FILE__,__LINE__); } else{ D.setSize(world.getVectSize()*world.getDistribCount()); D.setAllValues(0.0); for(unsigned long i=0; i<world.getVectSize()*world.getDistribCount(); i++){ D[i] = tmpD(0,i); } if (verboseLevel >=1) cout << "(TrainTargetJFA) Init D matrix from "<<config.getParam("DMatrix")<<endl; } } else{ unsigned long sS = world.getVectSize() * world.getDistribCount(); D.setSize(sS); D.setAllValues(0.0); if (verboseLevel >1) cout << "(TrainTargetJFA) Init D matrix to 0"<<endl; } // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String *id=linep->getElement(); // Get the Client ID (id) XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) if (verbose) cout << "(TrainTarget) Train model ["<<*id<<"]"<<endl; XList ndx; ndx.addLine() = featureFileListp; JFAAcc jfaAcc(ndx,config,"TrainTarget"); //Load V, U and D from existing matrices. jfaAcc.loadEV(V, config); jfaAcc.loadEC(U, config); jfaAcc.loadD(D); //Initialize VU matrix jfaAcc.initVU(); FeatureServer fs(config,featureFileListp); // Reading the features (from several files) SegServer segmentsServer; // Create the segment server for managing the segments/clusters LabelServer labelServer; // Create the lable server, for indexing the segments/clusters initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file verifyClusterFile(segmentsServer,fs,config); // Verify if the segments ending before the end of the feature files... MixtureGD & adaptedMixture = ms.duplicateMixture(world,DUPL_DISTRIB); // Creating final as a copy of the world model MixtureGD & clientMixture= ms.duplicateMixture(world,DUPL_DISTRIB); long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the cluster with in interest audio segments if (codeSelectedFrame==-1){ // No data for this model !!!!!!!!!!!!!! cout << " WARNING - NO DATA FOR TRAINING ["<<*id<<"]"; if (saveEmptyModel){ cout <<" World model is returned"<<endl; // In this case, the client model is the world model if (verbose) cout << "Save client model ["<<*id<<"]" << endl; adaptedMixture.save(*id, config); // Save the client model } } else{ SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments //Compute the JFA statistics jfaAcc.computeAndAccumulateJFAStat(selectedSegments,fs,config); //Estimate X and Y in one time for each speaker jfaAcc.storeAccs(); jfaAcc.estimateVUEVUT(config); jfaAcc.estimateAndInverseL_VU(config); jfaAcc.substractMplusDZ(config); jfaAcc.estimateYX(); //Reinitialise the accumulators jfaAcc.resetTmpAcc(); jfaAcc.restoreAccs(); //Split X and Y estimates jfaAcc.splitYX(); //Substract speaker and channel statistics M + VUYX jfaAcc.substractMplusVUYX(); //Estimate Z for each speaker jfaAcc.estimateZ(); //Reinitialise the accumulators jfaAcc.resetTmpAcc(); jfaAcc.restoreAccs(); bool varAdapt = false; if((config.existsParam("varAdapt")) && ( config.getParam("varAdapt").toBool() )){ varAdapt = true; } DoubleVector clientSV(jfaAcc.getSvSize(), jfaAcc.getSvSize()); clientSV.setSize(jfaAcc.getSvSize()); DoubleVector clientModel(jfaAcc.getSvSize(), jfaAcc.getSvSize()); clientModel.setSize(jfaAcc.getSvSize()); bool saveMixture = true; if((config.existsParam("saveMixture")) && !( config.getParam("saveMixture").toBool() )) saveMixture = false; bool saveSuperVector = true; if((config.existsParam("saveSuperVector")) && !( config.getParam("saveSuperVector").toBool() )) saveSuperVector = false; bool saveX = false; bool saveY = false; bool saveZ = false; if(config.existsParam("saveX")) saveX = config.getParam("saveX").toBool(); if(config.existsParam("saveY")) saveY = config.getParam("saveY").toBool(); if(config.existsParam("saveZ")) saveZ = config.getParam("saveZ").toBool(); String xExtension = ".x"; String yExtension = ".y"; String zExtension = ".z"; if(config.existsParam("xExtension")) xExtension = config.getParam("xExtension"); if(config.existsParam("yExtension")) yExtension = config.getParam("yExtension"); if(config.existsParam("zExtension")) zExtension = config.getParam("zExtension"); jfaAcc.getVYplusDZ(clientSV, 0); jfaAcc.getMplusVYplusDZ(clientModel, 0); //WARNING !!!!! only the SuperVector model is divided by the UBM Co-Variance. for(unsigned long i=0; i<jfaAcc.getSvSize(); i++){ clientSV[i] *= jfaAcc.getUbmInvVar()[i]; } //Create the ClientMixture to save if required if(saveMixture){ svToModel(clientModel, clientMixture); clientMixture.save(*id, config); } if(saveSuperVector){ String svPath=config.getParam("saveVectorFilesPath"); String svExt=config.getParam("vectorFilesExtension"); String svFile=svPath+*id+svExt; ((Matrix<double>)clientSV).save(svFile,config); } String svPath=config.getParam("saveVectorFilesPath"); if(saveX){ String xFile=svPath+*id+xExtension; jfaAcc.saveX(xFile,config); } if(saveY){ String yFile=svPath+*id+yExtension; jfaAcc.saveY(yFile,config); } if(saveZ){ String zFile=svPath+*id+zExtension; jfaAcc.saveZ(zFile,config); } long tid=ms.getMixtureIndex(*id); ms.deleteMixtures(tid,tid); ms.deleteUnusedDistribs(); } } } // fin try catch (Exception& e) {cout << e.toString().c_str() << endl;} return 0; }
//----------------------------------------------------------------------------------------------------------------------------------------------------------- int TrainTargetLFA(Config& config) { String inputClientListFileName = config.getParam("targetIdList"); String inputWorldFilename = config.getParam("inputWorldFilename"); String outputSERVERFilename = ""; if (config.existsParam("mixtureServer")) outputSERVERFilename =config.getParam("mixtureServer"); bool initByClient=false; // In this case, the init model is read from the file if (config.existsParam("initByClient")) initByClient=config.getParam("initByClient").toBool(); bool saveEmptyModel=false; if (config.existsParam("saveEmptyModel")) saveEmptyModel=config.getParam("saveEmptyModel").toBool(); // label for selected frames - Only the frames associated with this label, in the label files, will be used bool fixedLabelSelectedFrame=true; String labelSelectedFrames; if (config.existsParam("useIdForSelectedFrame")) // the ID of each speaker is used as labelSelectedFrame ? fixedLabelSelectedFrame=(config.getParam("useIdForSelectedFrame").toBool()==false); if (fixedLabelSelectedFrame) // the label is decided by the command line and is unique for the run labelSelectedFrames=config.getParam("labelSelectedFrames"); bool modelData=false; if (config.existsParam("useModelData")) modelData=config.getParam("useModelData").toBool(); String initModelS=inputWorldFilename; if (modelData) if (config.existsParam("initModel")) initModelS=config.getParam("initModel"); // Use a specific model for Em init bool outputAdaptParam=false; if (config.existsParam("superVectors")) outputAdaptParam=true; try{ XList inputClientList(inputClientListFileName,config); // read the Id + filenames for each client XLine * linep; inputClientList.getLine(0); MixtureServer ms(config); StatServer ss(config, ms); if (verbose) cout << "(TrainTarget) Latent Factor Analysis - Load world model [" << inputWorldFilename<<"]"<<endl; MixtureGD& world = ms.loadMixtureGD(inputWorldFilename); if (verbose) cout <<"(TrainTarget) Use["<<initModelS<<"] for initializing EM"<<endl; //LOAD JFA MAtrices unsigned long svsize=world.getDistribCount()*world.getVectSize(); Matrix<double> U, V; DoubleVector D(svsize,svsize); //Initialise EC matrix if(config.existsParam("eigenChannelMatrix")){ String uName = config.getParam("matrixFilesPath") + config.getParam("eigenChannelMatrix") + config.getParam("loadMatrixFilesExtension"); U.load (uName, config); if (verboseLevel >=1) cout << "(TrainTargetLFA) Init EC matrix from "<< config.getParam("eigenChannelMatrix") <<" from EigenChannel Matrix: "<<", rank: ["<<U.rows() << "] sv size: [" << U.cols() <<"]"<<endl; } else{ U.setDimensions(1,svsize); U.setAllValues(0.0); if (verboseLevel >1) cout << "(TrainTargetLFA) Init EC matrix to 0"<<endl; } V.setDimensions(1,svsize); V.setAllValues(0.0); if (verboseLevel >=1) cout << "(TrainTargetLFA) Init EV matrix to 0"<<endl; //Initialise the D matrix for MAP adaptation for(unsigned long i=0; i<world.getDistribCount(); i++){ for(unsigned long j = 0; j<world.getVectSize(); j++){ D[i*world.getVectSize()+j] = sqrt(1.0/(world.getDistrib(i).getCovInv(j)*config.getParam("regulationFactor").toDouble())); } } // *********** Target loop ***************** while ((linep=inputClientList.getLine()) != NULL){ // linep gives the XLine with the Id of a given client and the list of files String *id=linep->getElement(); // Get the Client ID (id) XLine featureFileListp=linep->getElements(); // Get the list of feature file for the client (end of the line) if (verbose) cout << "(TrainTargetLFA) Train model ["<<*id<<"]"<<endl; XList ndx; ndx.addLine() = featureFileListp; JFAAcc jfaAcc(ndx,config,"TrainTarget"); //Charger les matrices V, U et D a partir des objets matrice existant. jfaAcc.loadEV(V, config); jfaAcc.loadEC(U, config); jfaAcc.loadD(D); //Initialise VU matrix jfaAcc.initVU(); FeatureServer fs(config,featureFileListp); // Reading the features (from several files) SegServer segmentsServer; // Create the segment server for managing the segments/clusters LabelServer labelServer; // Create the lable server, for indexing the segments/clusters initializeClusters(featureFileListp,segmentsServer,labelServer,config); // Reading the segmentation files for each feature input file verifyClusterFile(segmentsServer,fs,config); // Verify if the segments ending before the end of the feature files... MixtureGD & adaptedMixture = ms.duplicateMixture(world,DUPL_DISTRIB); // Creating final as a copy of the world model MixtureGD & clientMixture= ms.duplicateMixture(world,DUPL_DISTRIB); long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames); // Get the index of the cluster with in interest audio segments if (codeSelectedFrame==-1){ // No data for this model !!!!!!!!!!!!!! cout << " WARNING - NO DATA FOR TRAINING ["<<*id<<"]"; if (saveEmptyModel){ cout <<" World model is returned"<<endl; // In this case, the client model is the world model if (verbose) cout << "Save client model ["<<*id<<"]" << endl; adaptedMixture.save(*id, config); // Save the client model } } else{ SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments //Compute the JFA statistics jfaAcc.computeAndAccumulateJFAStat(selectedSegments,fs,config); //Estimate X and Y in one time for each speaker jfaAcc.storeAccs(); jfaAcc.estimateVUEVUT(config); jfaAcc.estimateAndInverseL_VU(config); jfaAcc.substractMplusDZ(config); jfaAcc.estimateYX(); //Reinitialise the accumulators jfaAcc.resetTmpAcc(); jfaAcc.restoreAccs(); //Split X and Y estimates jfaAcc.splitYX(); //Substract speaker and channel statistics M + VUYX jfaAcc.substractMplusVUYX(); //Estimate Z for each speaker double tau = config.getParam("regulationFactor").toLong(); jfaAcc.estimateZMAP(tau); //Reinitialise the accumulators jfaAcc.resetTmpAcc(); jfaAcc.restoreAccs(); bool varAdapt = false; if((config.existsParam("varAdapt")) && ( config.getParam("varAdapt").toBool() )){ varAdapt = true; } DoubleVector clientModel(jfaAcc.getSvSize(), jfaAcc.getSvSize()); clientModel.setSize(jfaAcc.getSvSize()); jfaAcc.getMplusVYplusDZ(clientModel, 0); //Create the ClientMixture svToModel(clientModel, clientMixture); clientMixture.save(*id, config); long tid=ms.getMixtureIndex(*id); ms.deleteMixtures(tid,tid); ms.deleteUnusedDistribs(); } } } // fin try catch (Exception& e) {cout << e.toString().c_str() << endl;} return 0; }
/* add a client connection to the sockets list */ void UserManager::AddUser(int socket, ListenSocket* via, irc::sockets::sockaddrs* client, irc::sockets::sockaddrs* server) { /* NOTE: Calling this one parameter constructor for User automatically * allocates a new UUID and places it in the hash_map. */ LocalUser* New = NULL; try { New = new LocalUser(socket, client, server); } catch (...) { ServerInstance->Logs->Log("USERS", DEFAULT,"*** WTF *** Duplicated UUID! -- Crack smoking monkeys have been unleashed."); ServerInstance->SNO->WriteToSnoMask('a', "WARNING *** Duplicate UUID allocated!"); return; } UserIOHandler* eh = &New->eh; /* Give each of the modules an attempt to hook the user for I/O */ FOREACH_MOD(I_OnHookIO, OnHookIO(eh, via)); if (eh->GetIOHook()) { try { eh->GetIOHook()->OnStreamSocketAccept(eh, client, server); } catch (CoreException& modexcept) { ServerInstance->Logs->Log("SOCKET", DEBUG,"%s threw an exception: %s", modexcept.GetSource(), modexcept.GetReason()); } } ServerInstance->Logs->Log("USERS", DEBUG,"New user fd: %d", socket); this->unregistered_count++; /* The users default nick is their UUID */ New->nick.assign(New->uuid, 0, ServerInstance->Config->Limits.NickMax); (*(this->clientlist))[New->nick] = New; New->registered = REG_NONE; New->signon = ServerInstance->Time() + ServerInstance->Config->dns_timeout; New->lastping = 1; ServerInstance->Users->AddLocalClone(New); ServerInstance->Users->AddGlobalClone(New); New->localuseriter = this->local_users.insert(local_users.end(), New); if ((this->local_users.size() > ServerInstance->Config->SoftLimit) || (this->local_users.size() >= (unsigned int)ServerInstance->SE->GetMaxFds())) { ServerInstance->SNO->WriteToSnoMask('a', "Warning: softlimit value has been reached: %d clients", ServerInstance->Config->SoftLimit); this->QuitUser(New,"No more connections allowed"); return; } /* * First class check. We do this again in FullConnect after DNS is done, and NICK/USER is recieved. * See my note down there for why this is required. DO NOT REMOVE. :) -- w00t */ New->SetClass(); /* * Check connect class settings and initialise settings into User. * This will be done again after DNS resolution. -- w00t */ New->CheckClass(); if (New->quitting) return; /* * even with bancache, we still have to keep User::exempt current. * besides that, if we get a positive bancache hit, we still won't f**k * them over if they are exempt. -- w00t */ New->exempt = (ServerInstance->XLines->MatchesLine("E",New) != NULL); if (BanCacheHit *b = ServerInstance->BanCache->GetHit(New->GetIPString())) { if (!b->Type.empty() && !New->exempt) { /* user banned */ ServerInstance->Logs->Log("BANCACHE", DEBUG, "BanCache: Positive hit for " + New->GetIPString()); if (!ServerInstance->Config->MoronBanner.empty()) New->WriteServ("NOTICE %s :*** %s", New->nick.c_str(), ServerInstance->Config->MoronBanner.c_str()); this->QuitUser(New, b->Reason); return; } else { ServerInstance->Logs->Log("BANCACHE", DEBUG, "BanCache: Negative hit for " + New->GetIPString()); } } else { if (!New->exempt) { XLine* r = ServerInstance->XLines->MatchesLine("Z",New); if (r) { r->Apply(New); return; } } } if (!ServerInstance->SE->AddFd(eh, FD_WANT_FAST_READ | FD_WANT_EDGE_WRITE)) { ServerInstance->Logs->Log("USERS", DEBUG,"Internal error on new connection"); this->QuitUser(New, "Internal error handling connection"); } /* NOTE: even if dns lookups are *off*, we still need to display this. * BOPM and other stuff requires it. */ New->WriteServ("NOTICE Auth :*** Looking up your hostname..."); if (ServerInstance->Config->RawLog) New->WriteServ("NOTICE Auth :*** Raw I/O logging is enabled on this server. All messages, passwords, and commands are being recorded."); FOREACH_MOD(I_OnSetUserIP,OnSetUserIP(New)); if (New->quitting) return; FOREACH_MOD(I_OnUserInit,OnUserInit(New)); if (ServerInstance->Config->NoUserDns) { New->WriteServ("NOTICE %s :*** Skipping host resolution (disabled by server administrator)", New->nick.c_str()); New->dns_done = true; } else { New->StartDNSLookup(); } }