//__________________________________________________________________________________ _Variable* CheckReceptacleCommandID (_String* name, const long id, bool checkValid, bool isGlobal) { if (checkValid && (!name->IsValidIdentifier())) { WarnError (_String ("'") & *name & "' is not a variable identifier in call to " & _HY_ValidHBLExpressions.RetrieveKeyByPayload(id) & '.'); return nil; } long f = LocateVarByName (*name); if (f<0) { _Variable dummy (*name, isGlobal); f = LocateVarByName (*name); } return FetchVar(f); }
//__________________________________________________________________________________ _Variable* CheckReceptacle (_String* name, _String fID, bool checkValid, bool isGlobal) { if (checkValid && (!name->IsValidIdentifier())) { _String errMsg = *name & " is not a variable identifier in call to " & fID; WarnError (errMsg); return nil; } long f = LocateVarByName (*name); if (f<0) { _Variable dummy (*name, isGlobal); f = LocateVarByName (*name); } return FetchVar(f); }
//__________________________________________________________________________________ void CompileListOfUserExpressions (_SimpleList& varRefs,_List& rec, bool doAll) { rec.Clear(); if (varRefs.lLength == 0) { return; } long i; _SimpleList startVars; _VariableContainer* firstVar = (_VariableContainer*)LocateVar(varRefs.lData[0]); firstVar->ScanAndAttachVariables(); { _AVLList sA (&startVars); if (doAll) { firstVar->ScanForVariables (sA,sA); firstVar->ScanForGVariables (sA,sA); } firstVar->ScanForDVariables (sA,sA); sA.ReorderList (); } if (!doAll) { for (i=startVars.lLength-1; i>=0; i--) { if (firstVar->IsModelVar(i)) { startVars.Delete(i); } } } for (i=0; i<startVars.lLength; i++) { _String thisName (LocateVar(startVars.lData[i])->GetName()->Cut (LocateVar(startVars.lData[i])->GetName()->FindBackwards('.',0,-1),-1)); rec && &thisName; } for (i=varRefs.lLength-1; i>=1; i--) { firstVar = (_VariableContainer*)LocateVar(varRefs.lData[i]); firstVar->ScanAndAttachVariables(); firstVar->MatchParametersToList (rec,doAll); } for (i=rec.lLength-1; i>=0; i--) { _String* thisLine = ((_String*)rec(i)); thisLine->Trim(1,-1); if (doAll) if (LocateVarByName(*thisLine)<0) { *thisLine = _String('!')&*thisLine; } } }
//__________________________________________________________________________________ long DereferenceString (_PMathObj v, _PMathObj context, char reference_type){ if (v && v->ObjectClass () == STRING) { _FString * value = (_FString*)v; _String referencedVariable = *value->theString; if (reference_type == HY_STRING_LOCAL_DEREFERENCE && context) { referencedVariable = AppendContainerName(referencedVariable, (_VariableContainer*)context); } return LocateVarByName(referencedVariable); } return -1; }
//__________________________________________________________________________________ _Variable* CheckReceptacleCommandID (_String* name, const long id, bool checkValid, bool isGlobal, _ExecutionList* context) { if (checkValid && (!name->IsValidIdentifier())) { _String errMsg = _String ("'") & *name & "' is not a valid variable identifier in call to " & _HY_ValidHBLExpressions.RetrieveKeyByPayload(id) & '.'; if (context) { context->ReportAnExecutionError(errMsg); } else { WarnError (errMsg); } return nil; } long f = LocateVarByName (*name); if (f<0) { _Variable dummy (*name, isGlobal); f = LocateVarByName (*name); } return FetchVar(f); }
//__________________________________________________________________________________ _PMathObj FetchObjectFromVariableByType (_String* id, const unsigned long objectClass, long command_id, _String *errMsg) { if (id) { _Variable * v = FetchVar (LocateVarByName (*id)); if (v && (objectClass == HY_ANY_OBJECT || v->ObjectClass () == objectClass)) { return v->Compute(); } if (command_id >= 0 || errMsg) { if (command_id >= 0) { WarnError (_String ("'") & *id & ("' must refer to a ") & FetchObjectNameFromType (objectClass) & " in call to " &_HY_ValidHBLExpressions.RetrieveKeyByPayload(command_id) & '.'); } else { WarnError (errMsg->Replace ("_VAR_NAME_ID_", *id, true)); } } } return nil; }
void TrainModelNN (_String* model, _String* matrix) { _String errMsg; long modelIdx = modelNames.Find(model); _Parameter verbI; checkParameter (VerbosityLevelString, verbI, 0.0); char buffer [128]; if (modelIdx < 0) { errMsg = *model & " did not refer to an existring model"; } else { _Variable* boundsMatrix = FetchVar (LocateVarByName (*matrix)); if (boundsMatrix && (boundsMatrix->ObjectClass() == MATRIX)) { _Matrix * bmatrix = (_Matrix*) boundsMatrix->GetValue (); if (bmatrix->IsAStringMatrix() && (bmatrix->GetVDim () == 3)) { _Variable* modelMatrix = LocateVar (modelMatrixIndices.lData[modelIdx]); _SimpleList modelVariableList; { _AVLList mvla (&modelVariableList); modelMatrix->ScanForVariables (mvla, true); mvla.ReorderList(); } if (bmatrix->GetHDim () == modelVariableList.lLength) { // now map model variables to bounds matrix _SimpleList variableMap; _String *myName; for (long k = 0; k < modelVariableList.lLength; k++) { myName = ((_FString*)bmatrix->GetFormula(k,0)->Compute())->theString; long vID = LocateVarByName (*myName); if (vID < 0) { break; } vID = variableNames.GetXtra (vID); vID = modelVariableList.Find(vID); if (vID < 0) { break; } variableMap << vID; } if (variableMap.lLength == modelVariableList.lLength) { _Matrix vBounds (variableMap.lLength,2, false, true); long k2 = 0; for (; k2 < variableMap.lLength; k2++) { _Parameter lb = ((_FString*)bmatrix->GetFormula(k2,1)->Compute())->theString->toNum(), ub = ((_FString*)bmatrix->GetFormula(k2,2)->Compute())->theString->toNum(); if ( ub>lb || k2) { vBounds.Store (k2,0,lb); vBounds.Store (k2,1,ub); if (ub<=lb && vBounds (k2-1,0) <= vBounds (k2-1,1) && (!CheckEqual(vBounds (k2-1,0),0.0) || !CheckEqual(vBounds (k2-1,1),1.0))) { break; } } } if (k2 == modelVariableList.lLength) { // set up the sampling now _String fName = ProcessLiteralArgument (&ModelNNFile,nil); FILE* nnFile = doFileOpen (fName.getStr(), "w"); if (nnFile) { _Matrix* modelMatrix = (_Matrix*) LocateVar(modelMatrixIndices.lData[modelIdx])->GetValue(); _Parameter mainSteps, checkSteps, errorTerm, loopMax, hiddenNodes, absError, nn1, nn2; long fullDimension = modelMatrix->GetHDim() * modelMatrix->GetVDim(); checkParameter (ModelNNTrainingSteps, mainSteps, 10000.0); checkParameter (ModelNNVerificationSample, checkSteps, 500.0); checkParameter (ModelNNPrecision, errorTerm, 0.01); checkParameter (ModelNNTrainingSteps, loopMax, 10); checkParameter (ModelNNHiddenNodes, hiddenNodes, 5); checkParameter (ModelNNLearningRate, nn1, .3); checkParameter (ModelNNPersistenceRate, nn2, .1); Net** matrixNet = new Net* [fullDimension] ; for (long i = 0; i < fullDimension; i++) { checkPointer (matrixNet [i] = new Net (variableMap.lLength,(long)hiddenNodes,1,errorTerm,nn1,nn2,100,200,true)); //matrixNet[i]->verbose = true; } checkPointer (matrixNet); _List tIn, tOut; FILE* varSamples = doFileOpen ("variableSamples.out", "w"); fprintf (varSamples, "%s" ,LocateVar(modelVariableList.lData[0])->GetName()->getStr()); for (long vc = 1; vc < modelVariableList.lLength; vc++) { fprintf (varSamples, ",%s" ,LocateVar(modelVariableList.lData[variableMap.lData[vc]])->GetName()->getStr()); } fprintf (varSamples, "\n"); for (long itCount = 0; itCount < loopMax; itCount ++) { if (verbI > 5) { snprintf (buffer, sizeof(buffer), "\nNeural Network Pass %ld. Building a training set...\n", itCount); BufferToConsole (buffer); } while (tIn.countitems() < mainSteps) { NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut); } _Matrix inData (mainSteps, variableMap.lLength, false, true); _Parameter *md = inData.theData; for (long matrixC = 0; matrixC < mainSteps; matrixC++) { _Parameter * ed = ((_Matrix*)tIn (matrixC))->theData; fprintf (varSamples, "\n%g",*ed); *md = *ed; ed++; md++; for (long entryC = 1; entryC < variableMap.lLength; entryC++, ed++, md++) { *md = *ed; fprintf (varSamples, ",%g", *md); } } tIn.Clear(); if (verbI > 5) { BufferToConsole ( "Done Building Training Set. Training...\n"); } long lastDone = 0; for (long cellCount = 0; cellCount < fullDimension; cellCount++) { Net* thisCell = matrixNet[cellCount]; _Matrix outVector (mainSteps, 1, false, true); for (long oc = 0; oc < mainSteps; oc++) { outVector.theData[oc] = ((_Matrix*)tOut(oc))->theData[cellCount]; } thisCell->studyAll (inData.theData, outVector.theData, mainSteps); long nowDone = (cellCount+1)*100./fullDimension; if (nowDone > lastDone) { snprintf (buffer, sizeof(buffer),"%ld%% done\n", lastDone = nowDone); BufferToConsole (buffer); } } tOut.Clear(); if (verbI > 5) { BufferToConsole ( "Done Training. Resampling...\n"); } _PMathObj tObj = _Constant(0).Time(); _Parameter time1 = tObj->Value(), time2; while (tIn.countitems() < checkSteps) { NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut); } absError = 0.0; DeleteObject (tObj); tObj = _Constant(0).Time(); time2 = tObj->Value(); if (verbI > 5) { snprintf (buffer, sizeof(buffer),"Done Resampling in %g seconds. Computing Error...\n", time2-time1); BufferToConsole (buffer); } _Parameter maxValT, maxValE; for (long verCount = 0; verCount < checkSteps; verCount++) { _Parameter* inData = ((_Matrix*)tIn(verCount))->theData, * outData = ((_Matrix*)tOut(verCount))->theData; for (long cellCount = 0; cellCount < fullDimension; cellCount++) { Net *thisCell = matrixNet[cellCount]; _Parameter estVal = thisCell->eval(inData)[0], trueVal = outData[cellCount], localError; localError = estVal-trueVal; if (localError < 0) { localError = -localError; } if (absError < localError) { maxValT = trueVal; maxValE = estVal; absError = localError; } } } DeleteObject (tObj); tObj = _Constant(0).Time(); time1 = tObj->Value(); DeleteObject (tObj); if (verbI > 5) { snprintf (buffer, sizeof(buffer), "Done Error Checking in %g seconds. Got max abs error %g on the pair %g %g\n", time1-time2, absError, maxValT, maxValE); BufferToConsole (buffer); } if (absError <= errorTerm) { break; } } if (absError > errorTerm) { ReportWarning (_String("Couldn't achive desired precision in TrainModelNN. Achieved error of ") & absError); } fclose (varSamples); fprintf (nnFile,"{{\n\"%s\"", LocateVar(modelVariableList.lData[0])->GetName()->getStr()); _Matrix newBounds (modelVariableList.lLength, 2, false, true); if (vBounds(0,0)>vBounds(0,1)) { newBounds.Store (variableMap.lData[0],0,0.); newBounds.Store (variableMap.lData[0],1,1.); } else { newBounds.Store (variableMap.lData[0],0,vBounds(0,0)); newBounds.Store (variableMap.lData[0],1,vBounds(0,1)); } for (long varCounter = 1; varCounter < modelVariableList.lLength; varCounter ++) { fprintf (nnFile,",\n\"%s\"", LocateVar(modelVariableList.lData[varCounter])->GetName()->getStr()); if (vBounds(varCounter,0)>vBounds(varCounter,1)) { newBounds.Store (variableMap.lData[varCounter],0,0.); newBounds.Store (variableMap.lData[varCounter],1,1.); } else { newBounds.Store (variableMap.lData[varCounter],0,vBounds(varCounter,0)); newBounds.Store (variableMap.lData[varCounter],1,vBounds(varCounter,1)); } } fprintf (nnFile,"\n}}\n"); newBounds.toFileStr (nnFile); for (long i2 = 0; i2 < fullDimension; i2++) { matrixNet[i2]->save(nnFile); delete matrixNet [i2]; } fclose (nnFile); delete matrixNet; } else { errMsg = _String ("Failed to open ") & fName & " for writing"; } } else { errMsg = _String ("Invalid variable bounds in row ") & (k2+1) & " of the bounds matrix"; } } else { errMsg = *myName & " was not one of the model parameters"; } } else { errMsg = *matrix & " must be a have the same number of rows as the number of model parameters"; } } else { errMsg = *matrix & " must be a string matrix with 3 columns"; } } else { errMsg = *matrix & " was not the identifier of a valid matrix variable"; } } if (errMsg.sLength) { errMsg = errMsg & _String(" in call to TrainModelNN."); WarnError (errMsg); } }
//__________________________________________________________________________________ void mpiNormalLoop (int rank, int size, _String & baseDir) { long senderID = 0; ReportWarning ("[MPI] Entered mpiNormalLoop"); _String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node * resStr = nil; while (theMessage->sLength) { setParameter (mpiNodeID, (_Parameter)rank); setParameter (mpiNodeCount, (_Parameter)size); //ReportWarning (*theMessage); DeleteObject (resStr); resStr = nil; if (theMessage->startswith (mpiLoopSwitchToOptimize) ) { hyphyMPIOptimizerMode = theMessage->Cut(mpiLoopSwitchToOptimize.sLength,-1).toNum(); ReportWarning (_String("[MPI] Switched to mpiOptimizer loop with mode ") & hyphyMPIOptimizerMode); MPISendString (mpiLoopSwitchToOptimize,senderID); mpiOptimizerLoop (rank,size); ReportWarning ("[MPI] Returned from mpiOptimizer loop"); hyphyMPIOptimizerMode = _hyphyLFMPIModeNone; pathNames && & baseDir; } else if ( theMessage->Equal (&mpiLoopSwitchToBGM) ) { ReportWarning ("[MPI] Received signal to switch to mpiBgmLoop"); MPISendString (mpiLoopSwitchToBGM, senderID); // feedback to source to confirm receipt of message mpiBgmLoop (rank, size); ReportWarning ("[MPI] Returned from mpiBgmLoop"); } else { if (theMessage->beginswith ("#NEXUS")) { _String msgCopy (*theMessage); ReportWarning ("[MPI] Received a function to optimize"); ReadDataSetFile (nil,true,theMessage); ReportWarning ("[MPI] Done with the optimization"); _Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN)); if (lfName) { resStr = (_String*)(lfName->Compute()->toStr()); } else { _FString *lfID = (_FString*)FetchObjectFromVariableByType (&lf2SendBack, STRING); if (!lfID) { FlagError (_String("[MPI] Malformed MPI likelihood function optimization request - did not specify the LF name to return in variable ") & lf2SendBack & ".\n\n\n" ); break; } long f = likeFuncNamesList.Find (lfID->theString); if (f<0) { FlagError ("[MPI] Malformed MPI likelihood function optimization request - LF name to return did not refer to a well-defined likelihood function.\n\n\n"); break; } _Parameter pv; checkParameter (shortMPIReturn, pv ,0); resStr = (_String*)checkPointer(new _String (1024L,true)); ((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?_hyphyLFSerializeModeShortMPI:_hyphyLFSerializeModeLongMPI); resStr->Finalize(); } } else { _ExecutionList exL (*theMessage); _PMathObj res = exL.Execute(); resStr = res?(_String*)res->toStr():new _String ("0"); } checkPointer (resStr); MPISendString(*resStr,senderID); _Parameter keepState = 0.0; checkParameter (preserveSlaveNodeState, keepState, 0.0); if (keepState < 0.5) { PurgeAll (true); pathNames && & baseDir; } } DeleteObject (theMessage); theMessage = MPIRecvString (-1,senderID); } /*MPISendString(empty,senderID);*/ DeleteObject (resStr); DeleteObject (theMessage); }
//__________________________________________________________________________________ void mpiNormalLoop (int rank, int size, _String & baseDir) { long senderID = 0; ReportWarning ("Entered mpiNormalLoop"); _String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node * resStr = nil, _bgmSwitch ("_BGM_SWITCH_"), css("_CONTEXT_SWITCH_MPIPARTITIONS_"); while (theMessage->sLength) { setParameter (mpiNodeID, (_Parameter)rank); setParameter (mpiNodeCount, (_Parameter)size); //ReportWarning (*theMessage); DeleteObject (resStr); resStr = nil; if (theMessage->Equal (&css) ) { mpiPartitionOptimizer = true; ReportWarning ("Switched to mpiOptimizer loop"); MPISendString(css,senderID); mpiOptimizerLoop (rank,size); ReportWarning ("Returned from mpiOptimizer loop"); mpiPartitionOptimizer = false; pathNames && & baseDir; } else if ( theMessage->Equal (&_bgmSwitch) ) { ReportWarning ("Received signal to switch to mpiBgmLoop"); MPISendString (_bgmSwitch, senderID); // feedback to source to confirm receipt of message mpiBgmLoop (rank, size); ReportWarning ("Returned from mpiBgmLoop"); } else { if (theMessage->beginswith ("#NEXUS")) { _String msgCopy (*theMessage); ReportWarning ("Received a function to optimize"); ReadDataSetFile (nil,true,theMessage); ReportWarning ("Done with the optimization"); _Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN)); if (lfName) { resStr = (_String*)(lfName->Compute()->toStr()); } else { long f = LocateVarByName (lf2SendBack); if (f>=0) lfName = FetchVar(f); if (!(lfName&&(lfName->ObjectClass()==STRING))) { _String errMsg ("Malformed MPI likelihood function optimization request - missing LF name to return.\n\n\n"); errMsg = errMsg & msgCopy; FlagError (errMsg); break; } f = likeFuncNamesList.Find (((_FString*)lfName->Compute())->theString); if (f<0) { _String errMsg ("Malformed MPI likelihood function optimization request - invalid LF name to return.\n\n\n"); errMsg = errMsg & msgCopy; FlagError (errMsg); break; } _Parameter pv; checkParameter (shortMPIReturn, pv ,0); resStr = new _String (1024L,true); checkPointer (resStr); ((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?5:2); resStr->Finalize(); } } else { _ExecutionList exL (*theMessage); /*printf ("Received:\n %s\n", ((_String*)exL.toStr())->sData);*/ _PMathObj res = exL.Execute(); resStr = res?(_String*)res->toStr():new _String ("0"); } checkPointer (resStr); DeleteObject (theMessage); MPISendString(*resStr,senderID); _Parameter keepState = 0.0; checkParameter (preserveSlaveNodeState, keepState, 0.0); if (keepState < 0.5) { PurgeAll (true); pathNames && & baseDir; } } theMessage = MPIRecvString (-1,senderID); } /*MPISendString(empty,senderID);*/ DeleteObject (resStr); DeleteObject (theMessage); }
void _HYObjectInspector::KillObject (void) { _HYTable* dl = (_HYTable*)GetCellObject (HY_OBJECT_INSPECTOR_TABLE_ROW,0); _HYButtonBar* b1 = (_HYButtonBar*)GetCellObject (0,0); _HYPullDown* p1 = (_HYPullDown*)GetCellObject (0,2); _SimpleList sel; dl->GetSelection (sel); if (sel.lLength) { b1->EnableButton (0,true); long f; switch (p1->GetSelection()) { case 0: { for (f = 0; f<sel.lLength; f+=2) { long treeID = LocateVarByName (*(_String*)dl->GetCellData (0,sel.lData[f]/2)); if (treeID>=0) #ifndef USE_AVL_NAMES treeID = variableReindex.lData[treeID]; #else treeID = variableNames.GetXtra(treeID); #endif if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_BOLD) { _String prompt ("Tree "); prompt = prompt & *LocateVar (treeID)->GetName() & objectInspectorKillPrompt; if (warnTree||(!ProceedPromptWithCheck (prompt,donotWarnAgain,warnTree))) { continue; } } postTreeKillEvent (GetID(), treeID); } break; } case 1: { for (f = 0; f<sel.lLength; f+=2) { long dsID = dataSetNamesList.Find ((_String*)dl->GetCellData (0,sel.lData[f]/2)); if (dsID>=0) { if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_BOLD) { _String prompt ("Dataset "); prompt = prompt & *(_String*)dataSetNamesList(dsID) & objectInspectorKillPrompt; if (!((!warnTree)&&ProceedPromptWithCheck (prompt,donotWarnAgain,warnDS))) { continue; } } postDSKillEvent (GetID(), dsID); } } break; } case 2: { for (f = 0; f<sel.lLength; f+=2) { if (sel.lData[f]/2<modelTemplates.lLength) { _List* modelT = FindModelTemplate((_String*)dl->GetCellData (0,sel.lData[f]/2)); if (modelT) { // template model _String*fileLocation = (_String*)(*modelT)(2), errMsg (*(_String*)dl->GetCellData (0,sel.lData[f]/2)); if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_ITALIC) { errMsg = errMsg & " is a template model. If you wish to delete it, please do so by removing the appropriate file from the \"Substitution Models\" directory."; ProblemReport (errMsg); } else { errMsg = errMsg & " is about to be deleted. This action is NOT undoable."; if (ProceedPrompt (errMsg,(Ptr)this)) { if (remove (fileLocation->sData)) { errMsg = "File delete operation failed."; ProblemReport (errMsg); } else { errMsg = *(_String*)dl->GetCellData (0,sel.lData[f]/2); for (long k=0; k<modelTemplates.lLength; k++) { _List* thisList = (_List*)modelTemplates(k); if (errMsg.Equal((_String*)(*thisList)(0))) { modelTemplates.Delete (k); } } BuildListOfObjects (2); } } } } } } break; } } } }
//__________________________________________________________________________________ void DeleteTreeVariable (_String&name, _SimpleList& parms, bool doDeps) { DeleteTreeVariable(LocateVarByName (name), parms,doDeps); }
//__________________________________________________________________________________ void DeleteVariable (_String&name, bool deleteself) { DeleteVariable(LocateVarByName (name), deleteself); }