//__________________________________________________________________________________
_Variable* CheckReceptacleCommandID (_String* name, const long id, bool checkValid, bool isGlobal)
{
if (checkValid && (!name->IsValidIdentifier())) {
WarnError (_String ("'") & *name & "' is not a variable identifier in call to " & _HY_ValidHBLExpressions.RetrieveKeyByPayload(id) & '.');
return nil;
}
long f = LocateVarByName (*name);
if (f<0) {
_Variable dummy (*name, isGlobal);
f = LocateVarByName (*name);
}
return FetchVar(f);
}
//__________________________________________________________________________________
_Variable* CheckReceptacle (_String* name, _String fID, bool checkValid, bool isGlobal)
{
if (checkValid && (!name->IsValidIdentifier())) {
_String errMsg = *name & " is not a variable identifier in call to " & fID;
WarnError (errMsg);
return nil;
}
long f = LocateVarByName (*name);
if (f<0) {
_Variable dummy (*name, isGlobal);
f = LocateVarByName (*name);
}
return FetchVar(f);
}
//__________________________________________________________________________________
void CompileListOfUserExpressions (_SimpleList& varRefs,_List& rec, bool doAll)
{
rec.Clear();
if (varRefs.lLength == 0) {
return;
}
long i;
_SimpleList startVars;
_VariableContainer* firstVar = (_VariableContainer*)LocateVar(varRefs.lData[0]);
firstVar->ScanAndAttachVariables();
{
_AVLList sA (&startVars);
if (doAll) {
firstVar->ScanForVariables (sA,sA);
firstVar->ScanForGVariables (sA,sA);
}
firstVar->ScanForDVariables (sA,sA);
sA.ReorderList ();
}
if (!doAll) {
for (i=startVars.lLength-1; i>=0; i--) {
if (firstVar->IsModelVar(i)) {
startVars.Delete(i);
}
}
}
for (i=0; i<startVars.lLength; i++) {
_String thisName (LocateVar(startVars.lData[i])->GetName()->Cut
(LocateVar(startVars.lData[i])->GetName()->FindBackwards('.',0,-1),-1));
rec && &thisName;
}
for (i=varRefs.lLength-1; i>=1; i--) {
firstVar = (_VariableContainer*)LocateVar(varRefs.lData[i]);
firstVar->ScanAndAttachVariables();
firstVar->MatchParametersToList (rec,doAll);
}
for (i=rec.lLength-1; i>=0; i--) {
_String* thisLine = ((_String*)rec(i));
thisLine->Trim(1,-1);
if (doAll)
if (LocateVarByName(*thisLine)<0) {
*thisLine = _String('!')&*thisLine;
}
}
}
//__________________________________________________________________________________
long DereferenceString (_PMathObj v, _PMathObj context, char reference_type){
if (v && v->ObjectClass () == STRING) {
_FString * value = (_FString*)v;
_String referencedVariable = *value->theString;
if (reference_type == HY_STRING_LOCAL_DEREFERENCE && context) {
referencedVariable = AppendContainerName(referencedVariable, (_VariableContainer*)context);
}
return LocateVarByName(referencedVariable);
}
return -1;
}
//__________________________________________________________________________________
_Variable* CheckReceptacleCommandID (_String* name, const long id, bool checkValid, bool isGlobal, _ExecutionList* context)
{
if (checkValid && (!name->IsValidIdentifier())) {
_String errMsg = _String ("'") & *name & "' is not a valid variable identifier in call to " & _HY_ValidHBLExpressions.RetrieveKeyByPayload(id) & '.';
if (context) {
context->ReportAnExecutionError(errMsg);
} else {
WarnError (errMsg);
}
return nil;
}
long f = LocateVarByName (*name);
if (f<0) {
_Variable dummy (*name, isGlobal);
f = LocateVarByName (*name);
}
return FetchVar(f);
}
//__________________________________________________________________________________
_PMathObj FetchObjectFromVariableByType (_String* id, const unsigned long objectClass, long command_id, _String *errMsg)
{
if (id) {
_Variable * v = FetchVar (LocateVarByName (*id));
if (v && (objectClass == HY_ANY_OBJECT || v->ObjectClass () == objectClass)) {
return v->Compute();
}
if (command_id >= 0 || errMsg) {
if (command_id >= 0) {
WarnError (_String ("'") & *id & ("' must refer to a ") & FetchObjectNameFromType (objectClass) & " in call to "
&_HY_ValidHBLExpressions.RetrieveKeyByPayload(command_id) & '.');
} else {
WarnError (errMsg->Replace ("_VAR_NAME_ID_", *id, true));
}
}
}
return nil;
}
void TrainModelNN (_String* model, _String* matrix)
{
_String errMsg;
long modelIdx = modelNames.Find(model);
_Parameter verbI;
checkParameter (VerbosityLevelString, verbI, 0.0);
char buffer [128];
if (modelIdx < 0) {
errMsg = *model & " did not refer to an existring model";
} else {
_Variable* boundsMatrix = FetchVar (LocateVarByName (*matrix));
if (boundsMatrix && (boundsMatrix->ObjectClass() == MATRIX)) {
_Matrix * bmatrix = (_Matrix*) boundsMatrix->GetValue ();
if (bmatrix->IsAStringMatrix() && (bmatrix->GetVDim () == 3)) {
_Variable* modelMatrix = LocateVar (modelMatrixIndices.lData[modelIdx]);
_SimpleList modelVariableList;
{
_AVLList mvla (&modelVariableList);
modelMatrix->ScanForVariables (mvla, true);
mvla.ReorderList();
}
if (bmatrix->GetHDim () == modelVariableList.lLength) {
// now map model variables to bounds matrix
_SimpleList variableMap;
_String *myName;
for (long k = 0; k < modelVariableList.lLength; k++) {
myName = ((_FString*)bmatrix->GetFormula(k,0)->Compute())->theString;
long vID = LocateVarByName (*myName);
if (vID < 0) {
break;
}
vID = variableNames.GetXtra (vID);
vID = modelVariableList.Find(vID);
if (vID < 0) {
break;
}
variableMap << vID;
}
if (variableMap.lLength == modelVariableList.lLength) {
_Matrix vBounds (variableMap.lLength,2, false, true);
long k2 = 0;
for (; k2 < variableMap.lLength; k2++) {
_Parameter lb = ((_FString*)bmatrix->GetFormula(k2,1)->Compute())->theString->toNum(),
ub = ((_FString*)bmatrix->GetFormula(k2,2)->Compute())->theString->toNum();
if ( ub>lb || k2) {
vBounds.Store (k2,0,lb);
vBounds.Store (k2,1,ub);
if (ub<=lb && vBounds (k2-1,0) <= vBounds (k2-1,1) && (!CheckEqual(vBounds (k2-1,0),0.0) || !CheckEqual(vBounds (k2-1,1),1.0))) {
break;
}
}
}
if (k2 == modelVariableList.lLength) {
// set up the sampling now
_String fName = ProcessLiteralArgument (&ModelNNFile,nil);
FILE* nnFile = doFileOpen (fName.getStr(), "w");
if (nnFile) {
_Matrix* modelMatrix = (_Matrix*) LocateVar(modelMatrixIndices.lData[modelIdx])->GetValue();
_Parameter mainSteps,
checkSteps,
errorTerm,
loopMax,
hiddenNodes,
absError,
nn1,
nn2;
long fullDimension = modelMatrix->GetHDim() * modelMatrix->GetVDim();
checkParameter (ModelNNTrainingSteps, mainSteps, 10000.0);
checkParameter (ModelNNVerificationSample, checkSteps, 500.0);
checkParameter (ModelNNPrecision, errorTerm, 0.01);
checkParameter (ModelNNTrainingSteps, loopMax, 10);
checkParameter (ModelNNHiddenNodes, hiddenNodes, 5);
checkParameter (ModelNNLearningRate, nn1, .3);
checkParameter (ModelNNPersistenceRate, nn2, .1);
Net** matrixNet = new Net* [fullDimension] ;
for (long i = 0; i < fullDimension; i++) {
checkPointer (matrixNet [i] = new Net (variableMap.lLength,(long)hiddenNodes,1,errorTerm,nn1,nn2,100,200,true));
//matrixNet[i]->verbose = true;
}
checkPointer (matrixNet);
_List tIn,
tOut;
FILE* varSamples = doFileOpen ("variableSamples.out", "w");
fprintf (varSamples, "%s" ,LocateVar(modelVariableList.lData[0])->GetName()->getStr());
for (long vc = 1; vc < modelVariableList.lLength; vc++) {
fprintf (varSamples, ",%s" ,LocateVar(modelVariableList.lData[variableMap.lData[vc]])->GetName()->getStr());
}
fprintf (varSamples, "\n");
for (long itCount = 0; itCount < loopMax; itCount ++) {
if (verbI > 5) {
snprintf (buffer, sizeof(buffer), "\nNeural Network Pass %ld. Building a training set...\n", itCount);
BufferToConsole (buffer);
}
while (tIn.countitems() < mainSteps) {
NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut);
}
_Matrix inData (mainSteps, variableMap.lLength, false, true);
_Parameter *md = inData.theData;
for (long matrixC = 0; matrixC < mainSteps; matrixC++) {
_Parameter * ed = ((_Matrix*)tIn (matrixC))->theData;
fprintf (varSamples, "\n%g",*ed);
*md = *ed;
ed++;
md++;
for (long entryC = 1; entryC < variableMap.lLength; entryC++, ed++, md++) {
*md = *ed;
fprintf (varSamples, ",%g", *md);
}
}
tIn.Clear();
if (verbI > 5) {
BufferToConsole ( "Done Building Training Set. Training...\n");
}
long lastDone = 0;
for (long cellCount = 0; cellCount < fullDimension; cellCount++) {
Net* thisCell = matrixNet[cellCount];
_Matrix outVector (mainSteps, 1, false, true);
for (long oc = 0; oc < mainSteps; oc++) {
outVector.theData[oc] = ((_Matrix*)tOut(oc))->theData[cellCount];
}
thisCell->studyAll (inData.theData, outVector.theData, mainSteps);
long nowDone = (cellCount+1)*100./fullDimension;
if (nowDone > lastDone) {
snprintf (buffer, sizeof(buffer),"%ld%% done\n", lastDone = nowDone);
BufferToConsole (buffer);
}
}
tOut.Clear();
if (verbI > 5) {
BufferToConsole ( "Done Training. Resampling...\n");
}
_PMathObj tObj = _Constant(0).Time();
_Parameter time1 = tObj->Value(),
time2;
while (tIn.countitems() < checkSteps) {
NNMatrixSampler (0, vBounds, modelVariableList, variableMap, modelMatrix, tIn, tOut);
}
absError = 0.0;
DeleteObject (tObj);
tObj = _Constant(0).Time();
time2 = tObj->Value();
if (verbI > 5) {
snprintf (buffer, sizeof(buffer),"Done Resampling in %g seconds. Computing Error...\n", time2-time1);
BufferToConsole (buffer);
}
_Parameter maxValT,
maxValE;
for (long verCount = 0; verCount < checkSteps; verCount++) {
_Parameter* inData = ((_Matrix*)tIn(verCount))->theData,
* outData = ((_Matrix*)tOut(verCount))->theData;
for (long cellCount = 0; cellCount < fullDimension; cellCount++) {
Net *thisCell = matrixNet[cellCount];
_Parameter estVal = thisCell->eval(inData)[0],
trueVal = outData[cellCount],
localError;
localError = estVal-trueVal;
if (localError < 0) {
localError = -localError;
}
if (absError < localError) {
maxValT = trueVal;
maxValE = estVal;
absError = localError;
}
}
}
DeleteObject (tObj);
tObj = _Constant(0).Time();
time1 = tObj->Value();
DeleteObject (tObj);
if (verbI > 5) {
snprintf (buffer, sizeof(buffer), "Done Error Checking in %g seconds. Got max abs error %g on the pair %g %g\n", time1-time2, absError, maxValT, maxValE);
BufferToConsole (buffer);
}
if (absError <= errorTerm) {
break;
}
}
if (absError > errorTerm) {
ReportWarning (_String("Couldn't achive desired precision in TrainModelNN. Achieved error of ") & absError);
}
fclose (varSamples);
fprintf (nnFile,"{{\n\"%s\"", LocateVar(modelVariableList.lData[0])->GetName()->getStr());
_Matrix newBounds (modelVariableList.lLength, 2, false, true);
if (vBounds(0,0)>vBounds(0,1)) {
newBounds.Store (variableMap.lData[0],0,0.);
newBounds.Store (variableMap.lData[0],1,1.);
} else {
newBounds.Store (variableMap.lData[0],0,vBounds(0,0));
newBounds.Store (variableMap.lData[0],1,vBounds(0,1));
}
for (long varCounter = 1; varCounter < modelVariableList.lLength; varCounter ++) {
fprintf (nnFile,",\n\"%s\"", LocateVar(modelVariableList.lData[varCounter])->GetName()->getStr());
if (vBounds(varCounter,0)>vBounds(varCounter,1)) {
newBounds.Store (variableMap.lData[varCounter],0,0.);
newBounds.Store (variableMap.lData[varCounter],1,1.);
} else {
newBounds.Store (variableMap.lData[varCounter],0,vBounds(varCounter,0));
newBounds.Store (variableMap.lData[varCounter],1,vBounds(varCounter,1));
}
}
fprintf (nnFile,"\n}}\n");
newBounds.toFileStr (nnFile);
for (long i2 = 0; i2 < fullDimension; i2++) {
matrixNet[i2]->save(nnFile);
delete matrixNet [i2];
}
fclose (nnFile);
delete matrixNet;
} else {
errMsg = _String ("Failed to open ") & fName & " for writing";
}
} else {
errMsg = _String ("Invalid variable bounds in row ") & (k2+1) & " of the bounds matrix";
}
} else {
errMsg = *myName & " was not one of the model parameters";
}
} else {
errMsg = *matrix & " must be a have the same number of rows as the number of model parameters";
}
} else {
errMsg = *matrix & " must be a string matrix with 3 columns";
}
} else {
errMsg = *matrix & " was not the identifier of a valid matrix variable";
}
}
if (errMsg.sLength) {
errMsg = errMsg & _String(" in call to TrainModelNN.");
WarnError (errMsg);
}
}
//__________________________________________________________________________________
void mpiNormalLoop (int rank, int size, _String & baseDir)
{
long senderID = 0;
ReportWarning ("[MPI] Entered mpiNormalLoop");
_String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node
* resStr = nil;
while (theMessage->sLength) {
setParameter (mpiNodeID, (_Parameter)rank);
setParameter (mpiNodeCount, (_Parameter)size);
//ReportWarning (*theMessage);
DeleteObject (resStr);
resStr = nil;
if (theMessage->startswith (mpiLoopSwitchToOptimize) ) {
hyphyMPIOptimizerMode = theMessage->Cut(mpiLoopSwitchToOptimize.sLength,-1).toNum();
ReportWarning (_String("[MPI] Switched to mpiOptimizer loop with mode ") & hyphyMPIOptimizerMode);
MPISendString (mpiLoopSwitchToOptimize,senderID);
mpiOptimizerLoop (rank,size);
ReportWarning ("[MPI] Returned from mpiOptimizer loop");
hyphyMPIOptimizerMode = _hyphyLFMPIModeNone;
pathNames && & baseDir;
} else if ( theMessage->Equal (&mpiLoopSwitchToBGM) ) {
ReportWarning ("[MPI] Received signal to switch to mpiBgmLoop");
MPISendString (mpiLoopSwitchToBGM, senderID); // feedback to source to confirm receipt of message
mpiBgmLoop (rank, size);
ReportWarning ("[MPI] Returned from mpiBgmLoop");
} else {
if (theMessage->beginswith ("#NEXUS")) {
_String msgCopy (*theMessage);
ReportWarning ("[MPI] Received a function to optimize");
ReadDataSetFile (nil,true,theMessage);
ReportWarning ("[MPI] Done with the optimization");
_Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN));
if (lfName) {
resStr = (_String*)(lfName->Compute()->toStr());
} else {
_FString *lfID = (_FString*)FetchObjectFromVariableByType (&lf2SendBack, STRING);
if (!lfID) {
FlagError (_String("[MPI] Malformed MPI likelihood function optimization request - did not specify the LF name to return in variable ") & lf2SendBack & ".\n\n\n" );
break;
}
long f = likeFuncNamesList.Find (lfID->theString);
if (f<0) {
FlagError ("[MPI] Malformed MPI likelihood function optimization request - LF name to return did not refer to a well-defined likelihood function.\n\n\n");
break;
}
_Parameter pv;
checkParameter (shortMPIReturn, pv ,0);
resStr = (_String*)checkPointer(new _String (1024L,true));
((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?_hyphyLFSerializeModeShortMPI:_hyphyLFSerializeModeLongMPI);
resStr->Finalize();
}
} else {
_ExecutionList exL (*theMessage);
_PMathObj res = exL.Execute();
resStr = res?(_String*)res->toStr():new _String ("0");
}
checkPointer (resStr);
MPISendString(*resStr,senderID);
_Parameter keepState = 0.0;
checkParameter (preserveSlaveNodeState, keepState, 0.0);
if (keepState < 0.5) {
PurgeAll (true);
pathNames && & baseDir;
}
}
DeleteObject (theMessage);
theMessage = MPIRecvString (-1,senderID);
}
/*MPISendString(empty,senderID);*/
DeleteObject (resStr);
DeleteObject (theMessage);
}
//__________________________________________________________________________________
void mpiNormalLoop (int rank, int size, _String & baseDir)
{
long senderID = 0;
ReportWarning ("Entered mpiNormalLoop");
_String* theMessage = MPIRecvString (-1,senderID), // listen for messages from any node
* resStr = nil,
_bgmSwitch ("_BGM_SWITCH_"),
css("_CONTEXT_SWITCH_MPIPARTITIONS_");
while (theMessage->sLength)
{
setParameter (mpiNodeID, (_Parameter)rank);
setParameter (mpiNodeCount, (_Parameter)size);
//ReportWarning (*theMessage);
DeleteObject (resStr);
resStr = nil;
if (theMessage->Equal (&css) )
{
mpiPartitionOptimizer = true;
ReportWarning ("Switched to mpiOptimizer loop");
MPISendString(css,senderID);
mpiOptimizerLoop (rank,size);
ReportWarning ("Returned from mpiOptimizer loop");
mpiPartitionOptimizer = false;
pathNames && & baseDir;
}
else if ( theMessage->Equal (&_bgmSwitch) )
{
ReportWarning ("Received signal to switch to mpiBgmLoop");
MPISendString (_bgmSwitch, senderID); // feedback to source to confirm receipt of message
mpiBgmLoop (rank, size);
ReportWarning ("Returned from mpiBgmLoop");
}
else
{
if (theMessage->beginswith ("#NEXUS"))
{
_String msgCopy (*theMessage);
ReportWarning ("Received a function to optimize");
ReadDataSetFile (nil,true,theMessage);
ReportWarning ("Done with the optimization");
_Variable* lfName = FetchVar(LocateVarByName(MPI_NEXUS_FILE_RETURN));
if (lfName)
{
resStr = (_String*)(lfName->Compute()->toStr());
}
else
{
long f = LocateVarByName (lf2SendBack);
if (f>=0)
lfName = FetchVar(f);
if (!(lfName&&(lfName->ObjectClass()==STRING)))
{
_String errMsg ("Malformed MPI likelihood function optimization request - missing LF name to return.\n\n\n");
errMsg = errMsg & msgCopy;
FlagError (errMsg);
break;
}
f = likeFuncNamesList.Find (((_FString*)lfName->Compute())->theString);
if (f<0)
{
_String errMsg ("Malformed MPI likelihood function optimization request - invalid LF name to return.\n\n\n");
errMsg = errMsg & msgCopy;
FlagError (errMsg);
break;
}
_Parameter pv;
checkParameter (shortMPIReturn, pv ,0);
resStr = new _String (1024L,true);
checkPointer (resStr);
((_LikelihoodFunction*)likeFuncList (f))->SerializeLF(*resStr,pv>0.5?5:2);
resStr->Finalize();
}
}
else
{
_ExecutionList exL (*theMessage);
/*printf ("Received:\n %s\n", ((_String*)exL.toStr())->sData);*/
_PMathObj res = exL.Execute();
resStr = res?(_String*)res->toStr():new _String ("0");
}
checkPointer (resStr);
DeleteObject (theMessage);
MPISendString(*resStr,senderID);
_Parameter keepState = 0.0;
checkParameter (preserveSlaveNodeState, keepState, 0.0);
if (keepState < 0.5)
{
PurgeAll (true);
pathNames && & baseDir;
}
}
theMessage = MPIRecvString (-1,senderID);
}
/*MPISendString(empty,senderID);*/
DeleteObject (resStr);
DeleteObject (theMessage);
}
void _HYObjectInspector::KillObject (void)
{
_HYTable* dl = (_HYTable*)GetCellObject (HY_OBJECT_INSPECTOR_TABLE_ROW,0);
_HYButtonBar* b1 = (_HYButtonBar*)GetCellObject (0,0);
_HYPullDown* p1 = (_HYPullDown*)GetCellObject (0,2);
_SimpleList sel;
dl->GetSelection (sel);
if (sel.lLength) {
b1->EnableButton (0,true);
long f;
switch (p1->GetSelection()) {
case 0: {
for (f = 0; f<sel.lLength; f+=2) {
long treeID = LocateVarByName (*(_String*)dl->GetCellData (0,sel.lData[f]/2));
if (treeID>=0)
#ifndef USE_AVL_NAMES
treeID = variableReindex.lData[treeID];
#else
treeID = variableNames.GetXtra(treeID);
#endif
if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_BOLD) {
_String prompt ("Tree ");
prompt = prompt & *LocateVar (treeID)->GetName() & objectInspectorKillPrompt;
if (warnTree||(!ProceedPromptWithCheck (prompt,donotWarnAgain,warnTree))) {
continue;
}
}
postTreeKillEvent (GetID(), treeID);
}
break;
}
case 1: {
for (f = 0; f<sel.lLength; f+=2) {
long dsID = dataSetNamesList.Find ((_String*)dl->GetCellData (0,sel.lData[f]/2));
if (dsID>=0) {
if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_BOLD) {
_String prompt ("Dataset ");
prompt = prompt & *(_String*)dataSetNamesList(dsID) & objectInspectorKillPrompt;
if (!((!warnTree)&&ProceedPromptWithCheck (prompt,donotWarnAgain,warnDS))) {
continue;
}
}
postDSKillEvent (GetID(), dsID);
}
}
break;
}
case 2: {
for (f = 0; f<sel.lLength; f+=2) {
if (sel.lData[f]/2<modelTemplates.lLength) {
_List* modelT = FindModelTemplate((_String*)dl->GetCellData (0,sel.lData[f]/2));
if (modelT) { // template model
_String*fileLocation = (_String*)(*modelT)(2),
errMsg (*(_String*)dl->GetCellData (0,sel.lData[f]/2));
if (dl->cellTypes.lData[sel.lData[f]] & HY_TABLE_ITALIC) {
errMsg = errMsg & " is a template model. If you wish to delete it, please do so by removing the appropriate file from the \"Substitution Models\" directory.";
ProblemReport (errMsg);
} else {
errMsg = errMsg & " is about to be deleted. This action is NOT undoable.";
if (ProceedPrompt (errMsg,(Ptr)this)) {
if (remove (fileLocation->sData)) {
errMsg = "File delete operation failed.";
ProblemReport (errMsg);
} else {
errMsg = *(_String*)dl->GetCellData (0,sel.lData[f]/2);
for (long k=0; k<modelTemplates.lLength; k++) {
_List* thisList = (_List*)modelTemplates(k);
if (errMsg.Equal((_String*)(*thisList)(0))) {
modelTemplates.Delete (k);
}
}
BuildListOfObjects (2);
}
}
}
}
}
}
break;
}
}
}
}
//__________________________________________________________________________________
void DeleteTreeVariable (_String&name, _SimpleList& parms, bool doDeps)
{
DeleteTreeVariable(LocateVarByName (name), parms,doDeps);
}
//__________________________________________________________________________________
void DeleteVariable (_String&name, bool deleteself)
{
DeleteVariable(LocateVarByName (name), deleteself);
}
