int SetJobUserPrio(
char *RBuffer,
char *Buffer,
int FLAGS,
char *Auth,
long *BufSize)
{
char Name[MAX_MNAME];
int Priority;
mjob_t *J;
const char *FName = "SetJobUserPrio";
DBG(2,fUI) DPrint("%s(RBuffer,Buffer,%d,%s,BufSize)\n",
FName,
FLAGS,
Auth);
MUSScanF(RBuffer,"%x%s %d",
sizeof(Name),
Name,
&Priority);
if ((Priority < 0) || (Priority > 100))
{
strcpy(Buffer,"ERROR: user priority must be in the range 0 - 100\n");
return(FAILURE);
}
if (MJobFind(Name,&J,0) == FAILURE)
{
sprintf(Buffer,"Error: cannot locate job '%s'\n",
Name);
return(FAILURE);
}
/* security check */
if (!(FLAGS & ((1 << fAdmin1) | (1 << fAdmin2))))
{
if (strcasecmp(J->Cred.U->Name,Auth) != 0)
{
DBG(2,fUI) DPrint("INFO: user %s is not authorized to set user priority on job %s\n",
Auth,
J->Name);
sprintf(Buffer,"user %s is not authorized to set user priority on job %s\n",
Auth,
J->Name);
return(FAILURE);
}
}
J->UPriority = Priority;
DBG(2,fUI) DPrint("INFO: user priority on job '%s' set to %d\n",
Name,
Priority);
return(SUCCESS);
} /* END JobSetUserPrio() */
int MTrigLaunchInternalAction(
mtrig_t *T, /* I (optional) */
char *ActionString, /* I */
mbool_t *Complete, /* O */
char *Msg) /* O (optional,minsize=MMAX_LINE) */
{
/* FORMAT: <OTYPE>:<OID>:<ACTION>:<context information> */
char *ptr = NULL;
char *TokPtr = NULL;
char *tmpAString = NULL;
char ArgLine[MMAX_LINE*10];
char OID[MMAX_BUFFER];
char Action[MMAX_LINE*10];
enum MXMLOTypeEnum OType;
const char *FName = "MTrigLaunchInternalAction";
MDB(4,fSCHED) MLog("%s(%s,%s,Msg)\n",
FName,
((T != NULL) && (T->TName != NULL)) ? T->TName : "NULL",
(ActionString != NULL) ? ActionString : "NULL");
if (Msg != NULL)
Msg[0] = '\0';
if (Complete != NULL)
*Complete = TRUE;
if (ActionString == NULL)
{
if (Msg != NULL)
strcpy(Msg,"internal error");
return(FAILURE);
}
MUStrDup(&tmpAString,ActionString);
/* NOTE: this format and parsing is duplicated in MTrigCheckInternalAction()
any update here must be copied to that routine */
ptr = MUStrTok(tmpAString,":",&TokPtr);
if (ptr == NULL)
{
MDB(4,fSCHED) MLog("ERROR: invalid action");
if (Msg != NULL)
strcpy(Msg,"invalid action");
MUFree(&tmpAString);
return(FAILURE);
}
OType = (enum MXMLOTypeEnum)MUGetIndexCI(ptr,MXO,FALSE,mxoNONE);
if (OType == mxoNONE)
{
MDB(4,fSCHED) MLog("ERROR: invalid object type '%s' in %s\n",
ptr,
FName);
if (Msg != NULL)
strcpy(Msg,"invalid object type");
MUFree(&tmpAString);
return(FAILURE);
}
ptr = MUStrTok(NULL,":",&TokPtr);
if (ptr == NULL)
{
MDB(4,fSCHED) MLog("ERROR: cannot extract object ID in %s\n",
FName);
if (Msg != NULL)
strcpy(Msg,"invalid object id");
MUFree(&tmpAString);
return(FAILURE);
}
MUStrCpy(OID,ptr,sizeof(OID));
ptr = MUStrTok(NULL,":",&TokPtr);
if (ptr == NULL)
{
MDB(4,fSCHED) MLog("ERROR: cannot parse action in %s\n",
FName);
if (Msg != NULL)
strcpy(Msg,"cannot locate action");
MUFree(&tmpAString);
return(FAILURE);
}
MUStrCpy(Action,ptr,sizeof(Action));
if ((TokPtr != NULL) && (TokPtr[0] != '\0'))
MUStrCpy(ArgLine,TokPtr,sizeof(ArgLine));
else
ArgLine[0] = '\0';
MUFree(&tmpAString);
switch (OType)
{
case mxoJob:
{
mjob_t *J;
char *JID;
char *TokPtr;
/* OID FORMAT: <JID>[+<JID>]... */
mbool_t FailureDetected;
enum MJobCtlCmdEnum atype;
atype = (enum MJobCtlCmdEnum)MUGetIndexCI(Action,MJobCtlCmds,FALSE,mjcmNONE);
if (atype == mjcmNONE)
{
MDB(4,fSCHED) MLog("ERROR: cannot parse job action '%s' in %s\n",
Action,
FName);
return(FAILURE);
}
FailureDetected = FALSE;
JID = MUStrTok(OID,"+",&TokPtr);
while (JID != NULL)
{
if (!strcmp(JID,"-"))
{
if (T == NULL)
return(FAILURE);
J = (mjob_t *)T->O;
}
else if (MJobFind(JID,&J,mjsmExtended) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot locate job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
switch (atype)
{
case mjcmCancel:
{
char *CCodeCheck;
char UpperArgLine[MMAX_LINE];
MDB(7,fSCHED) MLog(" INFO: Job flaged for cancel by trigger action. '%s' in %s\n",
JID,
FName);
bmset(&J->IFlags, mjifCancel);
MUStrToUpper(ArgLine,UpperArgLine,MMAX_LINE);
/* If specified, allow trigger to set the completion code of the job */
if ((CCodeCheck = strstr(UpperArgLine,"CCODE=")) != NULL)
{
CCodeCheck += strlen("CCODE="); /* Skip the "CCODE:" part */
if (CCodeCheck[0] != '\0')
{
J->CompletionCode = strtol(CCodeCheck,NULL,10);
}
}
else
{
J->CompletionCode = 273; /* Completion code for a canceled job in TORQUE */
}
} /* END case mjcmCancel */
break;
case mjcmComplete:
{
char *CCodeCheck;
char UpperArgLine[MMAX_LINE];
if ((!bmisset(&J->Flags,mjfSystemJob) &&
!bmisset(&J->Flags,mjfNoRMStart)))
{
MDB(4,fSCHED) MLog("ERROR: job %s in bad state for complete in %s\n",
J->Name,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
/* the next three lines set the WCLimit to the exact runtime
* this also causes MS3WorkloadQuery to remove the job */
J->SpecWCLimit[0] = MSched.Time - J->StartTime;
J->SpecWCLimit[1] = MSched.Time - J->StartTime;
J->WCLimit = MSched.Time - J->StartTime;
if (MJOBISACTIVE(J) == TRUE)
{
MRsvJCreate(J,NULL,J->StartTime,mjrActiveJob,NULL,FALSE); /* just removes active rsv? */
}
MUStrToUpper(ArgLine,UpperArgLine,MMAX_LINE);
if ((CCodeCheck = strstr(UpperArgLine,"CCODE=")) != NULL)
{
CCodeCheck += strlen("CCODE="); /* Skip the "CCODE:" part */
if (CCodeCheck[0] != '0')
{
J->CompletionCode = strtol(CCodeCheck,NULL,10);
}
}
/* MSched.TrigCompletedJob = TRUE; -- not sure why we wanted to skip the UI phase here... */
} /* END BLOCK (case mjcmComplete) */
break;
case mjcmModify:
{
char *TokPtr;
char *Attr = NULL;
char *Value = NULL;
char *Msg = NULL;
enum MJobAttrEnum aindex;
enum MObjectSetModeEnum Op;
if (strstr(ArgLine,"+="))
Op = mAdd;
else if (strstr(ArgLine,"-="))
Op = mDecr;
else
Op = mSet;
Attr = MUStrTok(ArgLine,"+-=",&TokPtr);
Value = MUStrTok(NULL,"=: \t\n",&TokPtr);
Msg = MUStrTok(NULL,":\t\n",&TokPtr); /* extract message */
aindex = (enum MJobAttrEnum)MUGetIndexCI(Attr,MJobAttr,FALSE,mnaNONE);
switch (aindex)
{
case mjaHold:
if (MJobSetAttr(J,mjaHold,(Value != NULL) ? (void **)Value : (void **)"batch",mdfString,mSet) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot hold job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
if (Msg != NULL)
{
/* attach provided message */
MMBAdd(
&J->MessageBuffer,
Msg,
NULL,
mmbtHold,
0,
0,
NULL);
}
if (MJOBISACTIVE(J))
{
/* Set to idle but don't clear DRM and DRMJID */
MJobToIdle(J,FALSE,FALSE);
J->SubState = mjsstNONE;
}
/* checkpoint change (make this configurable at this point?) */
MOCheckpoint(mxoJob,(void *)J,FALSE,NULL);
break;
case mjaVariables:
{
char tmpLine[MMAX_LINE*2];
snprintf(tmpLine,sizeof(tmpLine),"%s=%s",
Value,
Msg);
if (MJobSetAttr(J,mjaVariables,(void **)tmpLine,mdfString,Op) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot set trigvar job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
} /* END BLOCK */
break;
default:
if (!strcasecmp(Attr,"gres"))
{
if ((Op == mAdd) &&
(MJobAddRequiredGRes(
J,
Value,
1,
mxaGRes,
FALSE,
FALSE) == SUCCESS))
{
/* SUCCESS - NO-OP */
}
else if ((Op == mDecr) &&
(MJobRemoveGRes(J,Value,TRUE) == SUCCESS))
{
/* SUCCESS -- Do Nothing */
}
else
{
MDB(4,fSCHED) MLog("ERROR: cannot adjust gres for job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
}
else
{
MDB(4,fSCHED) MLog("ERROR: cannot modify attribute '%s' for job '%s' in %s\n",
Attr,
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
break;
}
} /* END BLOCK (case mjcmModify) */
break;
case mjcmSignal:
{
char EMsg[MMAX_LINE];
int SC;
if (MRMJobSignal(J,-1,ArgLine,EMsg,&SC) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot send signal '%s' to job '%s' in %s\n",
ArgLine,
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
} /* END BLOCK (case mjcmSignal) */
break;
case mjcmStart:
/* FIXME: check reservation, check allocation */
{
J->SubState = mjsstNONE;
/* set "copy-tasklist" flag */
bmset(&J->IFlags,mjifCopyTaskList);
if (MRMJobStart(J,NULL,NULL) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot start job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
J->State = mjsRunning;
} /* END BLOCK (case mjcmStart) */
break;
case mjcmRequeue:
/* FIXME: check reservation, check allocation */
{
char EMsg[MMAX_LINE];
J->SubState = mjsstNONE;
/* set "copy-tasklist" flag */
if (MJobRequeue(J,NULL,"workflow failure",EMsg,NULL) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot requeue job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
J->State = mjsIdle;
} /* END BLOCK (case mjcmStart) */
default:
MDB(4,fSCHED) MLog("ERROR: action '%s' not handled in %s\n",
Action,
FName);
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (atype) */
JID = MUStrTok(NULL,"+",&TokPtr);
} /* END while (JID != NULL) */
if (FailureDetected == TRUE)
{
return(FAILURE);
}
} /* END BLOCK (case mxoJob) */
break;
case mxoNode:
{
enum MNodeCtlCmdEnum atype;
char *rsvProfile = NULL;
atype = (enum MNodeCtlCmdEnum)MUGetIndexCI(Action,MNodeCtlCmds,FALSE,mncmNONE);
if (!strcasecmp(Action,"reserve"))
{
char tmpLine[MMAX_LINE];
snprintf(tmpLine,sizeof(tmpLine),"auto-trigger-rsv.%s",
T->TName);
if (ArgLine[0] != '\0')
{
char *Attr = MUStrTok(ArgLine,"=",&TokPtr);
char *Value = MUStrTok(NULL,"= \t\n",&TokPtr);
if ((Attr != NULL) &&
(strncasecmp(Attr,"rsvprofile",strlen("rsvprofile")) == 0))
rsvProfile = Value;
}
if (T->OType != mxoRsv)
{
MNodeReserve(
(mnode_t *)T->O,
MCONST_EFFINF,
tmpLine,
rsvProfile,
"Reservation created by trigger");
return(SUCCESS);
}
else
{
return(FAILURE);
}
}
else if ((!strcasecmp(Action,"evacvms")) && (MSched.AllowVMMigration == TRUE))
{
/* Evacuate all vms on nodes held by this reservation */
char tmpMsg[MMAX_LINE];
int rc = SUCCESS;
mnl_t NL;
MNLInit(&NL);
MNLFromString(OID,&NL,1,FALSE);
rc = MNodeEvacVMsMultiple(&NL);
MNLFree(&NL);
/* Write out events */
snprintf(tmpMsg,sizeof(tmpMsg),"attempting to migrate vms from node '%s' for evacvms reservation '%s' (TID: %s)",
OID,
((mrsv_t *)T->O)->Name,
(T->TName != NULL) ? T->TName : "NULL");
MDB(5,fSTRUCT) MLog("INFO: %s\n",
tmpMsg);
return(rc);
} /* END if (!strcasecmp(Action,"evacvms")) */
if (atype == mncmNONE)
{
return(FAILURE);
}
switch (atype)
{
case mncmModify:
if (MSched.AggregateNodeActions == FALSE)
{
char *ptr;
char *TokPtr;
mnode_t *N;
char *Attr;
char *Value;
enum MNodeAttrEnum aindex;
/* always look on the bright side of life */
mbool_t NodeModifySuccessDetected;
/* FORMAT: <ARG>=<VALUE> */
Attr = MUStrTok(ArgLine,"=",&TokPtr);
Value = MUStrTok(NULL,"= \t\n",&TokPtr);
aindex = (enum MNodeAttrEnum)MUGetIndexCI(Attr,MNodeAttr,FALSE,mnaNONE);
NodeModifySuccessDetected = FALSE;
/* FORMAT: OID = node[,node]... */
ptr = MUStrTok(OID,",",&TokPtr);
if ((ptr == NULL) || (aindex == mnaNONE))
{
MDB(4,fSCHED) MLog("ERROR: unknown attribute to modify on node '%s'\n",
Attr);
return(FAILURE);
}
while (ptr != NULL)
{
if (MNodeFind(ptr,&N) == FAILURE)
{
ptr = MUStrTok(NULL,",",&TokPtr);
continue;
}
switch (aindex)
{
case mnaOS:
{
char EMsg[MMAX_LINE];
if (!strcmp(Value,MAList[meOpsys][0]))
{
if ((N->PowerSelectState == mpowOff) &&
(MNodeSetAttr(N,mnaPowerIsEnabled,(void **)"true",mdfString,mVerify) == FAILURE))
{
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
/* do not return failure just because one provisioning job fails */
}
}
else if (MNodeProvision(N,NULL,NULL,Value,TRUE,EMsg,NULL) == FAILURE)
{
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
/* do not return failure just because one provisioning job fails */
}
else
{
NodeModifySuccessDetected = TRUE;
}
}
break;
case mnaPowerIsEnabled:
/* NOTE: do not return failure just because node modify fails
* the routine will fail if all node modifies fail */
if (!strcmp(Value,"on"))
{
/* power node on */
if (MNodeSetAttr(N,mnaPowerIsEnabled,(void **)"true",mdfString,mVerify) == SUCCESS)
{
NodeModifySuccessDetected = TRUE;
}
}
else if (!strcasecmp(Value,"off"))
{
/* power node off */
if (MNodeSetAttr(N,mnaPowerIsEnabled,(void **)"false",mdfString,mVerify) == SUCCESS)
{
NodeModifySuccessDetected = TRUE;
}
}
else if (!strcasecmp(Value,"reset"))
{
char EMsg[MMAX_LINE];
/* reset node */
if (MNodeReboot(N,NULL,EMsg,NULL) == FAILURE)
{
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
}
else
{
NodeModifySuccessDetected = TRUE;
}
}
else if (!strcasecmp(Value,"on+"))
{
/* power node on, but trigger isn't finished until action is complete */
if (Complete != NULL)
*Complete = FALSE;
if (T != NULL)
MUStrDup(&T->CompleteAction,ActionString);
MNodeSetAttr(N,mnaPowerIsEnabled,(void **)"true",mdfString,mVerify);
T->PID = 1;
}
else if (!strcasecmp(Value,"off+"))
{
/* power node off, but trigger isn't finished until action is complete */
if (T != NULL)
MUStrDup(&T->CompleteAction,ActionString);
if (Complete != NULL)
*Complete = FALSE;
MNodeSetAttr(N,mnaPowerIsEnabled,(void **)"false",mdfString,mVerify);
T->PID = 1;
}
break;
default:
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (aindex) */
ptr = MUStrTok(NULL,",",&TokPtr);
} /* END while (ptr != NULL) */
/* if none of the node modify requests (non-aggregating) succeeded,
then return failure. (success just means it was submitted
at this stage) */
if ((aindex == mnaOS) &&
(NodeModifySuccessDetected == FALSE))
{
return (FAILURE);
}
} /* END if (MSched.AggregateNodeActions == FALSE) */
else
{
mnl_t NL;
char *Attr;
char *Value;
enum MNodeAttrEnum aindex;
int nindex;
/* FORMAT: <ARG>=<VALUE> */
Attr = MUStrTok(ArgLine,"=",&TokPtr);
Value = MUStrTok(NULL,"= \t\n",&TokPtr);
aindex = (enum MNodeAttrEnum)MUGetIndexCI(Attr,MNodeAttr,FALSE,mnaNONE);
/* FORMAT: OID = node[,node]... */
if (aindex == mnaNONE)
{
return(FAILURE);
}
MNLInit(&NL);
MNLFromString(OID,&NL,-1,FALSE);
/* NOTE: in the node aggregating case, return failure when there is an error
* performing node modify operations (e.g. communicating with the RM) */
switch (aindex)
{
case mnaNodeState:
{
enum MStatusCodeEnum SC;
char EMsg[MMAX_LINE];
if (MNodeModify(NULL,OID,mnaNodeState,Value,EMsg,&SC) == FAILURE)
{
MDB(2,fRM) MLog("WARNING: cannot modify state of '%s' = %s\n",
OID,
EMsg);
return(FAILURE);
}
} /* END case mnaNodeState */
break;
case mnaOS:
{
char EMsg[MMAX_LINE];
if (MNLGetemReady(&NL,Value,EMsg) == FAILURE)
{
/* NYI: append EMsg to Msg */
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
MNLFree(&NL);
return (FAILURE);
}
} /* END BLOCK case mnaOS */
break;
case mnaPowerIsEnabled:
{
mnode_t *N;
for (nindex = 0;MNLGetNodeAtIndex(&NL,nindex,&N) == SUCCESS;nindex++)
{
if (N == MSched.GN)
continue;
if ((!strcasecmp(Value,"on")) ||
(!strcasecmp(Value,"on+")))
{
/* power node on */
if ((N->PowerSelectState == mpowOn) ||
(MNodeGetSysJob(N,msjtPowerOn,MBNOTSET,NULL) == SUCCESS))
{
/* skip this node */
MDB(5,fSCHED) MLog("INFO: not powering node '%s' on--power select state is already 'ON'\n",
N->Name);
MNLRemove(&NL,N);
continue;
}
}
else if ((!strcasecmp(Value,"off")) ||
(!strcasecmp(Value,"off+")))
{
/* power node off */
if ((N->PowerSelectState == mpowOff) ||
(MNodeGetSysJob(N,msjtPowerOff,TRUE,NULL) == SUCCESS))
{
/* skip this node */
MDB(5,fSCHED) MLog("INFO: not powering node '%s' off--power select state is already 'OFF'\n",
N->Name);
MNLRemove(&NL,N);
continue;
}
}
} /* END for (nindex) */
} /* END BLOCK */
if (!strcmp(Value,"on"))
{
/* power node on */
if (MNLSetAttr(&NL,mnaPowerIsEnabled,(void **)"true",mdfString,mVerify) == FAILURE)
{
MNLFree(&NL);
return (FAILURE);
}
}
else if (!strcasecmp(Value,"off"))
{
/* power node off */
if (MNLSetAttr(&NL,mnaPowerIsEnabled,(void **)"false",mdfString,mVerify) == FAILURE)
{
MNLFree(&NL);
return (FAILURE);
}
}
else if (!strcasecmp(Value,"reset"))
{
char EMsg[MMAX_LINE];
/* reset node */
if (MNodeReboot(NULL,&NL,EMsg,NULL) == FAILURE)
{
MNLFree(&NL);
/* NYI: append EMsg to Msg */
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
return (FAILURE);
}
}
else if (!strcasecmp(Value,"on+"))
{
/* power node on, but trigger isn't finished until action is complete */
if (MNLSetAttr(&NL,mnaPowerIsEnabled,(void **)"true",mdfString,mVerify) == FAILURE)
{
MNLFree(&NL);
if (Complete != NULL)
*Complete = TRUE;
return(FAILURE);
}
if (Complete != NULL)
*Complete = FALSE;
if (T != NULL)
MUStrDup(&T->CompleteAction,ActionString);
T->PID = 1;
}
else if (!strcasecmp(Value,"off+"))
{
/* power node off, but trigger isn't finished until action is complete */
if (MNLSetAttr(&NL,mnaPowerIsEnabled,(void **)"false",mdfString,mVerify) == FAILURE)
{
MNLFree(&NL);
if (Complete != NULL)
*Complete = TRUE;
return(FAILURE);
}
if (Complete != NULL)
*Complete = FALSE;
if (T != NULL)
MUStrDup(&T->CompleteAction,ActionString);
T->PID = 1;
}
break;
default:
MNLFree(&NL);
return (FAILURE);
/*NOTREACHED*/
break;
} /* END switch (aindex) */
} /* END else */
break;
case mncmCreate:
{
int rc;
mnode_t *N;
char EMsg[MMAX_LINE];
char tmpLine[MMAX_LINE*10];
/* FORMAT: COUNT:OS */
if (MNodeFind(OID,&N) == FAILURE)
{
if (Msg != NULL)
snprintf(Msg,MMAX_LINE,"cannot locate hostlist '%s'",OID);
return(FAILURE);
}
/* FORMAT: COUNT:OS */
snprintf(tmpLine,sizeof(tmpLine),"%s:%s%s%s",
N->Name,
ArgLine,
((T->OType == mxoJob) && ((mjob_t *)T->O != NULL)) ? ":operationid=" : "",
((T->OType == mxoJob) && ((mjob_t *)T->O != NULL)) ? ((mjob_t *)T->O)->Name : "");
mstring_t Output(MMAX_LINE);
rc = MRMVMCreate(tmpLine,&Output,EMsg,NULL);
if (rc == FAILURE)
{
if (Msg != NULL)
MUStrCpy(Msg,EMsg,MMAX_LINE);
MUStrCpy(Msg,Output.c_str(),MMAX_LINE);
}
return(rc);
} /* END BLOCK (case mncmCreate) */
break;
case mncmMigrate:
{
int rc;
char *TokPtr = ArgLine;
char *VMName;
char *SrcNodeName;
char *DestNodeName;
mnode_t *SrcNode;
mnode_t *DestNode;
mln_t *VMLink;
char *VMList[2];
mnl_t DestNL;
/* FORMAT: <VM>:<DestNode>,... */
TokPtr = ArgLine;
VMName = MUStrTok(NULL,":=",&TokPtr);
DestNodeName = MUStrTok(NULL,":=",&TokPtr);
SrcNodeName = OID;
if ((DestNodeName == NULL) || (VMName == NULL) ||(SrcNodeName) == NULL)
{
return(FAILURE);
}
if (MNodeFind(SrcNodeName,&SrcNode) == FAILURE)
{
return(FAILURE);
}
if (MNodeFind(DestNodeName,&DestNode) == FAILURE)
{
return(FAILURE);
}
if (MULLCheck(SrcNode->VMList,VMName,&VMLink) == FAILURE)
{
/* Should we be checking this here? */
/* return(FAILURE); */
}
MNLInit(&DestNL);
VMList[0] = VMName;
MNLSetNodeAtIndex(&DestNL,0,DestNode);
MNLSetTCAtIndex(&DestNL,0,1);
MNLTerminateAtIndex(&DestNL,1);
VMList[1] = NULL;
rc = MRMJobMigrate((mjob_t *)T->O,VMList,&DestNL,Msg,NULL);
MNLFree(&DestNL);
return(rc);
} /* END BLOCK (case mncmMigrateVM) */
break;
default:
return(FAILURE);
/*NOTREACHED*/
break;
}
} /* END BLOCK (case mxoNode) */
break;
case mxoxJobGroup:
{
job_iter JTI;
marray_t JArray;
int jindex;
mjob_t *J;
char *JID;
char *ptr;
char *TokPtr;
/* OID FORMAT: <JID>[+<JID>]... */
mbool_t FailureDetected;
enum MJobCtlCmdEnum atype;
char *Attr = NULL;
char *Value = NULL;
char *Msg = NULL;
enum MJobAttrEnum aindex = mjaNONE;
atype = (enum MJobCtlCmdEnum)MUGetIndexCI(Action,MJobCtlCmds,FALSE,mjcmNONE);
if (atype == mjcmNONE)
{
MDB(4,fSCHED) MLog("ERROR: cannot parse job action '%s' in %s\n",
Action,
FName);
return(FAILURE);
}
if (atype == mjcmModify)
{
char *TokPtr;
Attr = MUStrTok(ArgLine,"+-=",&TokPtr);
Value = MUStrTok(NULL,"=: \t\n",&TokPtr);
Msg = MUStrTok(NULL,":\t\n",&TokPtr); /* extract message */
aindex = (enum MJobAttrEnum)MUGetIndexCI(Attr,MJobAttr,FALSE,mnaNONE);
}
FailureDetected = FALSE;
JID = MUStrTok(OID,"+",&TokPtr);
while (JID != NULL)
{
MUArrayListCreate(&JArray,sizeof(mjob_t *),1);
if (!strcmp(JID,"-"))
{
if (T == NULL)
return(FAILURE);
ptr = T->OID;
}
else
{
ptr = JID;
}
MJobIterInit(&JTI);
while (MJobTableIterate(&JTI,&J) == SUCCESS)
{
if ((J->JGroup != NULL) && (J->JGroup->Name != NULL) && (!strcmp(ptr,J->JGroup->Name)))
{
MUArrayListAppend(&JArray,&J);
}
}
if (MUArrayListGet(&JArray,0) == NULL)
{
MDB(4,fSCHED) MLog("ERROR: cannot locate job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
for (jindex = 0;jindex < JArray.NumItems;jindex++)
{
J = *(mjob_t **)MUArrayListGet(&JArray,jindex);
if (J == NULL)
continue;
switch (atype)
{
case mjcmCancel:
{
char *CCodeCheck;
char UpperArgLine[MMAX_LINE];
if (MJobCancel(J,NULL,FALSE,Msg,NULL) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot cancel job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
MUStrToUpper(ArgLine,UpperArgLine,MMAX_LINE);
/* If specified, allow trigger to set the completion code of the job */
if ((CCodeCheck = strstr(UpperArgLine,"CCODE=")) != NULL)
{
CCodeCheck += strlen("CCODE="); /* Skip the "CCODE:" part */
if (CCodeCheck[0] != '\0')
{
J->CompletionCode = strtol(CCodeCheck,NULL,10);
}
}
else
{
J->CompletionCode = 273; /* Completion code for a canceled job in TORQUE */
}
} /* END case mjcmCancel */
break;
case mjcmModify:
switch (aindex)
{
case mjaHold:
if (MJobSetAttr(
J,
mjaHold,
(Value != NULL) ? (void **)Value : (void **)"batch",
mdfString,
mSet) == FAILURE)
{
MDB(4,fSCHED) MLog("ERROR: cannot hold job '%s' in %s\n",
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
if (Msg != NULL)
{
/* attach provided message */
MMBAdd(
&J->MessageBuffer,
Msg,
NULL,
mmbtHold,
0,
0,
NULL);
}
if (J->SubState == mjsstProlog)
{
J->SubState = mjsstNONE;
}
/* checkpoint change (make this configurable at this point?) */
MOCheckpoint(mxoJob,(void *)J,FALSE,NULL);
break;
default:
MDB(4,fSCHED) MLog("ERROR: cannot modify attribute '%s' for job '%s' in %s\n",
Attr,
JID,
FName);
JID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
break;
} /* END switch (aindex) */
break;
default:
MDB(4,fSCHED) MLog("ERROR: action '%s' not handled in %s\n",
Action,
FName);
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (atype) */
} /* END for (jindex) */
JID = MUStrTok(NULL,"+",&TokPtr);
MUArrayListFree(&JArray);
} /* END while (JID != NULL) */
if (FailureDetected == TRUE)
{
return(FAILURE);
}
} /* END BLOCK (case mxoxJobGroup) */
break;
case mxoSched:
{
enum MSchedCtlCmdEnum atype;
atype = (enum MSchedCtlCmdEnum)MUGetIndexCI(Action,MSchedCtlCmds,FALSE,msctlNONE);
switch (atype)
{
case msctlVMMigrate:
{
enum MVMMigrationPolicyEnum aindex;
aindex = (enum MVMMigrationPolicyEnum)MUGetIndexCI(ArgLine,MVMMigrationPolicy,FALSE,mvmmmNONE);
return(MUIVMCtlPlanMigrations(aindex,FALSE,FALSE,NULL));
}
default:
return(FAILURE);
} /* END switch (atype) */
} /* END case mxoSched */
break;
case mxoTrig:
{
enum MSchedCtlCmdEnum atype;
atype = (enum MSchedCtlCmdEnum)MUGetIndexCI(Action,MSchedCtlCmds,FALSE,msctlNONE);
if (atype == msctlNONE)
{
return(FAILURE);
}
switch (atype)
{
case msctlModify:
{
char *TokPtr;
mtrig_t *T;
char *Attr;
char *Value;
enum MTrigAttrEnum aindex;
Attr = MUStrTok(ArgLine,"=",&TokPtr);
Value = MUStrTok(NULL,"\"",&TokPtr);
/* FORMAT: OID = trigid */
aindex = (enum MTrigAttrEnum)MUGetIndexCI(Attr,MTrigAttr,FALSE,mtaNONE);
if ((OID == NULL) || (OID[0] == '\0') || (aindex == mtaNONE))
{
return(FAILURE);
}
if (MTrigFind(OID,&T) == FAILURE)
{
return(FAILURE);
}
switch (aindex)
{
case mtaActionData:
if (MTrigSetAttr(T,aindex,Value,mdfString,mSet) == FAILURE)
{
return(FAILURE);
}
break;
default:
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (aindex) */
break;
} /* END case msctlModify */
default:
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (atype) */
} /* END BLOCK (case mxoTrig) */
break;
case mxoRM:
{
mrm_t *R = NULL;
char *RMID;
char *TokPtr;
mbool_t FailureDetected;
enum MRMCtlCmdEnum atype;
atype = (enum MRMCtlCmdEnum)MUGetIndexCI(Action,MRMCtlCmds,FALSE,mrmctlNONE);
if (atype == mrmctlNONE)
{
MDB(4,fSCHED) MLog("ERROR: cannot parse rm action '%s' in %s\n",
Action,
FName);
return(FAILURE);
}
FailureDetected = FALSE;
RMID = MUStrTok(OID,"+",&TokPtr);
while (RMID != NULL)
{
if (!strcmp(RMID, "-"))
{
if (T == NULL)
return(FAILURE);
R = (mrm_t *)T->O;
}
else if (MRMFind(RMID,&R) == FAILURE)
{
if (Msg != NULL)
strcpy(Msg,"cannot locate specified RM");
RMID = MUStrTok(NULL,"+",&TokPtr);
FailureDetected = TRUE;
continue;
}
switch (atype)
{
case mrmctlModify:
{
char *TokPtr;
enum MRMAttrEnum aindex;
char *Attr;
char *Value;
/* FORMAT: <attr>=<value> */
Attr = MUStrTok(ArgLine,"=",&TokPtr);
Value = MUStrTok(NULL,"\"",&TokPtr);
aindex = (enum MRMAttrEnum)MUGetIndexCI(Attr,MRMAttr,FALSE,mrmaNONE);
switch (aindex)
{
case mrmaState:
return(MRMSetAttr(R,aindex,(void **)Value,mdfString,mSet));
break;
default:
if (Msg != NULL)
strcpy(Msg,"cannot modify specified attribute");
return(FAILURE);
/* NOTREACHED */
break;
} /* END switch (aindex) */
} /* END BLOCK (case mrmctlModify) */
break;
case mrmctlDestroy:
MDB(3,fUI) MLog("WARNING: cannot destroy static RM\n");
if (Msg != NULL)
strcpy(Msg,"WARNING: cannot destroy static RM\n");
return(FAILURE);
/* NOTREACHED */
break;
case mrmctlPurge:
return(MRMPurge(R));
break;
default:
if (Msg != NULL)
strcpy(Msg,"action not supported for RM object");
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (atype) */
RMID = MUStrTok(NULL,"+",&TokPtr);
} /* END while (RMID != NULL */
if (FailureDetected == TRUE)
{
return(FAILURE);
}
} /* END BLOCK (case mxoRM) */
break;
case mxoRsv:
{
enum MRsvCtlCmdEnum atype;
atype = (enum MRsvCtlCmdEnum)MUGetIndexCI(Action,MRsvCtlCmds,FALSE,mrcmNONE);
if (atype == mrcmNONE)
{
return(FAILURE);
}
switch (atype)
{
case mrcmModify:
{
char *ptr;
char *TokPtr;
enum MRsvAttrEnum attr;
mrsv_t *R;
if (!strcmp(OID,"-"))
{
if (T == NULL)
return(FAILURE);
R = (mrsv_t *)T->O;
}
else if (MRsvFind(OID,&R,mraNONE) == FAILURE)
{
return(FAILURE);
}
ptr = MUStrTok(ArgLine,":",&TokPtr);
attr = (enum MRsvAttrEnum)MUGetIndexCI(ptr,MRsvAttr,FALSE,mraNONE);
switch (attr)
{
case mraACL:
{
enum MObjectSetModeEnum Mode;
if (MUStrIsEmpty(TokPtr))
Mode = mSet;
else if (strstr(TokPtr,"-=") != NULL)
Mode = mDecr;
else if (strstr(TokPtr,"+=") != NULL)
Mode = mAdd;
else
Mode = mSet;
MRsvSetAttr(R,mraACL,(void *)TokPtr,mdfString,Mode);
}
break;
default:
return(FAILURE);
/*NOTREACHED*/
break;
}
} /* END case mrcmModify */
break;
case mrcmDestroy:
{
mrsv_t *R;
if (!strcmp(OID,"-"))
{
if (T == NULL)
return(FAILURE);
R = (mrsv_t *)T->O;
}
else if (MRsvFind(OID,&R,mraNONE) == FAILURE)
{
return(FAILURE);
}
R->CancelIsPending = TRUE;
R->EndTime = MSched.Time - 1;
R->ExpireTime = 0;
/* remove reservation */
MRsvCheckStatus(NULL);
if ((long)R->StartTime <= MUIDeadLine)
{
/* adjust UI phase wake up time */
if (MSched.AdminEventAggregationTime >= 0)
MUIDeadLine = MIN(MUIDeadLine,(long)MSched.Time + MSched.AdminEventAggregationTime);
}
} /* END case mrcmDestroy */
break;
case mrcmOther:
{
mrsv_t *R;
if (!strcmp(OID,"-"))
{
if (T == NULL)
{
return(FAILURE);
}
R = (mrsv_t *)T->O;
}
else if (MRsvFind(OID,&R,mraNONE) == FAILURE)
{
return(FAILURE);
}
/* parse argline */
if (strstr(ArgLine,"submit") != NULL)
{
MJobSubmitFromJSpec(R->SpecJ,NULL,Msg);
} /* END if (strstr(ArgLine,"submit") != NULL) */
} /* END BLOCK (case mrcmOther) */
break;
default:
return(FAILURE);
/*NOTREACHED*/
break;
} /* END switch (atype) */
} /* END BLOCK (case mxoRsv) */
break;
case mxoxVPC:
{
mpar_t *VPC;
MVPCFind(OID,&VPC,TRUE);
if (!strcasecmp(Action,"destroy"))
{
MVPCDestroy(&VPC,ArgLine);
}
else
{
return(FAILURE);
}
} /* END case mxoxVPC */
break;
default:
if (Msg != NULL)
strcpy(Msg,"unsupported trigger object type");
return(FAILURE);
/* NOTREACHED */
break;
} /* END switch (OType) */
/*NOTE: return may (and does) return in above switch statement */
return(SUCCESS);
} /* END MTrigLaunchInternalAction() */
int MVMCRFromXML(
mvm_req_create_t *VMCR, /* I (modified) */
mxml_t *E) /* I */
{
int aindex;
if ((VMCR == NULL) || (E == NULL))
{
return(FAILURE);
}
for (aindex = 0;aindex < E->ACount;aindex++)
{
if (!strcmp("ID",E->AName[aindex]))
{
MUStrCpy(VMCR->VMID,E->AVal[aindex],sizeof(VMCR->VMID));
}
else if (!strcmp("VARS",E->AName[aindex]))
{
MUStrCpy(VMCR->Vars,E->AVal[aindex],sizeof(VMCR->Vars));
}
else if (!strcmp("STORAGE",E->AName[aindex]))
{
MUStrCpy(VMCR->Storage,E->AVal[aindex],sizeof(VMCR->Storage));
}
else if (!strcmp("ALIAS",E->AName[aindex]))
{
MUStrCpy(VMCR->Aliases,E->AVal[aindex],sizeof(VMCR->Aliases));
}
else if (!strcmp("TRIGGER",E->AName[aindex]))
{
MUStrCpy(VMCR->Triggers,E->AVal[aindex],sizeof(VMCR->Triggers));
}
else if (!strcmp("HYPERVISOR",E->AName[aindex]))
{
/* Find node for hypervisor */
MNodeFind(E->AVal[aindex],&VMCR->N);
}
else if (!strcmp("TEMPLATE",E->AName[aindex]))
{
MJobFind(E->AVal[aindex],&VMCR->JT,mjsmExtended);
}
else if (!strcmp("OWNERJOB",E->AName[aindex]))
{
MJobFind(E->AVal[aindex],&VMCR->OwnerJob,mjsmExtended);
}
else if (!strcmp("IMAGE",E->AName[aindex]))
{
VMCR->OSIndex = MUMAGetIndex(meOpsys,E->AVal[aindex],mAdd);
}
else if (!strcmp("WALLTIME",E->AName[aindex]))
{
VMCR->Walltime = strtol(E->AVal[aindex],NULL,10);
}
else if (!strcmp("SOVEREIGN",E->AName[aindex]))
{
VMCR->IsSovereign = MUBoolFromString(E->AName[aindex],FALSE);
}
else if (!strcmp("ISONETIMEUSE",E->AName[aindex]))
{
VMCR->IsOneTimeUse = MUBoolFromString(E->AName[aindex],FALSE);
}
else if (!strcmp("DISK",E->AName[aindex]))
{
VMCR->CRes.Disk = (int)strtol(E->AVal[aindex],NULL,10);
}
else if (!strcmp("MEM",E->AName[aindex]))
{
VMCR->CRes.Mem = (int)strtol(E->AVal[aindex],NULL,10);
}
else if (!strcmp("PROCS",E->AName[aindex]))
{
VMCR->CRes.Procs = (int)strtol(E->AVal[aindex],NULL,10);
}
else if (!strcmp("TRACKINGJ",E->AName[aindex]))
{
MUStrCpy(VMCR->TrackingJ,E->AVal[aindex],sizeof(VMCR->TrackingJ));
}
} /* END for (aindex = 0;aindex < E->ACount;aindex++) */
return(SUCCESS);
} /* END MVMCRFromXML() */
int MVMToXML(
mvm_t *V, /* I */
mxml_t *VE, /* I/O (modified,alloc) */
enum MVMAttrEnum *SAList) /* I (optional,terminated w/mvmaLAST) */
{
char tmpString[MMAX_LINE];
enum MVMAttrEnum *AList; /**< VM attribute list */
int aindex;
mbitmap_t BM;
const enum MVMAttrEnum DAList[] = {
mvmaActiveOS,
mvmaADisk,
mvmaAlias,
mvmaAMem,
mvmaAProcs,
mvmaCDisk,
mvmaCMem,
mvmaCProcs,
mvmaCPULoad,
mvmaContainerNode,
mvmaDDisk,
mvmaDMem,
mvmaDProcs,
mvmaDescription,
mvmaEffectiveTTL,
mvmaFlags,
mvmaGMetric,
mvmaID,
mvmaJobID,
mvmaLastMigrateTime,
mvmaLastSubState,
mvmaLastSubStateMTime,
mvmaLastUpdateTime,
mvmaMigrateCount,
mvmaNetAddr,
mvmaNextOS,
mvmaOSList,
mvmaPowerIsEnabled,
mvmaPowerState,
mvmaProvData,
mvmaRackIndex,
mvmaSlotIndex,
mvmaSovereign,
mvmaSpecifiedTTL,
mvmaStartTime,
mvmaState,
mvmaSubState,
mvmaStorageRsvNames,
mvmaVariables,
mvmaLAST };
if ((V == NULL) || (VE == NULL))
{
return(FAILURE);
}
if (SAList != NULL)
AList = SAList;
else
AList = (enum MVMAttrEnum *)DAList;
bmset(&BM,mcmXML);
for (aindex = 0;AList[aindex] != mvmaLAST;aindex++)
{
if ((MVMAToString(V,NULL,AList[aindex],tmpString,sizeof(tmpString),&BM) == FAILURE) ||
(tmpString[0] == '\0'))
{
continue;
}
if (AList[aindex] == mvmaVariables)
{
MUAddVarsToXML(VE,&V->Variables);
continue;
}
MXMLSetAttr(VE,(char *)MVMAttr[AList[aindex]],tmpString,mdfString);
} /* END for (aindex) */
/* Set GEvents */
MGEventListToXML(VE,&V->GEventsList);
MVMStorageMountsToXML(VE,V);
/* Triggers */
if (V->T != NULL)
{
mtrig_t *T;
int tindex;
mxml_t *TE = NULL;
for (tindex = 0;tindex < V->T->NumItems;tindex++)
{
T = (mtrig_t *)MUArrayListGetPtr(V->T,tindex);
if (MTrigIsReal(T) == FALSE)
continue;
TE = NULL;
MXMLCreateE(&TE,(char *)MXO[mxoTrig]);
MTrigToXML(T,TE,NULL);
MXMLAddE(VE,TE);
}
} /* END if (V->T != NULL) */
if (V->Action != NULL)
{
mln_t *tmpL = NULL;
mjob_t *J;
mxml_t *AE;
/* report pending actions */
for (tmpL = V->Action;tmpL != NULL;tmpL = tmpL->Next)
{
if ((MJobFind(tmpL->Name,&J,mjsmExtended) == FAILURE) ||
(J->System == NULL))
{
continue;
}
AE = NULL;
MXMLCreateE(&AE,"action");
MXMLSetAttr(AE,"id",(void **)tmpL->Name,mdfString);
MXMLSetAttr(AE,"type",(void **)MSysJobType[J->System->JobType],mdfString);
if (MJOBISACTIVE(J) == TRUE)
MXMLSetAttr(AE,"starttime",(void **)&J->StartTime,mdfLong);
else if (J->Rsv != NULL)
MXMLSetAttr(AE,"starttime",(void **)&J->Rsv->StartTime,mdfLong);
MXMLSetAttr(AE,"duration",(void **)&J->SpecWCLimit[0],mdfLong);
switch (J->System->JobType)
{
case msjtVMMigrate:
if (!MNLIsEmpty(&J->Req[0]->NodeList))
{
MXMLSetAttr(AE,"destination",(void **)MNLGetNodeName(&J->Req[0]->NodeList,0,NULL),mdfString);
}
break;
case msjtOSProvision:
MXMLSetAttr(AE,"os",(void **)MAList[meOpsys][J->System->ICPVar1],mdfString);
break;
#if 0
case msjtVMCreate:
if (J->System->Action != NULL)
{
char *ptr;
if ((ptr = strstr(J->System->Action,"create:")) != NULL)
{
/* report new VM configuration */
ptr += strlen("create:");
MXMLSetAttr(AE,"config",(void **)ptr,mdfString);
}
}
break;
#endif
default:
/* NO-OP */
break;
} /* END switch (J->System->JobType) */
MXMLAddE(VE,AE);
} /* END for (tmpL = V->Action) */
} /* END if ((V->Action != NULL) && (V->Action[0] != NULL)) */
/* VMTracking job */
if (V->TrackingJ != NULL)
{
MXMLSetAttr(VE,"TrackingJob",(void **)V->TrackingJ->Name,mdfString);
}
return(SUCCESS);
} /* END MVMToXML() */
int CPRecordLostJobs(
char *CPFile)
{
int count;
char *Buffer;
char *head;
char *ptr;
char *tokptr;
char Line[MAX_MLINE];
int buflen;
mjob_t tmpJ;
mreq_t tmpRQ;
mjob_t *J;
char tmpLine[MAX_MLINE];
char JobName[MAX_MLINE];
DBG(3,fCKPT) DPrint("CPRecordLostJobs(%s)\n",
CPFile);
/* load checkpoint file */
if ((Buffer = MFULoad(CPFile,1,macmWrite,&count,NULL)) == NULL)
{
DBG(1,fCKPT) DPrint("WARNING: cannot load checkpoint file '%s'\n",
CPFile);
return(FAILURE);
}
head = Buffer;
buflen = strlen(head);
ptr = MUStrTok(head,"\n",&tokptr);
while (ptr != NULL)
{
head = ptr + strlen(ptr) + 1;
if ((head - Buffer) > buflen)
{
head = Buffer + buflen;
}
if (!strncmp(ptr,MCPType[mcpJob],strlen(MCPType[mcpJob])))
{
/* determine job completion time estimate */
sscanf(Line,"%s %s",
tmpLine,
JobName);
if (MJobFind(JobName,&J,0) == FAILURE)
{
memset(&tmpJ,0,sizeof(tmpJ));
memset(&tmpRQ,0,sizeof(tmpRQ));
/* FIXME: only one req handled in RecordLostJobs() */
tmpJ.Req[0] = &tmpRQ;
if (MJobLoadCP(&tmpJ,ptr) == SUCCESS)
{
/* create temp record */
tmpJ.State = mjsLost;
MJobWriteStats(&tmpJ);
}
}
} /* END if (!strcmp(ptr,MCPType[])) */
} /* END while (ptr != NULL) */
return(SUCCESS);
} /* END CPRecordLostJobs() */
int __MNLCheckNodeSkipFilter(
mnode_t *N,
enum MPowerStateEnum PState,
mbitmap_t *CFlags,
mbool_t *ShouldSkip,
const char **SkipReason)
{
enum MSystemJobTypeEnum JobType;
mjob_t *J;
/* map power request to a JobType state */
JobType = (PState == mpowOn) ? msjtPowerOn : msjtPowerOff;
/* Initialize returned values to default state */
*ShouldSkip = FALSE;
*SkipReason = "unknown";
/* check for current Node 'state to see if it matches request */
if (N->PowerState == PState)
{
*ShouldSkip = TRUE;
*SkipReason = ((PState == mpowOn) ? "already ON" : "already OFF");
}
else
{
/* Check for other constraints to skip this node */
mln_t *tmpL = NULL;
/* Iterate over all active Node System jobs and see:
* that they can be found AND they are a poweroff/poweron
* system job, that is the same as the current request.
* That means they are already 'inflight' and doing
* what we already want, therefore no need to re-issue at this time
*/
for (tmpL = N->Action;tmpL != NULL;tmpL = tmpL->Next)
{
if ((MJobFind(tmpL->Name,&J,mjsmBasic) == SUCCESS) &&
(MJobGetSystemType(J) == JobType))
{
*ShouldSkip = TRUE;
*SkipReason = ((PState == mpowOn) ? "power ON in process" : "power OFF in process");
break;
}
}
/* If a power OFF request && NOT in FORCED mode && not already skipped:
* need to check for Active VMs and Jobs on the node and if present skip them.
* If in FORCED mode, then include busy nodes in the power down operation
*/
if ((mpowOff == PState) &&
((CFlags != NULL) && bmisset(CFlags,mcmForce)) &&
(*ShouldSkip == FALSE))
{
/* Check for VMs on the node, if any, filter this node out */
if ((N->VMList != NULL) && (MULLGetCount(N->VMList) > 0))
{
*ShouldSkip = TRUE;
*SkipReason = "has active VMs running";
}
else
{
/* Now check for active non-VM jobs on the node */
if (SUCCESS == MNodeCheckAllocation(N))
{
*ShouldSkip = TRUE;
*SkipReason = "has active jobs running";
}
}
} /* END if ((*ShouldSkip == FALSE) && (mpowOff == PState)) */
} /* END if(N->PowerState == PState) */
return(SUCCESS);
} /* END __MNLCheckNodeSkipFilter() */
int MRMToXML(
mrm_t *R, /* I */
mxml_t **RMEP, /* O */
enum MRMAttrEnum *SAList) /* I (optional) */
{
enum MRMAttrEnum DAList[] = {
mrmaAuthType,
mrmaAuthAList,
mrmaAuthCList,
mrmaAuthGList,
mrmaAuthQList,
mrmaAuthUList,
mrmaCheckpointSig,
mrmaCheckpointTimeout,
mrmaClusterQueryURL,
mrmaConfigFile,
mrmaCredCtlURL,
mrmaCSAlgo,
mrmaDataRM,
mrmaDefDStageInTime,
mrmaDefHighSpeedAdapter,
mrmaDataRM,
mrmaEPort,
mrmaFlags,
mrmaHost,
mrmaJobCount,
mrmaJobCounter,
mrmaJobStartCount,
mrmaLocalDiskFS,
/* mrmaMessages, - msgs are currently not safe for XML (need to update like job msgs) */
mrmaMinETime,
mrmaName,
mrmaNC,
mrmaNMPort,
mrmaNMServer,
mrmaNodeFailureRsvProfile,
mrmaNodeList,
mrmaOMap,
mrmaPartition,
mrmaPort,
mrmaProcs,
mrmaProfile,
mrmaResourceType,
mrmaSocketProtocol,
mrmaSQLData,
mrmaState,
mrmaStats,
mrmaSubmitCmd,
mrmaSuspendSig,
mrmaTargetUsage,
mrmaTimeout,
mrmaType,
mrmaUpdateTime,
mrmaVersion,
mrmaVMOwnerRM,
mrmaWireProtocol,
mrmaWorkloadQueryURL,
mrmaNONE };
int aindex;
int pindex;
enum MRMAttrEnum *AList;
char tmpBuffer[MMAX_BUFFER];
mxml_t *RME;
if ((R == NULL) || (RMEP == NULL))
{
return(FAILURE);
}
if ((*RMEP == NULL) && (MXMLCreateE(RMEP,(char *)MXO[mxoRM]) == FAILURE))
{
return(FAILURE);
}
RME = *RMEP;
if (SAList != NULL)
AList = SAList;
else
AList = DAList;
for (aindex = 0;AList[aindex] != mrmaNONE;aindex++)
{
if ((MRMAToString(R,AList[aindex],tmpBuffer,sizeof(tmpBuffer),0) == FAILURE) ||
(tmpBuffer[0] == '\0'))
{
continue;
}
MXMLSetAttr(
RME,
(char *)MRMAttr[AList[aindex]],
(void *)tmpBuffer,
mdfString);
} /* END for (aindex) */
/* checkpoint local queue information */
if (bmisset(&R->IFlags,mrmifLocalQueue) &&
(MSched.LoadAllJobCP == FALSE) &&
(MUHTIsInitialized(&R->U.S3.JobListIndex) == SUCCESS))
{
char *tmpBuf;
int tmpBufLen;
mhashiter_t JobListIter;
char *JobName = NULL;
mjob_t *J;
mxml_t *QE = NULL;
MUHTIterInit(&JobListIter);
MXMLCreateE(&QE,(char *)MXO[mxoQueue]);
tmpBuf = NULL;
tmpBufLen = 0;
while (MUHTIterate(&R->U.S3.JobListIndex,&JobName,NULL,NULL,&JobListIter) == SUCCESS)
{
if (MJobFind(JobName,&J,mjsmBasic) == SUCCESS)
{
if (MJobShouldCheckpoint(J,R) == FALSE)
continue;
}
MUStrAppend(&tmpBuf,&tmpBufLen,JobName,','); /* consider using MUSNXPrintF to decrease heap fragmentation */
}
MXMLSetAttr(QE,(char *)MXO[mxoJob],tmpBuf,mdfString);
MXMLAddE(RME,QE);
MUFree(&tmpBuf);
} /* END if ((bmisset(&R->IFlags,mrmifLocalQueue)) && ...) */
/* create peer interface child */
for (pindex = 0;pindex < MMAX_RMSUBTYPE;pindex++)
{
mxml_t *PE;
if (bmisset(&R->RTypes,mrmrtLicense) ||
bmisset(&R->RTypes,mrmrtNetwork) ||
bmisset(&R->RTypes,mrmrtProvisioning) ||
bmisset(&R->RTypes,mrmrtStorage))
{
/* non-compute RM's may not have interface type specified since they
may route directly through file or CLI interfaces */
if (R->P[pindex].RespTotalCount[0] <= 0)
continue;
}
else
{
if (R->P[pindex].Type == mpstNONE)
continue;
}
PE = NULL;
MXMLCreateE(&PE,"psi");
MPSIToXML(&R->P[pindex],PE);
MXMLAddE(RME,PE);
} /* END for (pindex) */
if (bmisset(&R->RTypes,mrmrtLicense))
{
int rindex;
int IC;
double tmpD;
double totalCfg;
double totalDed;
mxml_t *GRE;
IC = MAX(1,R->U.Nat.ICount);
totalCfg = 0.0;
totalDed = 0.0;
for (rindex = 0;rindex < MSched.M[mxoxGRes];rindex++)
{
if ((R->U.Nat.ResName == NULL) ||
(R->U.Nat.ResName[rindex][0] == '\0'))
break;
/* Only include license information that applies to this resource manager */
if (MSNLGetIndexCount(&R->U.Nat.ResConfigCount,rindex) == 0)
continue;
GRE = NULL;
MXMLCreateE(&GRE,"license");
MXMLSetAttr(GRE,"name",(void *)R->U.Nat.ResName[rindex],mdfString);
MXMLSetAttr(GRE,"ic",(void *)&IC,mdfInt);
tmpD = (double)(MSNLGetIndexCount(&R->U.Nat.ResAvailCount,rindex)) / IC;
MXMLSetAttr(GRE,"averageAvail",(void *)&tmpD,mdfDouble);
tmpD = (double)(MSNLGetIndexCount(&R->U.Nat.ResConfigCount,rindex)) / IC;
MXMLSetAttr(GRE,"averageConfig",(void *)&tmpD,mdfDouble);
/* NOTE: percentages below are over all iterations since Moab started */
tmpD = (double)(MSNLGetIndexCount(&R->U.Nat.ResFreeICount,rindex)) / IC;
MXMLSetAttr(GRE,"percentIdle",(void *)&tmpD,mdfDouble);
tmpD = (double)(MSNLGetIndexCount(&R->U.Nat.ResAvailICount,rindex) - MSNLGetIndexCount(&R->U.Nat.ResFreeICount,rindex)) / IC;
MXMLSetAttr(GRE,"percentActive",(void *)&tmpD,mdfDouble);
tmpD = (double)(R->U.Nat.ICount - MSNLGetIndexCount(&R->U.Nat.ResAvailICount,rindex)) / IC;
MXMLSetAttr(GRE,"percentBusy",(void *)&tmpD,mdfDouble);
if (R->U.Nat.N != NULL)
{
mnode_t *GN;
GN = (mnode_t *)R->U.Nat.N;
tmpD = (double)(MSNLGetIndexCount(&GN->CRes.GenericRes,rindex));
MXMLSetAttr(GRE,"currentConfig",(void *)&tmpD,mdfDouble);
tmpD = (double)(MSNLGetIndexCount(&GN->ARes.GenericRes,rindex));
totalCfg += tmpD;
MXMLSetAttr(GRE,"currentAvail",(void *)&tmpD,mdfDouble);
tmpD = (double)(MSNLGetIndexCount(&GN->DRes.GenericRes,rindex));
MXMLSetAttr(GRE,"currentDedicated",(void *)&tmpD,mdfDouble);
tmpD = (double)(MSNLGetIndexCount(&GN->DRes.GenericRes,rindex));
totalDed += tmpD;
}
MXMLAddE(RME,GRE);
} /* END for (rindex) */
tmpD = totalDed / MAX(1.0,totalCfg);
MXMLSetAttr(RME,"LOAD",(void *)&tmpD,mdfDouble);
} /* END if (bmisset(&R->RTypes,mrmrtLicense)) */
if (bmisset(&R->RTypes,mrmrtNetwork))
{
if (!MNLIsEmpty(&R->NL))
{
double tmpD;
mnode_t *N;
int gindex;
MNLGetNodeAtIndex(&R->NL,0,&N);
gindex = MUMAGetIndex(
meGRes,
"bandwidth",
mVerify);
if (gindex > 0)
{
tmpD = MSNLGetIndexCount(&N->ARes.GenericRes,gindex) / MAX(1.0,MSNLGetIndexCount(&N->CRes.GenericRes,gindex));
tmpD = 1.0 - tmpD;
MXMLSetAttr(RME,"LOAD",(void *)&tmpD,mdfDouble);
}
}
} /* END if (bmisset(&R->RTypes,mrmrtNetwork)) */
if (bmisset(&R->RTypes,mrmrtProvisioning))
{
/* report ratio of I/O (ie, install) capacity currently in use */
/* NYI */
}
if (bmisset(&R->RTypes,mrmrtStorage))
{
if (!MNLIsEmpty(&R->NL))
{
double tmpD;
mnode_t *N;
MNLGetNodeAtIndex(&R->NL,0,&N);
tmpD = N->ARes.Disk / MAX(1.0,N->CRes.Disk);
tmpD = 1.0 - tmpD;
MXMLSetAttr(RME,"LOAD",(void *)&tmpD,mdfDouble);
} /* END if (bmisset(&R->RTypes,mrmrtStorage)) */
}
return(SUCCESS);
} /* END MRMToXML() */
int MJobProcessTVariables(
mjob_t *J) /* I */
{
mjob_t *MasterJ = NULL;
mhashiter_t HTI;
char *VarName;
char *VarVal;
mbitmap_t VarBM;
char NamesToAdd[MMAX_TEMP_VARS][MMAX_LINE];
char ValuesToAdd[MMAX_TEMP_VARS][MMAX_LINE];
char NamesToAddMaster[MMAX_TEMP_VARS][MMAX_LINE];
char ValuesToAddMaster[MMAX_TEMP_VARS][MMAX_LINE];
mbitmap_t *BMToAddMaster[MMAX_TEMP_VARS];
int AddIndex;
int AddIndexMaster;
int tmpI;
char tmpLine[MMAX_LINE];
char *SetVal = NULL;
mbool_t DoVarExport = FALSE;
mbool_t NoGroupExport = FALSE; /* don't set the variable to export at the group level */
const char *FName = "MJobProcessTVariables";
MDB(3,fSCHED) MLog("%s(%s)\n",
FName,
J->Name);
if (MUHTIsEmpty(&J->Variables))
{
return(SUCCESS);
}
memset(NamesToAdd,0,sizeof(NamesToAdd));
memset(ValuesToAdd,0,sizeof(ValuesToAdd));
AddIndex = 0;
memset(NamesToAddMaster,0,sizeof(NamesToAddMaster));
memset(ValuesToAddMaster,0,sizeof(ValuesToAddMaster));
memset(BMToAddMaster,0,sizeof(BMToAddMaster));
AddIndexMaster = 0;
MUHTIterInit(&HTI);
while (MUHTIterate(&J->Variables,&VarName,(void **)&VarVal,&VarBM,&HTI) == SUCCESS)
{
DoVarExport = FALSE;
NoGroupExport = FALSE;
MDB(7,fSCHED) MLog("INFO: processing variable '%s'\n",
VarName);
SetVal = VarVal;
if (bmisset(&VarBM,mtvaExport))
DoVarExport = TRUE;
if (bmisset(&VarBM,mtvaIncr))
{
if (VarVal != NULL)
{
tmpI = (int)strtol((char *)VarVal,NULL,10);
tmpI++;
}
else
{
tmpI = 1;
}
snprintf(tmpLine,sizeof(tmpLine),"%d",tmpI);
SetVal = tmpLine;
if (AddIndex < MMAX_TEMP_VARS)
{
MUStrCpy(NamesToAdd[AddIndex],VarName,MMAX_LINE);
MUStrCpy(ValuesToAdd[AddIndex],SetVal,MMAX_LINE);
AddIndex++;
}
MJobWriteVarStats(J,mSet,VarName,SetVal,NULL);
}
else if ((!strcasecmp(VarName,"ccode")) &&
(MJOBISCOMPLETE(J) == TRUE))
{
DoVarExport = TRUE;
NoGroupExport = TRUE;
snprintf(tmpLine,sizeof(tmpLine),"%d",
J->CompletionCode);
SetVal = tmpLine;
if (AddIndex < MMAX_TEMP_VARS)
{
MUStrCpy(NamesToAdd[AddIndex],VarName,MMAX_LINE);
MUStrCpy(ValuesToAdd[AddIndex],SetVal,MMAX_LINE);
AddIndex++;
}
MJobWriteVarStats(J,mSet,VarName,SetVal,NULL);
#ifdef MYAHOO
#define MPBS_STDFILE_ERROR_CODE -9
/* check to see if we have a special completion code
* that should result in an error message */
if (J->CompletionCode == MPBS_STDFILE_ERROR_CODE)
{
mjob_t *JGroup;
char tmpLine[MMAX_LINE];
mstring_t tmpString(MMAX_LINE);
MJobAToMString(
J,
mjaAllocNodeList,
&tmpString,
mfmNONE);
snprintf(tmpLine,sizeof(tmpLine),"job '%s' failed to create stdout/stderr files on '%s'",
J->Name,
tmpString.c_str());
/* there has been an error setting the stdout/stderr file */
MMBAdd(
&J->MessageBuffer,
tmpLine,
NULL,
mmbtOther,
MSched.Time + MCONST_DAYLEN,
0,
NULL);
if (MJobGetGroupJob(J,&JGroup) == SUCCESS)
{
MMBAdd(
&JGroup->MessageBuffer,
tmpLine,
NULL,
mmbtOther,
MSched.Time + MCONST_DAYLEN,
0,
NULL);
}
}
#endif /* MYAHOO */
}
else if (!strcmp(VarName,"HOSTLIST") &&
(MJOBISCOMPLETE(J) == TRUE))
{
mstring_t tmp(MMAX_LINE);
if (MJobAToMString(
J,
mjaAllocNodeList,
&tmp) == SUCCESS)
{
/* can succeed, but not populate outgoing buffer */
if (!tmp.empty())
{
DoVarExport = TRUE;
NoGroupExport = TRUE;
if (AddIndex < MMAX_TEMP_VARS)
{
MUStrCpy(NamesToAdd[AddIndex],VarName,MMAX_LINE);
MUStrCpy(ValuesToAdd[AddIndex],tmp.c_str(),MMAX_LINE);
AddIndex++;
}
MJobWriteVarStats(J,mSet,VarName,tmp.c_str(),NULL);
}
}
}
if (bmisset(&VarBM,mtvaNoExport))
{
/* override our desire to export this variable if it
* is explicitly stated that this variable should NOT
* be exported */
DoVarExport = FALSE;
}
/* export variable to group job */
if ((DoVarExport == TRUE) &&
(J->JGroup != NULL))
{
if (MJobFind(J->JGroup->Name,&MasterJ,mjsmExtended) == SUCCESS)
{
mbitmap_t GroupBM;
char *GroupVal;
if ((bmisset(&VarBM,mtvaExport) && bmisset(&VarBM,mtvaIncr)) &&
(MUHTGet(&MasterJ->Variables,VarName,(void **)&GroupVal,&GroupBM) == SUCCESS))
{
if (GroupVal != NULL)
{
tmpI = (int)strtol(GroupVal,NULL,10);
tmpI++;
}
else
{
tmpI = 1;
}
snprintf(tmpLine,sizeof(tmpLine),"%d",tmpI);
SetVal = tmpLine;
bmset(&GroupBM,mtvaExport);
}
else if (NoGroupExport == TRUE)
{
/* prevent this variable from exporting up, further
* out of the group object */
bmset(&GroupBM,mtvaNoExport);
}
else if (DoVarExport == TRUE)
{
bmset(&GroupBM,mtvaExport);
}
if (AddIndexMaster < MMAX_TEMP_VARS)
{
MUStrCpy(NamesToAddMaster[AddIndexMaster],VarName,MMAX_LINE);
MUStrCpy(ValuesToAddMaster[AddIndexMaster],SetVal,MMAX_LINE);
BMToAddMaster[AddIndexMaster] = &GroupBM;
AddIndexMaster++;
}
MJobWriteVarStats(MasterJ,mSet,VarName,SetVal,NULL);
} /* END if ((MJobFind(J->JGroup) || ... */
} /* END if ((J->JGroup != NULL)... */
} /* END while (MUHTIterate()) */
if (!MUStrIsEmpty(NamesToAdd[0]))
{
for (AddIndex = 0;AddIndex < MMAX_TEMP_VARS;AddIndex++)
{
if (MUStrIsEmpty(NamesToAdd[AddIndex]))
break;
MUHTAdd(&J->Variables,NamesToAdd[AddIndex],strdup(ValuesToAdd[AddIndex]),NULL,MUFREE);
}
} /* END if (!MUStrIsEmpty(NamesToAdd[0])) */
if ((MasterJ != NULL) && !MUStrIsEmpty(NamesToAddMaster[0]))
{
for (AddIndexMaster = 0;AddIndexMaster < MMAX_TEMP_VARS;AddIndexMaster++)
{
if (MUStrIsEmpty(NamesToAddMaster[AddIndexMaster]))
break;
MUHTAdd(&J->Variables,NamesToAddMaster[AddIndexMaster],strdup(ValuesToAddMaster[AddIndexMaster]),BMToAddMaster[AddIndexMaster],MUFREE);
}
}
/* checkpoint jobs to ensure we don't lose newly updated variables in a crash */
MOCheckpoint(mxoJob,(void *)J,FALSE,NULL);
MOCheckpoint(mxoJob,(void *)MasterJ,FALSE,NULL);
return(SUCCESS);
} /* END MJobProcessTVariables() */
int MJobProcessTerminated(
mjob_t *J, /* I (modified) */
enum MJobStateEnum JState) /* I */
{
if (J == NULL)
{
return(FAILURE);
}
MSched.EnvChanged = TRUE;
if (J->Cost == 0.0)
MJobGetCost(J,FALSE,NULL);
if (MSched.PushEventsToWebService == TRUE)
{
MEventLog *Log = new MEventLog(meltJobEnd);
Log->SetCategory(melcEnd);
Log->SetFacility(melfJob);
Log->SetPrimaryObject(J->Name,mxoJob,(void *)J);
if (JState == mjsRemoved)
{
/* Job was unsuccessful */
Log->SetStatus("failure");
}
MEventLogExportToWebServices(Log);
delete Log;
} /* END if (MSched.PushEventsToWebService == TRUE) */
MJobUpdateFailRate(J);
/* update array job information */
if (bmisset(&J->Flags,mjfArrayJob))
{
MJobArrayUpdateJobInfo(J,JState);
}
/* Update any before dependencies */
if (J->Depend != NULL)
{
mdepend_t *JobDep;
mbool_t UpdateDep;
JobDep = J->Depend;
while (JobDep != NULL)
{
UpdateDep = FALSE;
switch (JobDep->Type)
{
case mdJobBefore:
/* Taken care of in MJobStart */
break;
case mdJobBeforeAny:
UpdateDep = TRUE;
break;
case mdJobBeforeSuccessful:
if (J->CompletionCode == 0)
UpdateDep = TRUE;
break;
case mdJobBeforeFailed:
if (J->CompletionCode != 0)
UpdateDep = TRUE;
break;
default:
break;
}
if (UpdateDep == TRUE)
{
mjob_t *OtherJ;
if (MJobFind(JobDep->Value,&OtherJ,mjsmExtended) == FAILURE)
{
/* Couldn't find the dependent job */
MDB(4,fSTAT) MLog("WARNING: could not find job '%s' to update %s dependency for job '%s'\n",
JobDep->Value,
MDependType[JobDep->Type],
J->Name);
}
else
{
/* Job Found, update any onCount dependencies */
mdepend_t *OtherDep;
OtherDep = OtherJ->Depend;
while (OtherDep != NULL)
{
char *UsedJobs = NULL;
if (OtherDep->Type != mdJobOnCount)
{
OtherDep = OtherDep->NextAnd;
continue;
}
if (OtherDep->DepList == NULL)
{
/* ctor the mstring_t */
OtherDep->DepList = new mstring_t(MMAX_LINE);
OtherDep->ICount = strtol(OtherDep->Value,NULL,10);
/* 0 is error in strtol */
if (OtherDep->ICount == 0)
{
OtherDep = OtherDep->NextAnd;
continue;
}
}
MUStrDup(&UsedJobs,OtherDep->DepList->c_str());
if ((UsedJobs != NULL) &&
(MUStrRemoveFromList(UsedJobs,J->Name,',',FALSE) != FAILURE))
{
/* Job dependency was already reported */
OtherDep = OtherDep->NextAnd;
continue;
}
*OtherDep->DepList += ',';
*OtherDep->DepList += J->Name;
OtherDep->ICount--;
if (UsedJobs != NULL)
MUFree(&UsedJobs);
OtherDep = OtherDep->NextAnd;
} /* END while (OtherDep != NULL) */
} /* END else (MJobFind returned SUCCESS) */
} /* END if (UpdateDep == TRUE) */
JobDep = JobDep->NextAnd;
}
}
/* If job is a VM destroy job, removed blocking flags on VM */
__MJobProcessVMDestroy(J);
/* Copy any VC variables to the VC. This is so that pending actions
will still have access to variables at the service level. This may
be removed later once pending actions are fully designed with the
workflow model */
MVCGrabVarsInHierarchy(J->ParentVCs,&J->Variables,TRUE);
if (MSched.EnableHighThroughput == FALSE)
MJobTransition(J,TRUE,FALSE);
return(SUCCESS);
} /* END MJobProcessTerminated() */
int __MJobProcessVMDestroy(
mjob_t *J)
{
mvm_t *VM = NULL;
if (J == NULL)
return(FAILURE);
if (J->System == NULL)
return(SUCCESS);
/* See if this job is a VM destroy job. If so, find the VM that it is trying
to destroy */
#if 0
if (J->System->JobType == msjtVMDestroy)
{
MVMFind(J->System->VM,&VM);
}
else if (J->System->JobType == msjtGeneric)
#endif
if (J->System->JobType == msjtGeneric)
{
mjob_t *OtherJob = NULL;
if ((J->TemplateExtensions == NULL) ||
(J->TemplateExtensions->JobReceivingAction == NULL) ||
(J->TemplateExtensions->TJobAction != mtjatDestroy))
{
/* This job is not a destroy job */
return(SUCCESS);
}
MJobFind(J->TemplateExtensions->JobReceivingAction,&OtherJob,mjsmBasic);
if ((OtherJob == NULL) ||
(OtherJob->System == NULL) ||
(OtherJob->System->VM == NULL))
{
/* Other job is not a VMTracking job */
return(SUCCESS);
}
MVMFind(OtherJob->System->VM,&VM);
/* Clear flag on the other job */
bmunset(&OtherJob->Flags,mjfDestroyTemplateSubmitted);
bmunset(&OtherJob->SpecFlags,mjfDestroyTemplateSubmitted);
}
else
{
return(SUCCESS);
}
if (VM == NULL)
return(FAILURE);
/* Found the VM, now clear any destroy flags with it */
bmunset(&VM->Flags,mvmfDestroyed);
bmunset(&VM->Flags,mvmfDestroyPending);
bmunset(&VM->Flags,mvmfDestroySubmitted);
return(SUCCESS);
} /* END __MJobProcessVMDestroy() */
int MJobProcessCompleted(
mjob_t **JP) /* I (freed on success, if job is not restarted) */
{
int nindex;
int rqindex;
enum MS3CodeDecadeEnum tmpS3C;
char Line[MMAX_LINE];
mnode_t *N;
mreq_t *RQ;
mjob_t *J;
char Message[MMAX_LINE];
const char *FName = "MJobProcessCompleted";
MDB(2,fSTAT) MLog("%s(%s)\n",
FName,
((JP != NULL) && (*JP != NULL)) ? (*JP)->Name : "NULL");
if ((JP == NULL) || (*JP == NULL))
{
return(FAILURE);
}
J = *JP;
if ((J->StartTime > 0) && (J->CompletionTime > J->StartTime))
{
long ComputedWallTime = J->CompletionTime - J->StartTime - J->SWallTime;
/* sanity check walltime */
if (J->AWallTime != ComputedWallTime)
{
MDB(7,fSCHED) MLog("INFO: job %s setting walltime to %ld from %ld in function %s\n",
J->Name,
ComputedWallTime,
J->AWallTime,
__FUNCTION__);
J->AWallTime = ComputedWallTime;
}
J->AWallTime = MAX(J->AWallTime,(long)J->Req[0]->RMWTime);
}
MJobProcessTerminated(J,mjsCompleted);
if (bmisset(&J->IFlags,mjifWasCanceled) &&
bmisset(&J->NotifyBM,mntJobFail) &&
(J->Credential.U->EMailAddress != NULL))
{
char tmpLine[MMAX_LINE];
mstring_t Msg(MMAX_LINE);
snprintf(tmpLine,sizeof(tmpLine),"Moab job '%s' was canceled",J->Name);
Msg = tmpLine;
MStringAppend(&Msg,"\n\n========== output of checkjob ==========\n\n");
MUICheckJob(J,&Msg,mptHard,NULL,NULL,NULL,NULL,0);
MSysSendMail(J->Credential.U->EMailAddress,NULL,tmpLine,NULL,Msg.c_str());
}
/* set/incr job variables (usually to be used by triggers) */
MJobProcessTVariables(J);
if (MAMHandleEnd(&MAM[0],(void *)J,mxoJob,Message,&tmpS3C) == FAILURE)
{
MDB(1,fSTAT) MLog("ERROR: Unable to register job end with accounting manager for job '%s'\n",
J->Name);
sprintf(Line,"AMFAILURE: Unable to register job end with accounting manager for job %s, reason %s (%s)\n",
J->Name,
(tmpS3C > ms3cNone && tmpS3C < ms3cLAST && MS3CodeDecade[tmpS3C] != NULL) ? MS3CodeDecade[tmpS3C] : "Unknown Failure",
Message);
MSysRegEvent(Line,mactNONE,1);
}
if ((J->TemplateExtensions != NULL) &&
(J->TemplateExtensions->TJobAction != mtjatNONE) &&
(J->TemplateExtensions->JobReceivingAction != NULL))
{
mjob_t *JobReceivingAction = NULL;
if (MJobFind(J->TemplateExtensions->JobReceivingAction,&JobReceivingAction,mjsmBasic) == SUCCESS)
{
switch (J->TemplateExtensions->TJobAction)
{
case mtjatDestroy:
if ((J->CompletionCode == 0) && (J->CompletionTime > 0))
{
char tmpMsg[MMAX_LINE];
snprintf(tmpMsg,sizeof(tmpMsg),"%s cancelled by destroy action job %s\n",
JobReceivingAction->Name,
J->Name);
bmset(&JobReceivingAction->IFlags,mjifDestroyByDestroyTemplate);
MJobCancel(JobReceivingAction,tmpMsg,FALSE,NULL,NULL);
}
else
{
/* Job failed, need to clear the flag on job so that we can resubmit
the destroy job */
bmunset(&JobReceivingAction->Flags,mjfDestroyTemplateSubmitted);
bmunset(&JobReceivingAction->SpecFlags,mjfDestroyTemplateSubmitted);
}
break;
default:
/* NO-OP */
break;
} /* END switch (J->TX->TJobAction) */
} /* END if (MJobFind(J->TX->JobReceivingAction) == SUCCESS) */
} /* END if ((J->TX != NULL) && ...) */
/* determine if job should be restarted due to policies */
/* (should other completion events happen before this occurs?) */
if ((MSched.JobFailRetryCount > 0) &&
(J->CompletionCode != 0) &&
!bmisset(&J->IFlags,mjifWasCanceled))
{
char EMsg[MMAX_LINE];
if (J->StartCount < MSched.JobFailRetryCount)
{
/* restart/requeue job */
MJobSetState(J,mjsStarting);
MJobRequeue(J,NULL,NULL,EMsg,NULL);
return(SUCCESS);
}
else if (MSched.JobFailRetryCount > 1)
{
/* only requeue the job with a hold if the user has enabled a
* JobFailRetryCount > 1 */
MJobSetState(J,mjsStarting);
MJobRequeue(J,NULL,NULL,EMsg,NULL);
/* update Moab's in-memory state */
MJobSetHold(J,mhUser,MMAX_TIME,mhrPolicyViolation,"restart count violation");
/* tell RM about change in hold */
if ((J->DestinationRM != NULL) &&
(J->DestinationRM->Type == mrmtPBS))
{
MRMJobModify(J,"hold",NULL,"user",mSet,NULL,NULL,NULL);
}
return(SUCCESS);
}
}
if (MJOBISCOMPLETE(J) == FALSE)
{
if (MJOBISACTIVE(J) == TRUE)
MJobSetState(J,mjsCompleted);
else if ((MJobIsArrayMaster(J) == TRUE) && (bmisset(&J->IFlags,mjifWasCanceled)))
MJobSetState(J,mjsRemoved);
else if (MJobIsArrayMaster(J) == TRUE)
/* Array master's state is always idle. Will need to handle case where
* one sub job fails so that the master fails as well in order fulfill
* job array dependencies. */
MJobSetState(J,mjsCompleted);
else
MJobSetState(J,mjsVacated);
}
#ifdef __NCSA
if (J->CompletionTime == J->StartTime)
{
sprintf(Line,"WARNING: job '%s' completed with a walltime of 0 seconds\n",
J->Name);
MSysRegEvent(Line,mactMail,0,1);
}
#endif /* __NCSA */
if (MPar[0].BFChunkDuration > 0)
{
MPar[0].BFChunkBlockTime = MSched.Time + MPar[0].BFChunkDuration;
}
if (J->Triggers != NULL)
{
if (J->CompletionCode != 0)
{
MOReportEvent(
(void *)J,
J->Name,
mxoJob,
mttFailure,
(J->CompletionTime > 0) ? J->CompletionTime : MSched.Time,
TRUE);
}
else
{
MOReportEvent(
(void *)J,
J->Name,
mxoJob,
mttEnd,
(J->CompletionTime > 0) ? J->CompletionTime : MSched.Time,
TRUE);
}
MSchedCheckTriggers(J->Triggers,-1,NULL);
}
/* provide feedback info to user */
MJobSendFB(J);
/* handle statistics */
MStatUpdateCompletedJobUsage(J,msmNONE,0);
/* create job stat record */
MOWriteEvent((void *)J,mxoJob,mrelJobComplete,NULL,MStat.eventfp,NULL);
/* modify node expected state and dedicated resources */
/* NOTE: for mjfRsvMap jobs should restore N->ARes (NYI) */
/*
TC = RQ->NodeList[nindex].TC;
MCResRemove(&N->ARes,&N->CRes,&RQ->DRes,TC,TRUE);
*/
nindex = 0;
for (rqindex = 0;J->Req[rqindex] != NULL;rqindex++)
{
RQ = J->Req[rqindex];
for (nindex = 0;MNLGetNodeAtIndex(&RQ->NodeList,nindex,&N) == SUCCESS;nindex++)
{
if (bmisset(&J->Flags,mjfRsvMap))
MCResAdd(&N->ARes,&N->CRes,&RQ->DRes,MNLGetTCAtIndex(&RQ->NodeList,nindex),TRUE);
MDB(3,fSTAT) MLog("INFO: node '%s' released from job %s\n",
N->Name,
J->Name);
/* determine new expected state for node */
if (MNodeCheckAllocation(N) != SUCCESS)
{
N->EState = mnsIdle;
}
else
{
N->EState = mnsActive;
}
if (N->EState == N->State)
N->SyncDeadLine = MMAX_TIME;
else
N->SyncDeadLine = MSched.Time + MSched.NodeSyncDeadline;
} /* END for (nindex) */
} /* END for (rqindex) */
MJobRemoveCP(J);
/* modify expected job state */
J->EState = J->State;
MJobCleanupForRemoval(J);
/* transition to MCJobHT table */
if (MQueueAddCJob(J) == FAILURE)
{
MJobRemove(JP);
}
return(SUCCESS);
} /* END MJobProcessCompleted() */
int MJobProcessRestrictedAttrs(
mjob_t *J, /* I */
char *EMsg) /* O */
{
const char *FName = "MJobProcessRestrictedAttrs";
MDB(5,fSCHED) MLog("%s(%s)\n",
FName,
(J != NULL) ? J->Name : NULL);
if ((J->Triggers != NULL) && (J->TemplateExtensions == NULL))
{
/* triggers can come from other places (ie. CLASSCFG[] JOBTRIGGERS) */
MOReportEvent((void *)J,J->Name,mxoJob,mttCreate,J->CTime,TRUE);
return(SUCCESS);
}
else if ((J->Triggers != NULL) && (J->TemplateExtensions != NULL))
{
MOReportEvent((void *)J,J->Name,mxoJob,mttCreate,J->CTime,TRUE);
}
if (J->TemplateExtensions == NULL)
{
return(SUCCESS);
}
if (J->TemplateExtensions->SpecTrig[0] != NULL)
{
char tmpBuf[MMAX_BUFFER];
int VarValBufSize;
int index;
mln_t *VarList = NULL;
for (index = 0;index < MDEF_VARVALBUFCOUNT;index++)
{
tmpVarValBuf2[index][0] = '\0';
}
if ((J->Credential.Q == NULL) || (!bmisset(&J->Credential.Q->Flags,mqfTrigger)))
{
MMBAdd(
&J->MessageBuffer,
"trigger specified but not allowed by QOS",
NULL,
mmbtOther,
MSched.Time + MCONST_DAYLEN,
0,
NULL);
}
else
{
int tindex;
marray_t HashList;
/* NOTE: loop through all elements in linked list (NYI) */
VarValBufSize = MDEF_VARVALBUFCOUNT - 1;
MJobGetVarList(
J,
&VarList,
tmpVarValBuf2,
&VarValBufSize);
MUArrayListCreate(&HashList,sizeof(mhash_t *),4);
if (J->Variables.Table != NULL)
{
MUArrayListAppendPtr(&HashList,&J->Variables);
}
if (J->JGroup != NULL)
{
mjob_t *tmpJ = NULL;
if (MJobFind(J->JGroup->Name,&tmpJ,mjsmExtended) == SUCCESS)
{
if (tmpJ->Variables.Table != NULL)
{
MUArrayListAppendPtr(&HashList,&tmpJ->Variables);
}
}
}
for (tindex = 0;tindex < MMAX_SPECTRIG;tindex++)
{
if (J->TemplateExtensions->SpecTrig[tindex] == NULL)
break;
MUInsertVarList(
J->TemplateExtensions->SpecTrig[tindex],
VarList, /* var name array */
(mhash_t **)HashList.Array, /* variables in hashtables */
NULL,
tmpBuf, /* O */
sizeof(tmpBuf),
TRUE);
MUArrayListFree(&HashList);
if (MTrigLoadString(&J->Triggers,tmpBuf,TRUE,FALSE,mxoJob,J->Name,NULL,EMsg) == FAILURE)
{
return(FAILURE);
}
MUFree(&J->TemplateExtensions->SpecTrig[tindex]);
} /* END for (tindex) */
#ifdef MYAHOO
/* make sure that holds continue to disable triggers, even after new ones are loaded */
if (J->Hold != 0)
{
MTrigDisable(J->Triggers);
}
#endif /* MYAHOO */
if (J->Triggers != NULL)
{
mtrig_t *T;
int tindex;
for (tindex = 0; tindex < J->Triggers->NumItems;tindex++)
{
T = (mtrig_t *)MUArrayListGetPtr(J->Triggers,tindex);
if (MTrigIsValid(T) == FALSE)
continue;
MTrigInitialize(T);
if (MSched.ForceTrigJobsIdle == TRUE)
bmset(&J->IFlags,mjifWaitForTrigLaunch);
} /* END for (tindex) */
/* report events for admin reservations */
MOReportEvent((void *)J,J->Name,mxoJob,mttCreate,J->CTime,TRUE);
}
} /* END else ((J->Cred.Q == NULL) || (!bmisset(&J->Cred.Q->Flags,mqfTrigger))) */
MULLFree(&VarList,MUFREE);
} /* END if (J->TX->SpecTrig != NULL) */
if (J->TemplateExtensions->SpecMinPreemptTime != NULL)
{
if ((J->Credential.Q == NULL) || (!bmisset(&J->Credential.Q->Flags,mqfPreemptCfg)))
{
MMBAdd(
&J->MessageBuffer,
"minpreempttime specified but not allowed by QOS",
NULL,
mmbtOther,
MSched.Time + MCONST_DAYLEN,
0,
NULL);
}
else
{
MJobSetAttr(J,mjaMinPreemptTime,(void **)J->TemplateExtensions->SpecMinPreemptTime,mdfString,mSet);
}
MUFree((char **)&J->TemplateExtensions->SpecMinPreemptTime);
} /* if (J->TX->SpecMinPreemptTime != NULL) */
if ((J->TemplateExtensions->SpecTemplates != NULL) && (J->TemplateExtensions->SpecTemplates[0] != '\0'))
{
/* Apply templates */
mjob_t *SJ;
char Templates[MMAX_LINE << 2];
char *Value;
char *ptr;
char tmpLine[MMAX_LINE << 2];
char *TokPtr2;
MUStrCpy(Templates,J->TemplateExtensions->SpecTemplates,sizeof(Templates));
Value = Templates;
if (!bmisset(&J->Flags,mjfTemplatesApplied))
{
if (Value[0] == '^')
{
bmset(&J->IFlags,mjifNoTemplates);
Value++;
}
TokPtr2 = Value;
strncpy(tmpLine,Value,sizeof(tmpLine));
while ((ptr = MUStrTok(TokPtr2,":,",&TokPtr2)) != NULL)
{
if ((MTJobFind(ptr,&SJ) == SUCCESS) &&
(SJ->TemplateExtensions != NULL) &&
(bmisset(&SJ->TemplateExtensions->TemplateFlags,mtjfTemplateIsSelectable)))
{
if (MJobApplySetTemplate(J,SJ,EMsg) == FAILURE)
{
return(FAILURE);
}
}
else
{
char errMsg[MMAX_LINE];
snprintf(errMsg,sizeof(errMsg),"WARNING: cannot apply template '%s'",
ptr);
MMBAdd(&J->MessageBuffer,errMsg,NULL,mmbtNONE,0,0,NULL);
}
if (!J->TemplateExtensions->ReqSetTemplates.empty())
MStringAppend(&J->TemplateExtensions->ReqSetTemplates,",");
MStringAppend(&J->TemplateExtensions->ReqSetTemplates,ptr);
}
}
bmset(&J->Flags,mjfTemplatesApplied);
bmset(&J->SysFlags,mjfTemplatesApplied);
/* Free the memory, we don't need it anymore */
MUFree(&J->TemplateExtensions->SpecTemplates);
} /* END if ((J->TX->SpecTemplates != NULL) && ...) */
if ((J->WCLimit == MMAX_TIME) && (!bmisset(&MSched.Flags,mschedfAllowInfWallTimeJobs)))
{
if (EMsg != NULL)
{
sprintf(EMsg,"infinite walltime jobs not allowed\n");
return(FAILURE);
}
}
if ((J->TemplateExtensions->SpecArray != NULL) &&
(J->TemplateExtensions->SpecArray[0] != '\0') &&
(J->Array == NULL) &&
(!bmisset(&J->IFlags,mjifWasLoadedFromCP)))
{
char *ptr;
char *TokPtr = NULL;
char *arrayStr = NULL;
mulong BM;
MUStrDup(&arrayStr,J->TemplateExtensions->SpecArray);
/* Alloc the Array control container */
if (MJobArrayAlloc(J) == FAILURE)
{
/* NO MEMORY ERROR */
return(FAILURE);
}
ptr = MUStrTok(arrayStr,"[]",&TokPtr);
MUStrCpy(J->Array->Name,ptr,MMAX_NAME);
ptr = MUStrTok(NULL,"[]",&TokPtr);
MUParseRangeString(
ptr,
&BM,
J->Array->Members,
MMAX_JOBARRAYSIZE,
&J->Array->Count,
NULL);
ptr = MUStrTok(NULL,"%\n\t",&TokPtr);
if ((ptr != NULL) && (ptr[0] != '\0'))
{
J->Array->Limit = strtol(ptr,NULL,10);
}
/* NO ERROR CHECKING ON 'NO MEMORY' CONDITION ??? */
J->Array->JPtrs = (mjob_t **)MUCalloc(1,sizeof(mjob_t *) * J->Array->Count + 1);
J->Array->JName = (char **)MUCalloc(1,sizeof(char *) * J->Array->Count + 1);
J->Array->JState = (enum MJobStateEnum *)MUCalloc(1,sizeof(enum MJobStateEnum) * J->Array->Count + 1);
MUFree(&arrayStr);
}
return(SUCCESS);
} /* END MJobProcessRestrictedAttrs() */
