bool
caInsert( ClassAd* target, ClassAd* source, const char* attr,
const char* prefix )
{
ExprTree* tree;
if( !attr ) {
EXCEPT( "caInsert called with NULL attribute" );
}
if( !target || !source ) {
EXCEPT( "caInsert called with NULL classad" );
}
MyString new_attr;
if( prefix ) {
new_attr = prefix;
}
new_attr += attr;
tree = source->LookupExpr( attr );
if( !tree ) {
target->Delete(new_attr.Value());
return false;
}
tree = tree->Copy();
if ( !target->Insert(new_attr.Value(), tree, false) ) {
dprintf( D_ALWAYS, "caInsert: Can't insert %s into target classad.\n", attr );
delete tree;
return false;
}
return true;
}
bool AttributeReference::
_Evaluate (EvalState &state, Value &val, ExprTree *&sig ) const
{
ExprTree *tree;
ExprTree *exprSig;
const ClassAd *curAd;
bool rval;
curAd = state.curAd;
exprSig = NULL;
rval = true;
switch( FindExpr( state , tree , exprSig , true ) ) {
case EVAL_FAIL:
rval = false;
break;
case EVAL_ERROR:
val.SetErrorValue( );
break;
case EVAL_UNDEF:
val.SetUndefinedValue( );
break;
case EVAL_OK:
{
if( state.depth_remaining <= 0 ) {
val.SetErrorValue();
state.curAd = curAd;
return false;
}
state.depth_remaining--;
rval = tree->Evaluate( state, val );
state.depth_remaining++;
break;
}
default: CLASSAD_EXCEPT( "ClassAd: Should not reach here" );
}
if(!rval || !(sig=new AttributeReference(exprSig,attributeStr,absolute))){
if( rval ) {
CondorErrno = ERR_MEM_ALLOC_FAILED;
CondorErrMsg = "";
}
delete exprSig;
sig = NULL;
return( false );
}
state.curAd = curAd;
return rval;
}
void ParametricAd::addUserTag ( const std::string& attr_name, const std::string& attr_value){
ExprTree* tree = Lookup ( JDL::USERTAGS ) ;
if ( tree ==NULL){
// UserTags attribute has still to be created
ClassAd ad ;
ad.InsertAttr ( attr_name , attr_value ) ;
ExprTree* tmp_expr = ad.Copy();
Insert ( JDL::USERTAGS , tmp_expr ) ;
}else{
// Already Existing UserTags attribute
if ( tree->GetKind () == classad::ExprTree::CLASSAD_NODE )
((ClassAd*)tree)->InsertAttr ( attr_name , attr_value ) ;
}
}
bool AttributeReference::
_Evaluate (EvalState &state, Value &val) const
{
ExprTree *tree;
ExprTree *dummy;
const ClassAd *curAd;
bool rval;
// find the expression and the evalstate
curAd = state.curAd;
switch( FindExpr( state, tree, dummy, false ) ) {
case EVAL_FAIL:
return false;
case EVAL_ERROR:
val.SetErrorValue();
state.curAd = curAd;
return true;
case EVAL_UNDEF:
val.SetUndefinedValue();
state.curAd = curAd;
return true;
case EVAL_OK:
{
if( state.depth_remaining <= 0 ) {
val.SetErrorValue();
state.curAd = curAd;
return false;
}
state.depth_remaining--;
rval = tree->Evaluate( state, val );
state.depth_remaining++;
state.curAd = curAd;
return rval;
}
default: CLASSAD_EXCEPT( "ClassAd: Should not reach here" );
}
return false;
}
void
HookPrepareJobClient::hookExited(int exit_status) {
HookClient::hookExited(exit_status);
if (WIFSIGNALED(exit_status) || WEXITSTATUS(exit_status) != 0) {
MyString status_msg = "";
statusString(exit_status, status_msg);
int subcode;
if (WIFSIGNALED(exit_status)) {
subcode = -1 * WTERMSIG(exit_status);
}
else {
subcode = WEXITSTATUS(exit_status);
}
MyString err_msg;
err_msg.sprintf("HOOK_PREPARE_JOB (%s) failed (%s)", m_hook_path,
status_msg.Value());
dprintf(D_ALWAYS|D_FAILURE,
"ERROR in StarterHookMgr::tryHookPrepareJob: %s\n",
err_msg.Value());
Starter->jic->notifyStarterError(err_msg.Value(), true,
CONDOR_HOLD_CODE_HookPrepareJobFailure, subcode);
Starter->RemoteShutdownFast(0);
}
else {
// Make an update ad from the stdout of the hook
MyString out(*getStdOut());
ClassAd updateAd;
updateAd.initFromString(out.Value(), NULL);
dprintf(D_FULLDEBUG, "Prepare hook output classad\n");
updateAd.dPrint(D_FULLDEBUG);
// Insert each expr from the update ad into the job ad
updateAd.ResetExpr();
ClassAd* job_ad = Starter->jic->jobClassAd();
const char *name;
ExprTree *et;
while (updateAd.NextExpr(name, et)) {
ExprTree *pCopy = et->Copy();
job_ad->Insert(name, pCopy, false);
}
dprintf(D_FULLDEBUG, "After Prepare hook: merged job classad:\n");
job_ad->dPrint(D_FULLDEBUG);
Starter->jobEnvironmentReady();
}
}
bool
MakeSignature( const ClassAd *ad, const References &refs, string &sig )
{
References::iterator itr;
ExprTree *expr;
Value val;
ClassAdUnParser unp;
sig = "";
for( itr=refs.begin( ); itr!= refs.end( ); itr++ ) {
if((expr=ad->Lookup(*itr))&&expr->GetKind()==ExprTree::LITERAL_NODE) {
unp.Unparse( sig, expr );
} else {
return( false );
}
sig += ":";
}
return( true );
}
int
classad_evaluate_boolean_expr(const char *s_in, const classad_expr_tree t_ex,
int *result)
{
ClassAd *ad;
ClassAdParser parser;
if (s_in == NULL || t_ex == NULL || result == NULL)
return C_CLASSAD_INVALID_ARG;
ExprTree *et = (ExprTree *)t_ex;
ad = parser.ParseClassAd(s_in);
if (ad == NULL) return C_CLASSAD_PARSE_ERROR;
int retcod = C_CLASSAD_NO_ERROR;
Value v;
et->SetParentScope(ad);
ad->EvaluateExpr(et, v);
et->SetParentScope(NULL);
bool tmp_res;
if (v.IsBooleanValue( tmp_res ))
{
if (tmp_res) *result = 1;
else *result = 0;
retcod = C_CLASSAD_NO_ERROR;
}
else retcod = C_CLASSAD_INVALID_VALUE;
delete ad;
return retcod;
}
void
VMRegister::requestHostClassAds(void)
{
// find host startd daemon
if( !m_vm_host_daemon )
m_vm_host_daemon = vmapi_findDaemon( m_vm_host_name, DT_STARTD);
if( !m_vm_host_daemon ) {
dprintf( D_FULLDEBUG, "Can't find host(%s) Startd daemon\n", m_vm_host_name );
return;
}
ClassAd query_ad;
query_ad.SetMyTypeName(QUERY_ADTYPE);
query_ad.SetTargetTypeName(STARTD_ADTYPE);
query_ad.Assign(ATTR_REQUIREMENTS, true);
char *addr = m_vm_host_daemon->addr();
Daemon hstartd(DT_STARTD, addr);
ReliSock ssock;
ssock.timeout( VM_SOCKET_TIMEOUT );
ssock.encode();
if( !ssock.connect(addr) ) {
dprintf( D_FULLDEBUG, "Failed to connect to host startd(%s)\n to get host classAd", addr);
return;
}
if(!hstartd.startCommand( QUERY_STARTD_ADS, &ssock )) {
dprintf( D_FULLDEBUG, "Failed to send QUERY_STARTD_ADS command to host startd(%s)\n", addr);
return;
}
if( !query_ad.put(ssock) ) {
dprintf(D_FULLDEBUG, "Failed to send query Ad to host startd(%s)\n", addr);
}
if( !ssock.end_of_message() ) {
dprintf(D_FULLDEBUG, "Failed to send query EOM to host startd(%s)\n", addr);
}
// Read host classAds
ssock.timeout( VM_SOCKET_TIMEOUT );
ssock.decode();
int more = 1, num_ads = 0;
ClassAdList adList;
ClassAd *ad;
while (more) {
if( !ssock.code(more) ) {
ssock.end_of_message();
return;
}
if(more) {
ad = new ClassAd;
if( !ad->initFromStream(ssock) ) {
ssock.end_of_message();
delete ad;
return;
}
adList.Insert(ad);
num_ads++;
}
}
ssock.end_of_message();
dprintf(D_FULLDEBUG, "Got %d classAds from host\n", num_ads);
// Although we can get more than one classAd from host machine,
// we use only the first one classAd
adList.Rewind();
ad = adList.Next();
#if !defined(WANT_OLD_CLASSADS)
ad->AddTargetRefs( TargetJobAttrs );
#endif
// Get each Attribute from the classAd
// added "HOST_" in front of each Attribute name
const char *name;
ExprTree *expr;
ad->ResetExpr();
while( ad->NextExpr(name, expr) ) {
MyString attr;
attr += "HOST_";
attr += name;
// Insert or Update an attribute to host_classAd in a VMRegister object
ExprTree * pTree = expr->Copy();
host_classad->Insert(attr.Value(), pTree, true);
}
}
void ParametricAd::checkInputSandbox(std::vector<std::string>& extracted){
GLITE_STACK_TRY("ParametricAd::checkInputSandbox(std::vector<std::string>&)");
// This vector contains all ISB string extracted Values
unsigned int iter_i = extracted.size();
// This Vector will replace the old InputSanbox with the extracted values
vector<ExprTree*> isVect ;
ExprTree* isbTree = Lookup (JDL::INPUTSB);
if (!isbTree){ /* No inputFiles found: return */ return; }
// These variables are needed when extracting
const string wmpURI = (hasAttribute(JDL::WMPISB_BASE_URI))? (getString(JDL::WMPISB_BASE_URI)):"";
const string isbURI= (hasAttribute(JDL::ISB_BASE_URI)) ? (getString(JDL::ISB_BASE_URI) ):"";
Value val ;
string isb ;
EvaluateExpr(isbTree,val);
inputRemotes.clear();
switch ( val.GetType() ){
case Value::UNDEFINED_VALUE:
inputRemotes.push_back(isbTree->Copy());
isVect.push_back(isbTree->Copy());
break;
case Value::STRING_VALUE:
val.IsStringValue(isb);
toBretrieved=extractFiles (JDL::INPUTSB, isb, extracted,
lookInto_b,wmpURI,isbURI,extractedAd.get()) || toBretrieved;
// put the extracted files back into the InputSandbox attribute
for(;iter_i<extracted.size();iter_i++){ // Iterate ONLY over new value(s)
val.SetStringValue(extracted[iter_i]); // create classad Value
isVect.push_back(Literal::MakeLiteral(val)); // Update ISB value
}
break;
case Value::LIST_VALUE:{
const ExprList *el;
val.IsListValue( el );
vector<ExprTree*> vectList ;
el->GetComponents(vectList) ;
for ( unsigned int i = 0; i< vectList.size() ; i++){
if (vectList[i]->GetKind()!=ExprTree::LITERAL_NODE){
// not a literal node
inputRemotes.push_back( vectList[i]->Copy() );
isVect.push_back(vectList[i]->Copy());
}
else if (vectList[i]->Evaluate(val)){
switch (val.GetType()){
case Value::STRING_VALUE:
val.IsStringValue(isb);
toBretrieved=extractFiles (JDL::INPUTSB, isb, extracted,
lookInto_b,wmpURI,isbURI,extractedAd.get()) || toBretrieved;
// put the extracted files back into the InputSandbox attribute
for(;iter_i<extracted.size();iter_i++){ // Iterate ONLY over new value(s)
val.SetStringValue(extracted[iter_i]); // create classad Value
isVect.push_back(Literal::MakeLiteral(val)); // Update ISB value
}
break;
case Value::UNDEFINED_VALUE:
// It's an Expression evaluated
inputRemotes.push_back( vectList[i]->Copy() );
isVect.push_back(vectList[i]->Copy());
break;
default:
throw AdMismatchException (__FILE__,__LINE__,
METHOD,WMS_JDLMISMATCH , JDL::INPUTSB );
}
}else{
// It's an Expression not evaluated, leave it unchanged
inputRemotes.push_back( vectList[i]->Copy() );
isVect.push_back(vectList[i]->Copy());
}
val.Clear() ;
}
}
break;
default:
throw AdMismatchException (__FILE__ , __LINE__ ,METHOD, WMS_JDLMISMATCH , JDL::INPUTSB );
break;
}
classad:ExprTree* tmp_expr = ExprList::MakeExprList(isVect);
Insert (JDL::INPUTSB , tmp_expr) ;
GLITE_STACK_JDL_CATCH_ALL() ; //Exiting from method: remove line from stack trace
}
int
ClassAdLog::ExamineTransaction(const char *key, const char *name, char *&val, ClassAd* &ad)
{
bool AdDeleted=false, ValDeleted=false, ValFound=false;
int attrsAdded = 0;
if (!active_transaction) return 0;
for (LogRecord *log = active_transaction->FirstEntry(key); log;
log = active_transaction->NextEntry()) {
switch (log->get_op_type()) {
case CondorLogOp_NewClassAd: {
if (AdDeleted) { // check to see if ad is created after a delete
AdDeleted = false;
}
break;
}
case CondorLogOp_DestroyClassAd: {
AdDeleted = true;
if ( ad ) {
delete ad;
ad = NULL;
attrsAdded = 0;
}
break;
}
case CondorLogOp_SetAttribute: {
char const *lname = ((LogSetAttribute *)log)->get_name();
if (name && strcasecmp(lname, name) == 0) {
if (ValFound) {
free(val);
}
val = strdup(((LogSetAttribute *)log)->get_value());
ValFound = true;
ValDeleted = false;
}
if (!name) {
if ( !ad ) {
ad = new ClassAd;
ASSERT(ad);
}
if (val) {
free(val);
val = NULL;
}
ExprTree* expr = ((LogSetAttribute *)log)->get_expr();
if (expr) {
expr = expr->Copy();
ad->Insert(lname, expr, false);
} else {
val = strdup(((LogSetAttribute *)log)->get_value());
ad->AssignExpr(lname, val);
}
attrsAdded++;
}
break;
}
case CondorLogOp_DeleteAttribute: {
char const *lname = ((LogDeleteAttribute *)log)->get_name();
if (name && strcasecmp(lname, name) == 0) {
if (ValFound) {
free(val);
}
ValFound = false;
ValDeleted = true;
}
if (!name) {
if (ad) {
ad->Delete(lname);
attrsAdded--;
}
}
break;
}
default:
break;
}
}
if ( name ) {
if (AdDeleted || ValDeleted) return -1;
if (ValFound) return 1;
return 0;
} else {
if (attrsAdded < 0 ) {
return 0;
}
return attrsAdded;
}
}
bool operator==(const ExprTree &tree1, const ExprTree &tree2)
{
return tree1.SameAs(&tree2);
}
bool AttributeReference::
_Flatten( EvalState &state, Value &val, ExprTree*&ntree, int*) const
{
ExprTree *tree;
ExprTree *dummy;
const ClassAd *curAd;
bool rval;
ntree = NULL; // Just to be safe... wenger 2003-12-11.
// find the expression and the evalstate
curAd = state.curAd;
switch( FindExpr( state, tree, dummy, false ) ) {
case EVAL_FAIL:
return false;
case EVAL_ERROR:
val.SetErrorValue();
state.curAd = curAd;
return true;
case EVAL_UNDEF:
if( expr && state.flattenAndInline ) {
ExprTree *expr_ntree = NULL;
Value expr_val;
if( state.depth_remaining <= 0 ) {
val.SetErrorValue();
state.curAd = curAd;
return false;
}
state.depth_remaining--;
rval = expr->Flatten(state,expr_val,expr_ntree);
state.depth_remaining++;
if( rval && expr_ntree ) {
ntree = MakeAttributeReference(expr_ntree,attributeStr);
if( ntree ) {
state.curAd = curAd;
return true;
}
}
delete expr_ntree;
}
if( !(ntree = Copy( ) ) ) {
CondorErrno = ERR_MEM_ALLOC_FAILED;
CondorErrMsg = "";
state.curAd = curAd;
return( false );
}
state.curAd = curAd;
return true;
case EVAL_OK:
{
// Don't flatten or inline a classad that's referred to
// by an attribute.
if ( tree->GetKind() == CLASSAD_NODE ) {
ntree = Copy( );
val.SetUndefinedValue( );
return true;
}
if( state.depth_remaining <= 0 ) {
val.SetErrorValue();
state.curAd = curAd;
return false;
}
state.depth_remaining--;
rval = tree->Flatten( state, val, ntree );
state.depth_remaining++;
// don't inline if it didn't flatten to a value, and clear cache
// do inline if FlattenAndInline was called
if( ntree ) {
if( state.flattenAndInline ) { // NAC
return true; // NAC
} // NAC
delete ntree;
ntree = Copy( );
val.SetUndefinedValue( );
}
state.curAd = curAd;
return rval;
}
default: CLASSAD_EXCEPT( "ClassAd: Should not reach here" );
}
return false;
}
void find_attribute_if(std::vector<std::string>* v, ExprTree* e, Predicate predicate, bool deep_find = false)
{
expression_trace_type exprTrace, exprStack;
if (!e) return;
else exprStack.push_back(e);
while (!exprStack.empty()) {
exprTrace.push_front( (e = exprStack.front()) );
exprStack.pop_front();
switch( e->GetKind() ) {
case ExprTree::LITERAL_NODE: break;
case ExprTree::OP_NODE:
{
ExprTree* e1 = 0, *e2 = 0, *e3 = 0;
Operation::OpKind ok;
dynamic_cast<Operation*>(e)->GetComponents(ok, e1, e2, e3);
if ( e3 ) exprStack.push_front(e3);
if ( e2 ) exprStack.push_front(e2);
if ( e1 ) exprStack.push_front(e1);
}
break;
case ExprTree::FN_CALL_NODE:
{
std::vector<ExprTree*> args;
std::string fn;
dynamic_cast<FunctionCall*>(e)->GetComponents(fn, args);
for( int i = args.size(); i; i--) exprStack.push_front( args[i-1] );
}
break;
case ExprTree::EXPR_LIST_NODE:
{
std::vector<ExprTree*> args;
dynamic_cast<ExprList*>(e)->GetComponents(args);
for( int i = args.size(); i; i--) exprStack.push_front( args[i-1] );
}
break;
case ExprTree::ATTRREF_NODE:
{
AttributeReference* a = dynamic_cast<AttributeReference*>(e);
ExprTree* reference_expr = 0;
std::string name;
bool absolute;
bool continue_find = false;
a->GetComponents(reference_expr, name, absolute);
const ClassAd* parent_scope = 0;
parent_scope=a->GetParentScope();
if (!reference_expr) { // simple reference "attr" and ".attr"...
// ...look-up the parent scope to solve the reference...
reference_expr = parent_scope->Lookup(name);
if(reference_expr) continue_find = true;
else if(predicate(std::make_pair(&exprTrace,a)) &&
find(v->begin(), v->end(), name) == v->end()) {
v->push_back(name);
}
}
else if (parent_scope && deep_find) {
// ...reference_expr is not a simple reference "expr.attr"
// and the deep resolution is requested.
std::string reference_name;
Value reference_value;
const ClassAd* reference_ad;
ExprTree* expr;
dynamic_cast<AttributeReference*>(reference_expr)->
GetComponents(expr, reference_name, absolute);
if (parent_scope->EvaluateAttr(reference_name, reference_value) &&
reference_value.IsClassAdValue(reference_ad)) {
reference_expr=reference_ad->Lookup(name);
continue_find = expr && !(expr->GetKind()==ExprTree::LITERAL_NODE);
if(!continue_find && predicate(std::make_pair(&exprTrace,a)) &&
find(v->begin(), v->end(), name) == v->end()) {
v->push_back(name);
}
}
}
// ...if the referenced expression has not been evaluated yet...
if (reference_expr &&
find(exprTrace.begin(), exprTrace.end(), reference_expr) == exprTrace.end()) {
if (continue_find) exprStack.push_front(reference_expr); // ...further search needed.
else if (predicate(std::make_pair(&exprTrace,a)) &&
find(v->begin(), v->end(), name) == v->end()) {
v->push_back(name);
}
}
}
break;
default:
assert( false );
}
}
}
// download the files associated with the jobads to the sandbox at td_sinful
// with the supplied capability.
// The work_ad should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
// ATTR_TREQ_JOBID_ALLOW_LIST
bool
DCTransferD::download_job_files(ClassAd *work_ad, CondorError * errstack)
{
ReliSock *rsock = NULL;
int timeout = 60 * 60 * 8; // transfers take a long time...
int i;
ClassAd reqad, respad;
std::string cap;
int ftp;
int invalid;
int protocol;
std::string reason;
int num_transfers;
ClassAd jad;
const char *lhstr = NULL;
ExprTree *tree = NULL;
//////////////////////////////////////////////////////////////////////////
// Connect to the transferd and authenticate
//////////////////////////////////////////////////////////////////////////
// This call with automatically connect to _addr, which was set in the
// constructor of this object to be the transferd in question.
rsock = (ReliSock*)startCommand(TRANSFERD_READ_FILES, Stream::reli_sock,
timeout, errstack);
if( ! rsock ) {
dprintf( D_ALWAYS, "DCTransferD::download_job_files: "
"Failed to send command (TRANSFERD_READ_FILES) "
"to the schedd\n" );
errstack->push("DC_TRANSFERD", 1,
"Failed to start a TRANSFERD_READ_FILES command.");
return false;
}
// First, if we're not already authenticated, force that now.
if (!forceAuthentication( rsock, errstack )) {
dprintf( D_ALWAYS, "DCTransferD::download_job_files() authentication "
"failure: %s\n", errstack->getFullText().c_str() );
errstack->push("DC_TRANSFERD", 1,
"Failed to authenticate properly.");
return false;
}
rsock->encode();
//////////////////////////////////////////////////////////////////////////
// Query the transferd about the capability/protocol and see if I can
// download my files. It will respond with a classad saying good or bad.
//////////////////////////////////////////////////////////////////////////
work_ad->LookupString(ATTR_TREQ_CAPABILITY, cap);
work_ad->LookupInteger(ATTR_TREQ_FTP, ftp);
reqad.Assign(ATTR_TREQ_CAPABILITY, cap);
reqad.Assign(ATTR_TREQ_FTP, ftp);
// This request ad to the transferd should contain:
// ATTR_TREQ_CAPABILITY
// ATTR_TREQ_FTP
reqad.put(*rsock);
rsock->end_of_message();
rsock->decode();
// This response ad from the transferd should contain:
// ATTR_TREQ_INVALID_REQUEST (set to true)
// ATTR_TREQ_INVALID_REASON
//
// OR
//
// ATTR_TREQ_INVALID_REQUEST (set to false)
// ATTR_TREQ_NUM_TRANSFERS
//
respad.initFromStream(*rsock);
rsock->end_of_message();
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if (invalid == TRUE) {
// The transferd rejected my attempt to upload the fileset
delete rsock;
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
respad.LookupInteger(ATTR_TREQ_NUM_TRANSFERS, num_transfers);
//////////////////////////////////////////////////////////////////////////
// Based upon the protocol I've chosen, use that method to download the
// files. When using the FileTrans protocol, a child process on the
// transferd side will be sending me individual job ads and then
// instantiating a filetransfer object for that ad.
//////////////////////////////////////////////////////////////////////////
dprintf(D_ALWAYS, "Receiving fileset");
work_ad->LookupInteger(ATTR_TREQ_FTP, protocol);
switch(protocol) {
case FTP_CFTP: // download the files using the FileTransfer Object
for (i = 0; i < num_transfers; i++) {
// Grab a job ad the server is sending us so we know what
// to receive.
jad.initFromStream(*rsock);
rsock->end_of_message();
// translate the job ad by replacing the
// saved SUBMIT_ attributes so the download goes into the
// correct place.
jad.ResetExpr();
while( jad.NextExpr(lhstr, tree) ) {
if ( lhstr && strncasecmp("SUBMIT_",lhstr,7)==0 ) {
// this attr name starts with SUBMIT_
// compute new lhs (strip off the SUBMIT_)
const char *new_attr_name = strchr(lhstr,'_');
ExprTree * pTree;
ASSERT(new_attr_name);
new_attr_name++;
// insert attribute
pTree = tree->Copy();
jad.Insert(new_attr_name, pTree, false);
}
} // while next expr
// instantiate a filetransfer object and have it accept the
// files.
FileTransfer ftrans;
if ( !ftrans.SimpleInit(&jad, false, false, rsock) )
{
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to initate uploading of files.");
return false;
}
// We want files to be copied to their final places, so apply
// any filename remaps when downloading.
if ( !ftrans.InitDownloadFilenameRemaps(&jad) ) {
return false;
}
ftrans.setPeerVersion( version() );
if ( !ftrans.DownloadFiles() ) {
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Failed to download files.");
return false;
}
dprintf(D_ALWAYS | D_NOHEADER, ".");
}
rsock->end_of_message();
dprintf(D_ALWAYS | D_NOHEADER, "\n");
break;
default:
// Bail due to user error. This client doesn't support the unknown
// protocol.
delete rsock;
errstack->push("DC_TRANSFERD", 1,
"Unknown file transfer protocol selected.");
return false;
break;
}
//////////////////////////////////////////////////////////////////////////
// Get the response from the transferd once it sees a completed
// movement of files to the child process.
//////////////////////////////////////////////////////////////////////////
rsock->decode();
respad.initFromStream(*rsock);
rsock->end_of_message();
// close up shop
delete rsock;
respad.LookupInteger(ATTR_TREQ_INVALID_REQUEST, invalid);
if ( invalid == TRUE ) {
respad.LookupString(ATTR_TREQ_INVALID_REASON, reason);
errstack->push("DC_TRANSFERD", 1, reason.c_str());
return false;
}
return true;
}
bool
Metric::evaluate(char const *attr_name,classad::Value &result,classad::ClassAd &metric_ad,classad::ClassAd const &daemon_ad,MetricTypeEnum type,ExtArray<MyString> *regex_groups,char const *regex_attr)
{
bool retval = true;
ExprTree *expr = NULL;
if( !(expr=metric_ad.Lookup(attr_name)) ) {
return true;
}
classad::ClassAd const *ad = &daemon_ad;
ClassAd daemon_ad_copy;
if( regex_attr ) {
// make a modifiable copy of daemon_ad and insert regex_attr = regex_attr, so the user-defined expression can refer to it
daemon_ad_copy = daemon_ad;
ad = &daemon_ad_copy;
daemon_ad_copy.AssignExpr(ATTR_REGEX,regex_attr);
}
expr->SetParentScope(ad);
if( !ad->EvaluateExpr(expr,result) ||
(type == STRING && !result.IsStringValue()) ||
(type == DOUBLE && !result.IsNumber()) ||
(type == FLOAT && !result.IsNumber()) ||
(type == INT8 && !result.IsIntegerValue()) ||
(type == UINT8 && !result.IsIntegerValue()) ||
(type == INT16 && !result.IsIntegerValue()) ||
(type == UINT16 && !result.IsIntegerValue()) ||
(type == INT32 && !result.IsIntegerValue()) ||
(type == UINT32 && !result.IsIntegerValue()) ||
(type == BOOLEAN && !result.IsBooleanValue()) )
{
retval = false;
classad::ClassAdUnParser unparser;
std::string expr_str;
unparser.Unparse(expr_str,expr);
dprintf(D_FULLDEBUG,"Failed to evaluate the following%s%s: %s=%s\n",
name.empty() ? "" : " attribute of metric ",
name.c_str(),
attr_name,
expr_str.c_str());
}
expr->SetParentScope(&metric_ad);
// do regex macro substitutions
if( regex_groups && regex_groups->length() > 0 ) {
std::string str_value;
if( result.IsStringValue(str_value) && str_value.find("\\")!=std::string::npos ) {
std::string new_str_value;
const char *ch = str_value.c_str();
while( *ch ) {
if( *ch == '\\' ) {
ch++;
if( !isdigit(*ch) ) {
new_str_value += *(ch++);
}
else {
char *endptr = NULL;
long index = strtol(ch,&endptr,10);
ch = endptr;
if( index < regex_groups->length() ) {
new_str_value += (*regex_groups)[index];
}
}
}
else {
new_str_value += *(ch++);
}
}
result.SetStringValue(new_str_value);
}
}
return retval;
}
void BaseJob::JobAdUpdateFromSchedd( const ClassAd *new_ad, bool full_ad )
{
static const char *held_removed_update_attrs[] = {
ATTR_JOB_STATUS,
ATTR_HOLD_REASON,
ATTR_HOLD_REASON_CODE,
ATTR_HOLD_REASON_SUBCODE,
ATTR_LAST_HOLD_REASON,
ATTR_RELEASE_REASON,
ATTR_LAST_RELEASE_REASON,
ATTR_ENTERED_CURRENT_STATUS,
ATTR_NUM_SYSTEM_HOLDS,
ATTR_REMOVE_REASON,
NULL
};
int new_condor_state;
new_ad->LookupInteger( ATTR_JOB_STATUS, new_condor_state );
if ( new_condor_state == condorState ) {
if ( !full_ad ) {
MergeClassAds( jobAd, const_cast<ClassAd*>(new_ad), true, false );
}
return;
}
if ( new_condor_state == REMOVED && condorState == HELD ) {
int release_status = IDLE;
jobAd->LookupInteger( ATTR_JOB_STATUS_ON_RELEASE, release_status );
if ( release_status == REMOVED ) {
// We already know about this REMOVED state and have
// decided to go on hold afterwards, so ignore this
// "update".
return;
}
}
if ( new_condor_state == REMOVED || new_condor_state == HELD ) {
for ( int i = 0; held_removed_update_attrs[i] != NULL; i++ ) {
ExprTree *expr;
if ( (expr = new_ad->LookupExpr( held_removed_update_attrs[i] )) != NULL ) {
ExprTree * pTree = expr->Copy();
jobAd->Insert( held_removed_update_attrs[i], pTree, false );
} else {
jobAd->Delete( held_removed_update_attrs[i] );
}
jobAd->SetDirtyFlag( held_removed_update_attrs[i], false );
}
if ( new_condor_state == HELD && writeUserLog && !holdLogged ) {
// TODO should this log event be delayed until gridmanager is
// done dealing with the job?
WriteHoldEventToUserLog( jobAd );
holdLogged = true;
}
// If we're about to put a job on hold and learn that it's been
// removed, make sure the state returns to removed when it is
// released. This is normally checked in JobHeld(), but it's
// possible to learn of the removal just as we're about to
// update the schedd with the hold.
if ( new_condor_state == REMOVED && condorState == HELD ) {
bool dirty;
jobAd->GetDirtyFlag( ATTR_JOB_STATUS, NULL, &dirty );
if ( dirty ) {
jobAd->Assign( ATTR_JOB_STATUS_ON_RELEASE, REMOVED );
}
}
condorState = new_condor_state;
// TODO do we need to call UpdateRuntimeStats() here?
UpdateRuntimeStats();
SetEvaluateState();
} else if ( new_condor_state == COMPLETED ) {
condorState = new_condor_state;
// TODO do we need to update any other attributes?
SetEvaluateState();
} else if ( !full_ad ) {
MergeClassAds( jobAd, const_cast<ClassAd*>(new_ad), true, false );
}
}
/*********************************************************************
*
* Function: process_evaluate
* Purpose: Given a line that begins with "evaluate", parse the rest
* of the line and perform the evaluation.
*
*********************************************************************/
static void process_evaluate(const string &line,
int token_start, int line_number,
const Parameters ¶meters,
ErrorCount *errors)
{
string classad_name, expression_string;
ExprTree *expr;
ClassAdParser parser;
classad_name = extract_token(&token_start, line);
expr = NULL;
if ((unsigned) token_start < line.size()) {
expression_string = line.substr(token_start, line.size() - token_start);
} else {
expression_string = "";
}
if (!classad_name.compare("")
|| !expression_string.compare("")) {
cout << "Error: Missing evaluate information on line " << line_number
<< "." << endl;
cout << " Format: evaluate <classad> <expression>"
<< endl;
errors->IncrementErrors();
}
else {
if (!parser.ParseExpression(expression_string, expr)) {
cout << "Error: Can't parse expression (" << expression_string
<< ") on line " << line_number << "." << endl;
errors->IncrementErrors();
}
else {
ClassAd *classad = classads[classad_name];
if (classad == NULL) {
cout << "Error: Unknown ClassAd: \"" << classad_name
<< "\" on line " << line_number << "." << endl;
errors->IncrementErrors();
}
else {
Value value;
expr->SetParentScope(classad);
if (!classad->EvaluateExpr(expr, value)) {
cout << "Error: Can't evaluate expression (" << expression_string
<< ") on line " <<line_number << "." << endl;
errors->IncrementErrors();
}
else {
PrettyPrint unparser;
string value_string;
unparser.Unparse(value_string, value);
cout << "OK: Evaluating \"" << expression_string
<< "\" in " << classad_name
<< " evaluates to " << value_string
<< " on line " << line_number << endl;
}
}
}
}
if (expr != NULL) {
delete expr;
}
return;
}
bool
DCSchedd::receiveJobSandbox(const char* constraint, CondorError * errstack, int * numdone /*=0*/)
{
if(numdone) { *numdone = 0; }
ExprTree *tree = NULL;
const char *lhstr;
int reply;
int i;
ReliSock rsock;
int JobAdsArrayLen;
bool use_new_command = true;
if ( version() ) {
CondorVersionInfo vi( version() );
if ( vi.built_since_version(6,7,7) ) {
use_new_command = true;
} else {
use_new_command = false;
}
}
// // // // // // // //
// On the wire protocol
// // // // // // // //
rsock.timeout(20); // years of research... :)
if( ! rsock.connect(_addr) ) {
dprintf( D_ALWAYS, "DCSchedd::receiveJobSandbox: "
"Failed to connect to schedd (%s)\n", _addr );
return false;
}
if ( use_new_command ) {
if( ! startCommand(TRANSFER_DATA_WITH_PERMS, (Sock*)&rsock, 0,
errstack) ) {
dprintf( D_ALWAYS, "DCSchedd::receiveJobSandbox: "
"Failed to send command (TRANSFER_DATA_WITH_PERMS) "
"to the schedd\n" );
return false;
}
} else {
if( ! startCommand(TRANSFER_DATA, (Sock*)&rsock, 0, errstack) ) {
dprintf( D_ALWAYS, "DCSchedd::receiveJobSandbox: "
"Failed to send command (TRANSFER_DATA) "
"to the schedd\n" );
return false;
}
}
// First, if we're not already authenticated, force that now.
if (!forceAuthentication( &rsock, errstack )) {
dprintf( D_ALWAYS,
"DCSchedd::receiveJobSandbox: authentication failure: %s\n",
errstack ? errstack->getFullText().c_str() : "" );
return false;
}
rsock.encode();
// Send our version if using the new command
if ( use_new_command ) {
// Need to use a named variable, else the wrong version of
// code() is called.
char *my_version = strdup( CondorVersion() );
if ( !rsock.code(my_version) ) {
dprintf(D_ALWAYS,"DCSchedd:receiveJobSandbox: "
"Can't send version string to the schedd\n");
free( my_version );
return false;
}
free( my_version );
}
// Send the constraint
char * nc_constraint = strdup( constraint ); // de-const
if ( !rsock.code(nc_constraint) ) {
free( nc_constraint );
dprintf(D_ALWAYS,"DCSchedd:receiveJobSandbox: "
"Can't send JobAdsArrayLen to the schedd\n");
return false;
}
free( nc_constraint );
if ( !rsock.end_of_message() ) {
std::string errmsg;
formatstr(errmsg,
"Can't send initial message (version + constraint) to schedd (%s)",
_addr);
dprintf(D_ALWAYS,"DCSchedd::receiveJobSandbox: %s\n", errmsg.c_str());
if( errstack ) {
errstack->push(
"DCSchedd::receiveJobSandbox",
CEDAR_ERR_EOM_FAILED,
errmsg.c_str());
}
return false;
}
// Now, read how many jobs matched the constraint.
rsock.decode();
if ( !rsock.code(JobAdsArrayLen) ) {
std::string errmsg;
formatstr(errmsg,
"Can't receive JobAdsArrayLen from the schedd (%s)",
_addr);
dprintf(D_ALWAYS,"DCSchedd::receiveJobSandbox: %s\n", errmsg.c_str());
if( errstack ) {
errstack->push(
"DCSchedd::receiveJobSandbox",
CEDAR_ERR_GET_FAILED,
errmsg.c_str());
}
return false;
}
rsock.end_of_message();
dprintf(D_FULLDEBUG,"DCSchedd:receiveJobSandbox: "
"%d jobs matched my constraint (%s)\n",
JobAdsArrayLen, constraint);
// Now read all the files via the file transfer object
for (i=0; i<JobAdsArrayLen; i++) {
FileTransfer ftrans;
ClassAd job;
// grab job ClassAd
if ( !getClassAd(&rsock, job) ) {
std::string errmsg;
formatstr(errmsg, "Can't receive job ad %d from the schedd", i);
dprintf(D_ALWAYS, "DCSchedd::receiveJobSandbox: %s\n", errmsg.c_str());
if( errstack ) {
errstack->push(
"DCSchedd::receiveJobSandbox",
CEDAR_ERR_GET_FAILED,
errmsg.c_str());
}
return false;
}
rsock.end_of_message();
// translate the job ad by replacing the
// saved SUBMIT_ attributes
job.ResetExpr();
while( job.NextExpr(lhstr, tree) ) {
if ( lhstr && strncasecmp("SUBMIT_",lhstr,7)==0 ) {
// this attr name starts with SUBMIT_
// compute new lhs (strip off the SUBMIT_)
const char *new_attr_name = strchr(lhstr,'_');
ExprTree * pTree;
ASSERT(new_attr_name);
new_attr_name++;
// insert attribute
pTree = tree->Copy();
job.Insert(new_attr_name, pTree, false);
}
} // while next expr
if ( !ftrans.SimpleInit(&job,false,false,&rsock) ) {
if( errstack ) {
int cluster = -1, proc = -1;
job.LookupInteger(ATTR_CLUSTER_ID,cluster);
job.LookupInteger(ATTR_PROC_ID,proc);
errstack->pushf(
"DCSchedd::receiveJobSandbox",
FILETRANSFER_INIT_FAILED,
"File transfer initialization failed for target job %d.%d",
cluster, proc );
}
return false;
}
// We want files to be copied to their final places, so apply
// any filename remaps when downloading.
if ( !ftrans.InitDownloadFilenameRemaps(&job) ) {
return false;
}
if ( use_new_command ) {
ftrans.setPeerVersion( version() );
}
if ( !ftrans.DownloadFiles() ) {
if( errstack ) {
FileTransfer::FileTransferInfo ft_info = ftrans.GetInfo();
int cluster = -1, proc = -1;
job.LookupInteger(ATTR_CLUSTER_ID,cluster);
job.LookupInteger(ATTR_PROC_ID,proc);
errstack->pushf(
"DCSchedd::receiveJobSandbox",
FILETRANSFER_DOWNLOAD_FAILED,
"File transfer failed for target job %d.%d: %s",
cluster, proc, ft_info.error_desc.Value() );
}
return false;
}
}
rsock.end_of_message();
rsock.encode();
reply = OK;
rsock.code(reply);
rsock.end_of_message();
if(numdone) { *numdone = JobAdsArrayLen; }
return true;
}
void
VMUniverseMgr::publish( ClassAd* ad, amask_t /*mask*/ )
{
if( !ad ) {
return;
}
if( !m_starter_has_vmcode || ( m_vm_type.Length() == 0 )) {
ad->Assign(ATTR_HAS_VM, false);
return;
}
ad->Assign(ATTR_HAS_VM, true);
// publish the number of still executable Virtual machines
if( m_vm_max_num > 0 ) {
int avail_vm_num = m_vm_max_num - numOfRunningVM();
ad->Assign(ATTR_VM_AVAIL_NUM, avail_vm_num);
}else {
// no limit of the number of executable VM
ad->Assign(ATTR_VM_AVAIL_NUM, VM_AVAIL_UNLIMITED_NUM);
}
// we will publish all information provided by vmgahp server
m_vmgahp_info.ResetExpr();
ExprTree* expr = NULL;
const char *attr_name = NULL;
while(m_vmgahp_info.NextExpr(attr_name, expr)) {
// we need to adjust available vm memory
if( strcasecmp(attr_name, ATTR_VM_MEMORY) == MATCH ) {
int freemem = getFreeVMMemSize();
ad->Assign(ATTR_VM_MEMORY, freemem);
}else if( strcasecmp(attr_name, ATTR_VM_NETWORKING) == MATCH ) {
ad->Assign(ATTR_VM_NETWORKING, m_vm_networking);
}else {
ExprTree * pTree = expr->Copy();
ad->Insert(attr_name, pTree, false);
}
}
// Now, we will publish mac and ip addresses of all guest VMs.
MyString all_macs;
MyString all_ips;
VMStarterInfo *info = NULL;
const char* guest_ip = NULL;
const char* guest_mac = NULL;
m_vm_starter_list.Rewind();
while( m_vm_starter_list.Next(info) ) {
guest_ip = info->getIPForVM();
if( guest_ip ) {
if( all_ips.IsEmpty() == false ) {
all_ips += ",";
}
all_ips += guest_ip;
}
guest_mac = info->getMACForVM();
if( guest_mac ) {
if( all_macs.IsEmpty() == false ) {
all_macs += ",";
}
all_macs += guest_mac;
}
}
if( all_ips.IsEmpty() == false ) {
ad->Assign(ATTR_VM_ALL_GUEST_IPS, all_ips);
}
if( all_macs.IsEmpty() == false ) {
ad->Assign(ATTR_VM_ALL_GUEST_MACS, all_macs);
}
}
int
unwind_attributes(classad_context cad, char *attribute_name, char ***results)
{
if (cad == NULL) return C_CLASSAD_INVALID_CONTEXT;
if ((results == NULL) || (attribute_name == NULL))
return C_CLASSAD_INVALID_ARG;
ClassAd *ad = (ClassAd *)cad;
ExprTree *et;
bool need_to_delete_et = false;
et = ad->Lookup(attribute_name);
if (et == NULL)
{
return C_CLASSAD_VALUE_NOT_FOUND;
}
if (et->GetKind() == ExprTree::LITERAL_NODE)
{
// The attribute was probably stringified. Try to parse it.
Value v;
EvalState state;
state.SetScopes( ad );
et->Evaluate(state,v);
std::string strres;
if (v.IsStringValue( strres ))
{
ClassAdParser parser;
et=NULL;
parser.ParseExpression(strres,et);
need_to_delete_et = true;
}
}
BinaryOpUnwind res_unp;
std::string result;
res_unp.Unparse(result, et);
int n_results;
if (*results == NULL)
{
n_results = 0;
(*results) = (char **)malloc(sizeof(char **));
if ((*results) == NULL) return C_CLASSAD_OUT_OF_MEMORY;
(*results)[0] = NULL;
}
else
{
for (n_results = 0; (*results)[n_results] != NULL; n_results++) /*NOP*/ ;
}
std::vector<std::string>::const_iterator it;
for (it = res_unp.m_unwind_output.begin();
it != res_unp.m_unwind_output.end(); ++it)
{
n_results++;
char **new_results;
new_results = (char **)realloc(*results, (n_results+1)*sizeof(char *));
if (new_results == NULL)
{
if (need_to_delete_et) delete et;
return C_CLASSAD_OUT_OF_MEMORY;
}
(*results) = new_results;
(*results)[n_results] = NULL;
(*results)[n_results-1] = strdup(it->c_str());
if (((*results)[n_results-1]) == NULL)
{
if (need_to_delete_et) delete et;
return C_CLASSAD_OUT_OF_MEMORY;
}
}
if (need_to_delete_et) delete et;
return C_CLASSAD_NO_ERROR;
}
void
LiveJobImpl::Set ( const char *_name, const char *_value )
{
if ( strcasecmp ( _name, ATTR_JOB_SUBMISSION ) == 0 )
{
std::string val = TrimQuotes( _value );
// TODO: grab the cluster from our key
PROC_ID id = getProcByString(m_job->GetKey());
if (m_job) {
m_job->SetSubmission ( val.c_str(), id.cluster );
}
}
// our status is changing...decrement for old one
if ( strcasecmp ( _name, ATTR_JOB_STATUS ) == 0 )
{
if ( m_job )
{
m_job->SetStatus(this->GetStatus());
m_job->DecrementSubmission ();
}
}
if ( strcasecmp ( _name, ATTR_OWNER ) == 0 )
{
// need to leave an owner for this job
// to be picked up soon
// if we are in here, we don't have m_submission
PROC_ID id = getProcByString(m_job->GetKey());
std::string val = TrimQuotes( _value );
g_ownerless_clusters[id.cluster] = val;
m_job->UpdateSubmission(id.cluster,val.c_str());
}
// parse the type
ExprTree *expr;
if ( ParseClassAdRvalExpr ( _value, expr ) )
{
dprintf ( D_ALWAYS,
"error: parsing %s[%s] = %s, skipping\n",
m_job->GetKey(), _name, _value );
return;
}
// add this value to the classad
classad::Value value;
expr->Evaluate(value);
switch ( value.GetType() )
{
case classad::Value::INTEGER_VALUE:
int i;
from_string<int> ( i, std::string ( _value ), std::dec );
m_full_ad->Assign ( _name, i );
break;
case classad::Value::REAL_VALUE:
float f;
from_string<float> ( f, std::string ( _value ), std::dec );
m_full_ad->Assign ( _name, f );
break;
case classad::Value::STRING_VALUE:
m_full_ad->Assign ( _name, _value );
break;
default:
m_full_ad->AssignExpr ( _name, _value );
break;
}
delete expr;
expr = NULL;
// our status has changed...increment for new one
if ( strcasecmp ( _name, ATTR_JOB_STATUS ) == 0 )
{
if ( m_job )
{
m_job->SetStatus(this->GetStatus());
m_job->IncrementSubmission ();
}
}
}