// Remove/replace characters from the string so it can be used as an attribute name // it changes the string that is passed to it. first leading an trailing spaces // are removed, then Characters that are invalid in compatible classads // (basically anthing but [a-zA-Z0-9_]) is replaced with chReplace. // if chReplace is 0, then invalid characters are removed. // if compact is true, then multiple consecutive runs of chReplace // are changed to a single instance. // return value is the length of the resulting string. // int cleanStringForUseAsAttr(MyString &str, char chReplace/*=0*/, bool compact/*=true*/) { // have 0 mean 'remove' since we can't actually use it as a replacement char // we'll actually implement it by replacing invalid chars with spaces, // and then compacting to remove all of the spaces. if (0 == chReplace) { chReplace = ' '; compact = true; } // trim the input and replace invalid chars with chReplace str.trim(); for (int ii = 0; ii < str.Length(); ++ii) { char ch = str[ii]; if (ch == '_' || (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z')) continue; str.setChar(ii,chReplace); } // if compact, convert runs of chReplace with a single instance, // unless chReplace is ' ', then remove them entirely. if (compact) { if (chReplace == ' ') str.replaceString(" ",""); else { MyString tmp; tmp += chReplace; tmp += chReplace; str.replaceString(tmp.Value(), tmp.Value()+1); } } str.trim(); return str.Length(); }
// iterate across the Job's var values, and for any which have $(JOB) in them, // substitute it. This substitution is draconian and will always happen. void Job::ResolveVarsInterpolations(void) { MyString *val; varValsFromDag->Rewind(); while( (val = varValsFromDag->Next()) != NULL ) { // XXX No way to escape $(JOB) in case, for some crazy reason, you // want a filename component actually to be '$(JOB)'. // It isn't hard to fix, I'll do it later. val->replaceString("$(JOB)", GetJobName()); } }
void SanitizeSubmitterName(MyString &name) { // We /may/ (will!) use the name as the name of an // attribute, so we must strip invalid characters that we // expect to find in the name. static const int invalid_char_len = 4; static const char *invalid_chars[invalid_char_len] = {"-", "@", ".", " "}; // XXX: Invert this, use [a-zA-Z_][a-zA-Z0-9_]* for (int i = 0; i < invalid_char_len; i++) { while (-1 != name.find(invalid_chars[i])) { name.replaceString(invalid_chars[i], "_"); } } }
int GLExecPrivSepHelper::run_script(ArgList& args,MyString &error_desc) { if (!proxy_valid_right_now()) { dprintf(D_ALWAYS, "GLExecPrivSepHelper::run_script: not invoking glexec since the proxy is not valid!\n"); error_desc += "The job proxy is not valid."; return INVALID_PROXY_RC; } /* Note that set_user_priv is a no-op if condor is running as non-root (the "usual" mode for invoking glexec) */ priv_state priv_saved = set_user_priv(); FILE* fp = my_popen(args, "r", TRUE); set_priv(priv_saved); if (fp == NULL) { dprintf(D_ALWAYS, "GLExecPrivSepHelper::run_script: " "my_popen failure on %s: errno=%d (%s)\n", args.GetArg(0), errno, strerror(errno)); return -1; } MyString str; while (str.readLine(fp, true)); priv_saved = set_user_priv(); int ret = my_pclose(fp); set_priv(priv_saved); if (ret != 0) { str.trim(); dprintf(D_ALWAYS, "GLExecPrivSepHelper::run_script: %s exited " "with status %d and following output:\n%s\n", args.GetArg(0), ret, str.Value()); error_desc.formatstr_cat("%s exited with status %d and the following output: %s", condor_basename(args.GetArg(0)), ret, str.Value()); error_desc.replaceString("\n","; "); } return ret; }
void handle_termination( PROC *proc, char *notification, int *jobstatus, char const *coredir ) { MyString escapedbuf; int status = *jobstatus; MyString coredir_buf; dprintf(D_FULLDEBUG, "handle_termination() called.\n"); ASSERT (JobAd != NULL ); switch( WTERMSIG(status) ) { case -1: /* On Digital Unix, WTERMSIG returns -1 if we weren't killed by a sig. This is the same case as sig 0 */ case 0: /* If core, bad executable -- otherwise a normal exit */ if( WCOREDUMP(status) && WEXITSTATUS(status) == ENOEXEC ) { (void)sprintf( notification, "is not executable." ); dprintf( D_ALWAYS, "Shadow: Job file not executable\n" ); ExitReason = JOB_KILLED; } else if( WCOREDUMP(status) && WEXITSTATUS(status) == 0 ) { (void)sprintf(notification, "was killed because it was not properly linked for execution \nwith Version 6 Condor.\n" ); MainSymbolExists = FALSE; ExitReason = JOB_KILLED; } else { (void)sprintf(notification, "exited with status %d.", WEXITSTATUS(status) ); dprintf(D_ALWAYS, "Shadow: Job exited normally with status %d\n", WEXITSTATUS(status) ); ExitReason = JOB_EXITED; JobExitStatus = WEXITSTATUS(status); } proc->status = COMPLETED; proc->completion_date = time( (time_t *)0 ); JobAd->Assign(ATTR_ON_EXIT_BY_SIGNAL,false); JobAd->Assign( ATTR_ON_EXIT_CODE, WEXITSTATUS(status)); // set up the terminate pending "state" JobAd->Assign(ATTR_TERMINATION_PENDING,true); // this can have newlines and crap in it, for now, just replace them // with spaces. I know it is ugly, but the classadlog class which // writes the job queue log file can't have literal newlines and such // in the values. :( escapedbuf = notification; escapedbuf.replaceString("\n", " "); escapedbuf.replaceString("\t", " "); JobAd->Assign(ATTR_TERMINATION_REASON,escapedbuf.Value()); JobAd->Assign(ATTR_TERMINATION_EXITREASON, ExitReason); break; case SIGKILL: /* Kicked off without a checkpoint */ dprintf(D_ALWAYS, "Shadow: Job was kicked off without a checkpoint\n" ); DoCleanup(); ExitReason = JOB_NOT_CKPTED; #if 0 /* This is a problem for the new feature where we choose our executable dynamically, so don't do it. */ if( stat(ICkptName,&st_buf) < 0) { dprintf(D_ALWAYS,"No initial ckpt found\n"); ExitReason = JOB_NO_CKPT_FILE; } #endif /* in here, we disregard what the user wanted. Otherwise doing a condor_rm will result in the wanting to be resubmitted or held by the shadow. */ JobAd->Assign(ATTR_ON_EXIT_HOLD_CHECK,false); JobAd->Assign(ATTR_ON_EXIT_REMOVE_CHECK,true); // set up the terminate pending "state" JobAd->Assign(ATTR_TERMINATION_PENDING,true); JobAd->Assign(ATTR_TERMINATION_EXITREASON,ExitReason); break; case SIGQUIT: /* Kicked off, but with a checkpoint */ dprintf(D_ALWAYS, "Shadow: Job was checkpointed\n" ); proc->status = IDLE; ExitReason = JOB_CKPTED; /* in here, we disregard what the user wanted. Otherwise doing a condor_vacate will result in the wanting to be resubmitted or held by the shadow. */ JobAd->Assign(ATTR_ON_EXIT_HOLD_CHECK,false); // this can have newlines and crap in it, for now, just replace them // with spaces. I know it is ugly, but the classadlog class which // writes the job queue log file can't have literal newlines and such // in the values. :( escapedbuf = notification; escapedbuf.replaceString("\n", " "); escapedbuf.replaceString("\t", " "); JobAd->Assign(ATTR_TERMINATION_REASON, escapedbuf.Value()); JobAd->Assign(ATTR_ON_EXIT_REMOVE_CHECK,true); /* add in the signature of the checkpointing host for this completed ckpt */ if (LastCkptPlatform != NULL) { JobAd->Assign(ATTR_LAST_CHECKPOINT_PLATFORM,LastCkptPlatform); } break; default: /* Job exited abnormally */ if (coredir == NULL) { ASSERT( condor_getcwd(coredir_buf) ); coredir = coredir_buf.Value(); } if( WCOREDUMP(status) ) { MyString corepath; if( strcmp(proc->rootdir, "/") == 0 ) { (void)sprintf(notification, "was killed by signal %d.\nCore file is %s/core.%d.%d.", WTERMSIG(status) , coredir, proc->id.cluster, proc->id.proc); corepath.sprintf("%s/core.%d.%d", coredir, proc->id.cluster, proc->id.proc); } else { (void)sprintf(notification, "was killed by signal %d.\nCore file is %s%s/core.%d.%d.", WTERMSIG(status) ,proc->rootdir, coredir, proc->id.cluster, proc->id.proc); corepath.sprintf("%s%s/core.%d.%d", proc->rootdir, coredir, proc->id.cluster, proc->id.proc); } JobAd->Assign(ATTR_JOB_CORE_FILENAME,corepath.Value()); ExitReason = JOB_COREDUMPED; } else { (void)sprintf(notification, "was killed by signal %d.", WTERMSIG(status)); ExitReason = JOB_KILLED; } dprintf(D_ALWAYS, "Shadow: %s\n", notification); proc->status = COMPLETED; proc->completion_date = time( (time_t *)0 ); JobAd->Assign(ATTR_ON_EXIT_BY_SIGNAL,true); JobAd->Assign(ATTR_ON_EXIT_SIGNAL,WTERMSIG(status)); // set up the terminate pending "state" JobAd->Assign(ATTR_TERMINATION_PENDING,true); // this can have newlines and crap in it, for now, just replace them // with spaces. I know it is ugly, but the classadlog class which // writes the job queue log file can't have literal newlines and such // in the values. :( escapedbuf = notification; escapedbuf.replaceString("\n", " "); escapedbuf.replaceString("\t", " "); JobAd->Assign(ATTR_TERMINATION_REASON,escapedbuf.Value()); JobAd->Assign(ATTR_TERMINATION_EXITREASON, ExitReason); break; } }
//------------------------------------------------------------------------- bool condor_submit( const Dagman &dm, const char* cmdFile, CondorID& condorID, const char* DAGNodeName, MyString &DAGParentNodeNames, List<Job::NodeVar> *vars, int retry, const char* directory, const char *workflowLogFile, bool hold_claim ) { TmpDir tmpDir; MyString errMsg; if ( !tmpDir.Cd2TmpDir( directory, errMsg ) ) { debug_printf( DEBUG_QUIET, "Could not change to node directory %s: %s\n", directory, errMsg.Value() ); return false; } ArgList args; // construct arguments to condor_submit to add attributes to the // job classad which identify the job's node name in the DAG, the // node names of its parents in the DAG, and the job ID of DAGMan // itself; then, define submit_event_notes to print the job's node // name inside the submit event in the userlog // NOTE: we specify the job ID of DAGMan using only its cluster ID // so that it may be referenced by jobs in their priority // attribute (which needs an int, not a string). Doing so allows // users to effectively "batch" jobs by DAG so that when they // submit many DAGs to the same schedd, all the ready jobs from // one DAG complete before any jobs from another begin. args.AppendArg( dm.condorSubmitExe ); args.AppendArg( "-a" ); MyString nodeName = MyString(ATTR_DAG_NODE_NAME_ALT) + " = " + DAGNodeName; args.AppendArg( nodeName.Value() ); // append a line adding the parent DAGMan's cluster ID to the job ad args.AppendArg( "-a" ); MyString dagJobId = MyString( "+" ) + ATTR_DAGMAN_JOB_ID + " = " + dm.DAGManJobId._cluster; args.AppendArg( dagJobId.Value() ); // now we append a line setting the same thing as a submit-file macro // (this is necessary so the user can reference it in the priority) args.AppendArg( "-a" ); MyString dagJobIdMacro = MyString( "" ) + ATTR_DAGMAN_JOB_ID + " = " + dm.DAGManJobId._cluster; args.AppendArg( dagJobIdMacro.Value() ); args.AppendArg( "-a" ); MyString submitEventNotes = MyString( "submit_event_notes = DAG Node: " ) + DAGNodeName; args.AppendArg( submitEventNotes.Value() ); ASSERT( workflowLogFile ); // We need to append the DAGman default log file to // the log file list args.AppendArg( "-a" ); std::string dlog( "dagman_log = " ); dlog += workflowLogFile; args.AppendArg( dlog.c_str() ); debug_printf( DEBUG_VERBOSE, "Adding a DAGMan workflow log %s\n", workflowLogFile ); // Now append the mask debug_printf( DEBUG_VERBOSE, "Masking the events recorded in the DAGMAN workflow log\n" ); args.AppendArg( "-a" ); std::string dmask("+"); dmask += ATTR_DAGMAN_WORKFLOW_MASK; dmask += " = \""; const char *eventMask = getEventMask(); debug_printf( DEBUG_VERBOSE, "Mask for workflow log is %s\n", eventMask ); dmask += eventMask; dmask += "\""; args.AppendArg( dmask.c_str() ); // Suppress the job's log file if that option is enabled. if ( dm._suppressJobLogs ) { debug_printf( DEBUG_VERBOSE, "Suppressing node job log file\n" ); args.AppendArg( "-a" ); args.AppendArg( "log = ''" ); } ArgList parentNameArgs; parentNameArgs.AppendArg( "-a" ); MyString parentNodeNames = MyString( "+DAGParentNodeNames = " ) + "\"" + DAGParentNodeNames + "\""; parentNameArgs.AppendArg( parentNodeNames.Value() ); // set any VARS specified in the DAG file MyString anotherLine; ListIterator<Job::NodeVar> varsIter(*vars); Job::NodeVar nodeVar; while ( varsIter.Next(nodeVar) ) { // Substitute the node retry count if necessary. Note that // we can't do this in Job::ResolveVarsInterpolations() // because that's only called at parse time. MyString value = nodeVar._value; MyString retryStr( retry ); value.replaceString( "$(RETRY)", retryStr.Value() ); MyString varStr = nodeVar._name + " = " + value; args.AppendArg( "-a" ); args.AppendArg( varStr.Value() ); } // Set the special DAG_STATUS variable (mainly for use by // "final" nodes). args.AppendArg( "-a" ); MyString var = "DAG_STATUS = "; var += dm.dag->_dagStatus; args.AppendArg( var.Value() ); // Set the special FAILED_COUNT variable (mainly for use by // "final" nodes). args.AppendArg( "-a" ); var = "FAILED_COUNT = "; var += dm.dag->NumNodesFailed(); args.AppendArg( var.Value() ); // how big is the command line so far MyString display; args.GetArgsStringForDisplay( &display ); int cmdLineSize = display.Length(); parentNameArgs.GetArgsStringForDisplay( &display ); int DAGParentNodeNamesLen = display.Length(); // how many additional chars must we still add to command line // NOTE: according to the POSIX spec, the args + // environ given to exec() cannot exceed // _POSIX_ARG_MAX, so we also need to calculate & add // the size of environ** to reserveNeeded int reserveNeeded = strlen( cmdFile ); int maxCmdLine = _POSIX_ARG_MAX; // if we don't have room for DAGParentNodeNames, leave it unset if( cmdLineSize + reserveNeeded + DAGParentNodeNamesLen > maxCmdLine ) { debug_printf( DEBUG_NORMAL, "Warning: node %s has too many parents " "to list in its classad; leaving its DAGParentNodeNames " "attribute undefined\n", DAGNodeName ); check_warning_strictness( DAG_STRICT_3 ); } else { args.AppendArgsFromArgList( parentNameArgs ); } if( hold_claim ){ args.AppendArg( "-a" ); MyString holdit = MyString("+") + MyString(ATTR_JOB_KEEP_CLAIM_IDLE) + " = " + dm._claim_hold_time; args.AppendArg( holdit.Value() ); } if (dm._submitDagDeepOpts.suppress_notification) { args.AppendArg( "-a" ); MyString notify = MyString("notification = never"); args.AppendArg( notify.Value() ); } args.AppendArg( cmdFile ); bool success = do_submit( args, condorID, dm.prohibitMultiJobs ); if ( !tmpDir.Cd2MainDir( errMsg ) ) { debug_printf( DEBUG_QUIET, "Could not change to original directory: %s\n", errMsg.Value() ); success = false; } return success; }
/** * This is the logic that is used to take a parameter string * given for a specific field in the cron schedule and expand it * out into a range of int's that can be looked up quickly later on * We must be given the index number of the field we're going to parse * and a min/max for the range of values allowed for the attribute. * If the parameter is invalid, we will report an error and return false * This will prevent them from querying nextRunTime() for runtimes * * @param attribute_idx - the index for the parameter in CronTab::attributes * @param min - the mininum value in the range for this parameter * @param max - the maximum value in the range for this parameter * @return true if we were able to create the range of values **/ bool CronTab::expandParameter( int attribute_idx, int min, int max ) { MyString *param = this->parameters[attribute_idx]; ExtArray<int> *list = this->ranges[attribute_idx]; // // Make sure the parameter is valid // The validation method will have already printed out // the error message to the log // MyString error; if ( ! CronTab::validateParameter( attribute_idx, param->Value(), error ) ) { dprintf( D_ALWAYS, "%s", error.Value() ); // // Store the error in case they want to email // the user to tell them that they goofed // CronTab::errorLog += error; return ( false ); } // // Remove any spaces // param->replaceString(" ", ""); // // Now here's the tricky part! We need to expand their parameter // out into a range that can be put in array of integers // First start by spliting the string by commas // param->Tokenize(); const char *_token; while ( ( _token = param->GetNextToken( CRONTAB_DELIMITER, true ) ) != NULL ) { MyString token( _token ); int cur_min = min, cur_max = max, cur_step = 1; // ------------------------------------------------- // STEP VALUES // The step value is independent of whether we have // a range, the wildcard, or a single number. // ------------------------------------------------- if ( token.find( CRONTAB_STEP ) > 0 ) { // // Just look for the step value to replace // the current step value. The other code will // handle the rest // token.Tokenize(); const char *_temp; // // Take out the numerator, keep it for later // const char *_numerator = token.GetNextToken( CRONTAB_STEP, true ); if ( ( _temp = token.GetNextToken( CRONTAB_STEP, true ) ) != NULL ) { MyString stepStr( _temp ); stepStr.trim(); cur_step = atoi( stepStr.Value() ); } // // Now that we have the denominator, put the numerator back // as the token. This makes it easier to parse later on // token = _numerator; } // STEP // ------------------------------------------------- // RANGE // If it's a range, expand the range but make sure we // don't go above/below our limits // Note that the find will ignore the token if the // range delimiter is in the first character position // ------------------------------------------------- if ( token.find( CRONTAB_RANGE ) > 0 ) { // // Split out the integers // token.Tokenize(); MyString *_temp; int value; // // Min // _temp = new MyString( token.GetNextToken( CRONTAB_RANGE, true ) ); _temp->trim(); value = atoi( _temp->Value() ); cur_min = ( value >= min ? value : min ); delete _temp; // // Max // _temp = new MyString( token.GetNextToken( CRONTAB_RANGE, true ) ); _temp->trim(); value = atoi( _temp->Value() ); cur_max = ( value <= max ? value : max ); delete _temp; // ------------------------------------------------- // WILDCARD // This will select all values for the given range // ------------------------------------------------- } else if ( token.find( CRONTAB_WILDCARD ) >= 0 ) { // // For this we do nothing since it will just // be the min-max range // // Day of Week Special Case // The day of week specifier is kind of weird // If it's the wildcard, then it doesn't mean // include all like the other fields. The reason // why we don't want to do that is because later // on we are going to expand the day of the week // field to be actual dates in a month in order // to figure out when to run next, so we don't // want to expand out the wildcard for all days // if the day of month field is restricted // Read the cron manpage! // if ( attribute_idx == CRONTAB_DOW_IDX ) { continue; } // ------------------------------------------------- // SINGLE VALUE // They just want a single value to be added // Note that a single value with a step like "2/3" will // work in this code but its meaningless unless its whole number // ------------------------------------------------- } else { // // Replace the range to be just this value only if it // fits in the min/max range we were given // int value = atoi(token.Value()); if ( value >= min && value <= max ) { cur_min = cur_max = value; } } // // Fill out the numbers based on the range using // the step value // int ctr; for ( ctr = cur_min; ctr <= cur_max; ctr++ ) { // // Day of Week Special Case // The crontab specifications lets Sunday be // represented with either a 0 or a 7. Our // dayOfWeek() method in date_util.h uses 0-6 // So if this the day of the week attribute and // we are given a 7 for Sunday, just convert it // to a zero // int temp = ctr; if ( attribute_idx == CRONTAB_DOW_IDX && temp == CRONTAB_DAY_OF_WEEK_MAX ) { temp = CRONTAB_DAY_OF_WEEK_MIN; } // // Make sure this value isn't alreay added and // that it falls in our step listing for the range // if ( ( ( temp % cur_step ) == 0 ) && !this->contains( *list, temp ) ) { list->add( temp ); } } // FOR } // WHILE // // Sort! Makes life easier later on // this->sort( *list ); return ( true ); }
//------------------------------------------------------------------------- bool GetConfigAndAttrs( /* const */ StringList &dagFiles, bool useDagDir, MyString &configFile, StringList &attrLines, MyString &errMsg ) { bool result = true; // Note: destructor will change back to original directory. TmpDir dagDir; dagFiles.rewind(); char *dagFile; while ( (dagFile = dagFiles.next()) != NULL ) { // // Change to the DAG file's directory if necessary, and // get the filename we need to use for it from that directory. // const char * newDagFile; if ( useDagDir ) { MyString tmpErrMsg; if ( !dagDir.Cd2TmpDirFile( dagFile, tmpErrMsg ) ) { AppendError( errMsg, MyString("Unable to change to DAG directory ") + tmpErrMsg ); return false; } newDagFile = condor_basename( dagFile ); } else { newDagFile = dagFile; } StringList configFiles; // Note: destructor will close file. MultiLogFiles::FileReader reader; errMsg = reader.Open( newDagFile ); if ( errMsg != "" ) { return false; } MyString logicalLine; while ( reader.NextLogicalLine( logicalLine ) ) { if ( logicalLine != "" ) { // Note: StringList constructor removes leading // whitespace from lines. StringList tokens( logicalLine.Value(), " \t" ); tokens.rewind(); const char *firstToken = tokens.next(); if ( !strcasecmp( firstToken, "config" ) ) { // Get the value. const char *newValue = tokens.next(); if ( !newValue || !strcmp( newValue, "" ) ) { AppendError( errMsg, "Improperly-formatted " "file: value missing after keyword " "CONFIG" ); result = false; } else { // Add the value we just found to the config // files list (if it's not already in the // list -- we don't want duplicates). configFiles.rewind(); char *existingValue; bool alreadyInList = false; while ( ( existingValue = configFiles.next() ) ) { if ( !strcmp( existingValue, newValue ) ) { alreadyInList = true; } } if ( !alreadyInList ) { // Note: append copies the string here. configFiles.append( newValue ); } } //some DAG commands are needed for condor_submit_dag, too... } else if ( !strcasecmp( firstToken, "SET_JOB_ATTR" ) ) { // Strip of DAGMan-specific command name; the // rest we pass to the submit file. logicalLine.replaceString( "SET_JOB_ATTR", "" ); logicalLine.trim(); if ( logicalLine == "" ) { AppendError( errMsg, "Improperly-formatted " "file: value missing after keyword " "SET_JOB_ATTR" ); result = false; } else { attrLines.append( logicalLine.Value() ); } } } } reader.Close(); // // Check the specified config file(s) against whatever we // currently have, setting the config file if it hasn't // been set yet, flagging an error if config files conflict. // configFiles.rewind(); char * cfgFile; while ( (cfgFile = configFiles.next()) ) { MyString cfgFileMS = cfgFile; MyString tmpErrMsg; if ( MakePathAbsolute( cfgFileMS, tmpErrMsg ) ) { if ( configFile == "" ) { configFile = cfgFileMS; } else if ( configFile != cfgFileMS ) { AppendError( errMsg, MyString("Conflicting DAGMan ") + "config files specified: " + configFile + " and " + cfgFileMS ); result = false; } } else { AppendError( errMsg, tmpErrMsg ); result = false; } } // // Go back to our original directory. // MyString tmpErrMsg; if ( !dagDir.Cd2MainDir( tmpErrMsg ) ) { AppendError( errMsg, MyString("Unable to change to original directory ") + tmpErrMsg ); result = false; } } return result; }