/** Parse a JOB or SUBDAG line from a DAG file.
@param dagLine: the line we're parsing
@param tokens: tokens of this line
@param fileType: "submit" or "DAG" (to be used in error message)
@param submitOrDagFile: if successful, this will point to submit or
nested DAG file name
@param directory: if successful, this will point to directory (NULL
if not specified)
@return 0 if successful, 1 if failed
*/
int
parseJobOrDagLine( const char *dagLine, StringList &tokens,
const char *fileType, const char *&submitOrDagFile,
const char *&directory )
{
const char *nodeName = tokens.next();
if ( !nodeName) {
fprintf( stderr, "No node name specified in line: <%s>\n", dagLine );
return 1;
}
submitOrDagFile = tokens.next();
if ( !submitOrDagFile ) {
fprintf( stderr, "No %s file specified in "
"line: <%s>\n", fileType, dagLine );
return 1;
}
directory = NULL;
const char *dirKeyword = tokens.next();
if ( dirKeyword && !strcasecmp( dirKeyword, "DIR" ) ) {
directory = tokens.next();
if ( !directory ) {
fprintf( stderr, "No directory specified in "
"line: <%s>\n", dagLine );
return 1;
}
}
return 0;
}
void
DaemonList::init( daemon_t type, const char* host_list, const char* pool_list )
{
Daemon* tmp;
char* host;
char const *pool = NULL;
StringList foo;
StringList pools;
if( host_list ) {
foo.initializeFromString( host_list );
foo.rewind();
}
if( pool_list ) {
pools.initializeFromString( pool_list );
pools.rewind();
}
while( true ) {
host = foo.next();
pool = pools.next();
if( !host && !pool ) {
break;
}
tmp = buildDaemon( type, host, pool );
append( tmp );
}
}
bool
IpVerify::lookup_user(NetStringList *hosts, UserHash_t *users, char const *user, char const *ip, char const *hostname, bool is_allow_list)
{
if( !hosts || !users ) {
return false;
}
ASSERT( user );
// we look up by ip OR by hostname, not both
ASSERT( !ip || !hostname );
ASSERT( ip || hostname);
StringList hostmatches;
if( ip ) {
hosts->find_matches_withnetwork(ip,&hostmatches);
}
else if( hostname ) {
hosts->find_matches_anycase_withwildcard(hostname,&hostmatches);
}
char const * hostmatch;
hostmatches.rewind();
while( (hostmatch=hostmatches.next()) ) {
StringList *userlist;
ASSERT( users->lookup(hostmatch,userlist) != -1 );
if (userlist->contains_anycase_withwildcard(user)) {
dprintf ( D_SECURITY, "IPVERIFY: matched user %s from %s to %s list\n",
user, hostmatch, is_allow_list ? "allow" : "deny" );
return true;
}
}
return false;
}
/** Get the command-line options we want to preserve from the .condor.sub
file we're overwriting, and plug them into the shallowOpts structure.
Note that it's *not* an error for the .condor.sub file to not exist.
@param shallowOpts: the condor_submit_dag shallow options
@return 0 if successful, 1 if failed
*/
int
getOldSubmitFlags(SubmitDagShallowOptions &shallowOpts)
{
// It's not an error for the submit file to not exist.
if ( fileExists( shallowOpts.strSubFile ) ) {
StringList logicalLines;
MyString error = MultiLogFiles::fileNameToLogicalLines(
shallowOpts.strSubFile, logicalLines );
if ( error != "" ) {
fprintf( stderr, "Error reading submit file: %s\n",
error.Value() );
return 1;
}
logicalLines.rewind();
const char *subLine;
while ( (subLine = logicalLines.next()) ) {
StringList tokens( subLine, " \t" );
tokens.rewind();
const char *first = tokens.next();
if ( first && !strcasecmp( first, "arguments" ) ) {
if ( parseArgumentsLine( subLine, shallowOpts ) != 0 ) {
return 1;
}
}
}
}
return 0;
}
DCloudResource::BatchStatusResult DCloudResource::StartBatchStatus()
{
ASSERT(status_gahp);
StringList instance_ids;
StringList statuses;
const char *instance_id;
const char *status;
int rc = status_gahp->dcloud_status_all( ResourceName(), m_username,
m_password, instance_ids,
statuses );
if ( rc == GAHPCLIENT_COMMAND_PENDING ) {
return BSR_PENDING;
}
if ( rc != 0 ) {
dprintf( D_ALWAYS, "Error attempting a Deltacloud batch status query: %s\n", status_gahp->getErrorString() );
return BSR_ERROR;
}
DCloudJob *next_job;
List<DCloudJob> my_jobs;
registeredJobs.Rewind();
while ( (next_job = (DCloudJob *)registeredJobs.Next()) ) {
my_jobs.Insert( next_job );
}
instance_ids.rewind();
statuses.rewind();
while ( (instance_id = instance_ids.next()) &&
(status = statuses.next()) ) {
MyString hashname;
hashname.formatstr( "%s#%s", ResourceName(), instance_id );
DCloudJob *job = NULL;
// TODO We can get rid of the hashtable.
rc = DCloudJob::JobsByInstanceId.lookup(
HashKey( hashname.Value() ), job );
if ( rc != 0 ) {
// Job not found. Probably okay; we might see jobs
// submitted via other means, or jobs we've abandoned.
dprintf( D_FULLDEBUG, "Job %s on remote host is unknown. Skipping.\n", hashname.Value() );
continue;
}
ASSERT( job );
job->StatusUpdate( status );
my_jobs.Delete( job );
}
my_jobs.Rewind();
while ( (next_job = my_jobs.Next()) ) {
next_job->StatusUpdate( NULL );
}
return BSR_DONE;
}
int set_status_print_mask_from_stream (
const char * streamid,
bool is_filename,
const char ** pconstraint)
{
std::string where_expr;
std::string messages;
StringList attrs;
printmask_aggregation_t aggregation;
SimpleInputStream * pstream = NULL;
*pconstraint = NULL;
FILE *file = NULL;
if (MATCH == strcmp("-", streamid)) {
pstream = new SimpleFileInputStream(stdin, false);
} else if (is_filename) {
file = safe_fopen_wrapper_follow(streamid, "r");
if (file == NULL) {
fprintf(stderr, "Can't open select file: %s\n", streamid);
return -1;
}
pstream = new SimpleFileInputStream(file, true);
} else {
pstream = new StringLiteralInputStream(streamid);
}
ASSERT(pstream);
int err = SetAttrListPrintMaskFromStream(
*pstream,
*getCondorStatusPrintFormats(),
pm,
pmHeadFoot,
aggregation,
group_by_keys,
where_expr,
attrs,
messages);
delete pstream; pstream = NULL;
if ( ! err) {
if (aggregation != PR_NO_AGGREGATION) {
fprintf(stderr, "print-format aggregation not supported\n");
return -1;
}
if ( ! where_expr.empty()) {
*pconstraint = pm.store(where_expr.c_str());
//if ( ! validate_constraint(*pconstraint)) {
// formatstr_cat(messages, "WHERE expression is not valid: %s\n", *pconstraint);
//}
}
// convert projection list into the format that condor status likes. because programmers.
attrs.rewind();
const char * attr;
while ((attr = attrs.next())) { projList.AppendArg(attr); }
}
if ( ! messages.empty()) { fprintf(stderr, "%s", messages.c_str()); }
return err;
}
MyString
MultiLogFiles::loadValueFromSubFile(const MyString &strSubFilename,
const MyString &directory, const char *keyword)
{
dprintf( D_FULLDEBUG, "MultiLogFiles::loadValueFromSubFile(%s, %s, %s)\n",
strSubFilename.Value(), directory.Value(), keyword );
TmpDir td;
if ( directory != "" ) {
MyString errMsg;
if ( !td.Cd2TmpDir(directory.Value(), errMsg) ) {
dprintf(D_ALWAYS, "Error from Cd2TmpDir: %s\n", errMsg.Value());
return "";
}
}
StringList logicalLines;
if ( fileNameToLogicalLines( strSubFilename, logicalLines ) != "" ) {
return "";
}
MyString value("");
// Now look through the submit file logical lines to find the
// value corresponding to the keyword.
const char *logicalLine;
while( (logicalLine = logicalLines.next()) != NULL ) {
MyString submitLine(logicalLine);
MyString tmpValue = getParamFromSubmitLine(submitLine, keyword);
if ( tmpValue != "" ) {
value = tmpValue;
}
}
//
// Check for macros in the value -- we currently don't
// handle those.
//
if ( value != "" ) {
if ( strchr(value.Value(), '$') ) {
dprintf(D_ALWAYS, "MultiLogFiles: macros not allowed "
"in %s in DAG node submit files\n", keyword);
value = "";
}
}
if ( directory != "" ) {
MyString errMsg;
if ( !td.Cd2MainDir(errMsg) ) {
dprintf(D_ALWAYS, "Error from Cd2MainDir: %s\n", errMsg.Value());
return "";
}
}
return value;
}
void
cleanup_execute_dirs( StringList &list )
{
char const *exec_path;
list.rewind();
while( (exec_path = list.next()) ) {
#if defined(WIN32)
dynuser nobody_login;
// remove all users matching this prefix
nobody_login.cleanup_condor_users("condor-run-");
// get rid of everything in the execute directory
Directory execute_dir(exec_path);
execute_dir.Rewind();
while ( execute_dir.Next() ) {
check_recovery_file( execute_dir.GetFullPath() );
}
execute_dir.Remove_Entire_Directory();
#else
// if we're using PrivSep, the Switchboard will only allow
// us to remove subdirectories of EXECUTE - so we need to
// list them and ask the Switchboard to delete each one
//
pair_strings_vector root_dirs = root_dir_list();
for (pair_strings_vector::const_iterator it=root_dirs.begin(); it != root_dirs.end(); ++it) {
const char * exec_path_full = dirscat(it->second.c_str(), exec_path);
if(exec_path_full) {
dprintf(D_FULLDEBUG, "Looking at %s\n",exec_path_full);
}
Directory execute_dir( exec_path_full, PRIV_ROOT );
execute_dir.Rewind();
while ( execute_dir.Next() ) {
check_recovery_file( execute_dir.GetFullPath() );
}
if (privsep_enabled()) {
execute_dir.Rewind();
while (execute_dir.Next()) {
dprintf(D_FULLDEBUG, "Attempting to remove %s\n",execute_dir.GetFullPath());
privsep_remove_dir(execute_dir.GetFullPath());
}
}
else {
execute_dir.Remove_Entire_Directory();
}
delete [] exec_path_full;
}
#endif
}
}
void
check_execute_dir_perms( StringList &list )
{
char const *exec_path;
list.rewind();
while( (exec_path = list.next()) ) {
check_execute_dir_perms( exec_path );
}
}
MyString
MultiLogFiles::CombineLines(StringList &listIn, char continuation,
const MyString &filename, StringList &listOut)
{
dprintf( D_FULLDEBUG, "MultiLogFiles::CombineLines(%s, %c)\n",
filename.Value(), continuation );
listIn.rewind();
// Physical line is one line in the file.
const char *physicalLine;
while ( (physicalLine = listIn.next()) != NULL ) {
// Logical line is physical lines combined as needed by
// continuation characters (backslash).
MyString logicalLine(physicalLine);
while ( logicalLine[logicalLine.Length()-1] == continuation ) {
// Remove the continuation character.
logicalLine.setChar(logicalLine.Length()-1, '\0');
// Append the next physical line.
physicalLine = listIn.next();
if ( physicalLine ) {
logicalLine += physicalLine;
} else {
MyString result = MyString("Improper file syntax: ") +
"continuation character with no trailing line! (" +
logicalLine + ") in file " + filename;
dprintf(D_ALWAYS, "MultiLogFiles: %s\n", result.Value());
return result;
}
}
listOut.append(logicalLine.Value());
}
return ""; // blank means okay
}
bool TestClear() {
BEGIN_TEST;
StringList list;
list.push_front("bar");
EXPECT_NONNULL(list.first());
list.clear();
EXPECT_NULL(list.next());
EXPECT_NULL(list.first());
EXPECT_EQ(list.length(), 0);
END_TEST;
}
int
MultiLogFiles::getQueueCountFromSubmitFile(const MyString &strSubFilename,
const MyString &directory, MyString &errorMsg)
{
dprintf( D_FULLDEBUG,
"MultiLogFiles::getQueueCountFromSubmitFile(%s, %s)\n",
strSubFilename.Value(), directory.Value() );
int queueCount = 0;
errorMsg = "";
MyString fullpath("");
if ( directory != "" ) {
fullpath = directory + DIR_DELIM_STRING + strSubFilename;
} else {
fullpath = strSubFilename;
}
StringList logicalLines;
if ( (errorMsg = fileNameToLogicalLines( strSubFilename,
logicalLines)) != "" ) {
return -1;
}
// Now look through the submit file logical lines to find any
// queue commands, and count up the total number of job procs
// to be queued.
const char * paramName = "queue";
const char *logicalLine;
while( (logicalLine = logicalLines.next()) != NULL ) {
MyString submitLine(logicalLine);
submitLine.Tokenize();
const char *DELIM = " ";
const char *rawToken = submitLine.GetNextToken( DELIM, true );
if ( rawToken ) {
MyString token(rawToken);
token.trim();
if ( !strcasecmp(token.Value(), paramName) ) {
rawToken = submitLine.GetNextToken( DELIM, true );
if ( rawToken ) {
queueCount += atoi( rawToken );
} else {
queueCount++;
}
}
}
}
return queueCount;
}
// Utility function.
void
pushStringListBack( std::vector< YourString > & v, StringList & sl ) {
const char * text = NULL;
sl.rewind();
int count = 0;
if( sl.number() > 0 ) {
while( (text = sl.next()) ) {
v.push_back( text );
++count;
}
}
ASSERT( count == sl.number() );
v.push_back( NULLSTRING );
}
bool
isSuperUser( const char* user )
{
if( ! (user)) {
return false;
}
super_users.rewind();
char * next;
while ((next = super_users.next())) {
if (strcmp (user, next ) == 0) {
return true;
}
}
return false;
}
// passing over REPLICATION_LIST configuration parameter, turning all the
// addresses into canonical <ip:port> form and inserting them all, except for
// the address of local replication daemon, into 'm_replicationDaemonsList'.
void
AbstractReplicatorStateMachine::initializeReplicationList( char* buffer )
{
StringList replicationAddressList;
char* replicationAddress = NULL;
bool isMyAddressPresent = false;
Sinful my_addr( daemonCore->InfoCommandSinfulString( ) );
replicationAddressList.initializeFromString( buffer );
// initializing a list unrolls it, that's why the rewind is needed to bring
// it to the beginning
replicationAddressList.rewind( );
/* Passing through the REPLICATION_LIST configuration parameter, stripping
* the optional <> brackets off, and extracting the host name out of
* either ip:port or hostName:port entries
*/
while( (replicationAddress = replicationAddressList.next( )) ) {
char* sinfulAddress = utilToSinful( replicationAddress );
if( sinfulAddress == NULL ) {
char bufArray[BUFSIZ];
sprintf( bufArray,
"AbstractReplicatorStateMachine::initializeReplicationList"
" invalid address %s\n", replicationAddress );
utilCrucialError( bufArray );
continue;
}
if( my_addr.addressPointsToMe( Sinful(sinfulAddress) ) ) {
isMyAddressPresent = true;
}
else {
m_replicationDaemonsList.insert( sinfulAddress );
}
// pay attention to release memory allocated by malloc with free and by
// new with delete here utilToSinful returns memory allocated by malloc
free( sinfulAddress );
}
if( !isMyAddressPresent ) {
utilCrucialError( "ReplicatorStateMachine::initializeReplicationList "
"my address is not present in REPLICATION_LIST" );
}
}
void
IpVerify::UserHashToString(UserHash_t *user_hash, MyString &result)
{
ASSERT( user_hash );
user_hash->startIterations();
MyString host;
StringList *users;
char const *user;
while( user_hash->iterate(host,users) ) {
if( users ) {
users->rewind();
while( (user=users->next()) ) {
result.sprintf_cat(" %s/%s",
user,
host.Value());
}
}
}
}
void
ReplicatorStateMachine::initializeClassAd()
{
if( m_classAd != NULL) {
delete m_classAd;
m_classAd = NULL;
}
m_classAd = new ClassAd();
SetMyTypeName(*m_classAd, "Replication");
SetTargetTypeName(*m_classAd, "");
m_name.formatstr( "replication@%s -p %d", get_local_fqdn().Value(),
daemonCore->InfoCommandPort( ) );
m_classAd->Assign( ATTR_NAME, m_name.Value( ) );
m_classAd->Assign( ATTR_MY_ADDRESS,
daemonCore->InfoCommandSinfulString( ) );
// publish list of replication nodes
char* buffer = param( "REPLICATION_LIST" );
if ( NULL == buffer ) {
EXCEPT( "ReplicatorStateMachine: No replication list!!" );
}
char* replAddress = NULL;
StringList replList;
MyString attrReplList;
MyString comma;
replList.initializeFromString( buffer );
replList.rewind( );
while( ( replAddress = replList.next() ) ) {
attrReplList += comma;
attrReplList += replAddress;
comma = ",";
}
m_classAd->Assign( ATTR_REPLICATION_LIST, attrReplList.Value( ) );
// publish DC attributes
daemonCore->publish(m_classAd);
free(buffer);
}
/** Recursively call condor_submit_dag on nested DAGs.
@param deepOpts: the condor_submit_dag deep options
@return 0 if successful, 1 if failed
*/
int
doRecursion( SubmitDagDeepOptions &deepOpts,
SubmitDagShallowOptions &shallowOpts )
{
int result = 0;
shallowOpts.dagFiles.rewind();
// Go through all DAG files specified on the command line...
StringList submitFiles;
const char *dagFile;
while ( (dagFile = shallowOpts.dagFiles.next()) ) {
// Get logical lines from this DAG file.
StringList logicalLines;
MyString error = MultiLogFiles::fileNameToLogicalLines(
dagFile, logicalLines );
if ( error != "" ) {
fprintf( stderr, "Error reading DAG file: %s\n",
error.Value() );
return 1;
}
// Find and parse JOB and SUBDAG lines.
logicalLines.rewind();
const char *dagLine;
while ( (dagLine = logicalLines.next()) ) {
StringList tokens( dagLine, " \t" );
tokens.rewind();
const char *first = tokens.next();
if ( first && !strcasecmp( first, "JOB" ) ) {
// Get the submit file and directory from the DAG
// file line.
const char *subFile;
const char *directory;
if ( parseJobOrDagLine( dagLine, tokens, "submit",
subFile, directory ) != 0 ) {
return 1;
}
// Now figure out whether JOB line is a nested DAG.
MyString submitFile( subFile );
// If submit file ends in ".condor.sub", we assume it
// refers to a sub-DAG.
int start = submitFile.find( DAG_SUBMIT_FILE_SUFFIX );
if ( start >= 0 &&
start + (int)strlen( DAG_SUBMIT_FILE_SUFFIX) ==
submitFile.Length() ) {
// Change submit file name to DAG file name.
submitFile.replaceString( DAG_SUBMIT_FILE_SUFFIX, "" );
// Now run condor_submit_dag on the DAG file.
if ( runSubmitDag( deepOpts, submitFile.Value(),
directory, false ) != 0 ) {
result = 1;
}
}
} else if ( first && !strcasecmp( first, "SUBDAG" ) ) {
const char *inlineOrExt = tokens.next();
if ( strcasecmp( inlineOrExt, "EXTERNAL" ) ) {
fprintf( stderr, "ERROR: only SUBDAG EXTERNAL is supported "
"at this time (line: <%s>)\n", dagLine );
return 1;
}
// Get the nested DAG file and directory from the DAG
// file line.
const char *nestedDagFile;
const char *directory;
if ( parseJobOrDagLine( dagLine, tokens, "DAG",
nestedDagFile, directory ) != 0 ) {
return 1;
}
// Now run condor_submit_dag on the DAG file.
if ( runSubmitDag( deepOpts, nestedDagFile, directory,
false ) != 0 ) {
result = 1;
}
}
}
}
return result;
}
static void generateSqlite3ForMember(sqlite3*db,MemberDef *md,Definition *def)
{
// + declaration/definition arg lists
// + reimplements
// + reimplementedBy
// + exceptions
// + const/volatile specifiers
// - examples
// + source definition
// + source references
// + source referenced by
// - body code
// + template arguments
// (templateArguments(), definitionTemplateParameterLists())
// - call graph
// enum values are written as part of the enum
if (md->memberType()==MemberType_EnumValue) return;
if (md->isHidden()) return;
//if (md->name().at(0)=='@') return; // anonymous member
// group members are only visible in their group
//if (def->definitionType()!=Definition::TypeGroup && md->getGroupDef()) return;
QCString memType;
#if 0
// member
idx = sqlite3_bind_parameter_index(stmt, ":refid");
sqlite3_bind_text(stmt, idx, memberOutputFileBase(md).data(),-1,SQLITE_TRANSIENT);
idx = sqlite3_bind_parameter_index(stmt,":kind");
sqlite3_bind_int(stmt, idx, md->memberType());
idx = sqlite3_bind_parameter_index(stmt, ":name");
sqlite3_bind_text(stmt, idx, md->name().data(),-1,SQLITE_TRANSIENT);
#endif
// memberdef
bindTextParameter(i_s_memberdef,":refid",md->anchor());
bindIntParameter(i_s_memberdef,":kind",md->memberType());
bindIntParameter(i_s_memberdef,":prot",md->protection());
bindIntParameter(i_s_memberdef,":static",md->isStatic());
bool isFunc=FALSE;
switch (md->memberType())
{
case MemberType_Function: // fall through
case MemberType_Signal: // fall through
case MemberType_Friend: // fall through
case MemberType_DCOP: // fall through
case MemberType_Slot:
isFunc=TRUE;
break;
default:
break;
}
if (isFunc)
{
LockingPtr<ArgumentList> al = md->argumentList();
if (al!=0 && al->constSpecifier)
{
bindIntParameter(i_s_memberdef,":const",al->constSpecifier);
}
bindIntParameter(i_s_memberdef,":explicit",md->isExplicit());
bindIntParameter(i_s_memberdef,":inline",md->isInline());
bindIntParameter(i_s_memberdef,":final",md->isFinal());
bindIntParameter(i_s_memberdef,":sealed",md->isSealed());
bindIntParameter(i_s_memberdef,":new",md->isNew());
bindIntParameter(i_s_memberdef,":optional",md->isOptional());
bindIntParameter(i_s_memberdef,":required",md->isRequired());
bindIntParameter(i_s_memberdef,":virt",md->virtualness());
}
// place in the arguments and linkify the arguments
if (md->memberType() == MemberType_Variable)
{
bindIntParameter(i_s_memberdef,":mutable",md->isMutable());
bindIntParameter(i_s_memberdef,":initonly",md->isInitonly());
}
else if (md->memberType() == MemberType_Property)
{
bindIntParameter(i_s_memberdef,":readable",md->isReadable());
bindIntParameter(i_s_memberdef,":writable",md->isWritable());
bindIntParameter(i_s_memberdef,":gettable",md->isGettable());
bindIntParameter(i_s_memberdef,":settable",md->isSettable());
if (md->isAssign() || md->isCopy() || md->isRetain())
{
int accessor = md->isAssign() ? md->isAssign() :
(md->isCopy() ? md->isCopy() : md->isRetain()) ;
bindIntParameter(i_s_memberdef,":accessor",accessor);
}
}
else if (md->memberType() == MemberType_Event)
{
bindIntParameter(i_s_memberdef,":addable",md->isAddable());
bindIntParameter(i_s_memberdef,":removable",md->isRemovable());
bindIntParameter(i_s_memberdef,":raisable",md->isRaisable());
}
if (md->memberType()!=MemberType_Define &&
md->memberType()!=MemberType_Enumeration
)
{
QCString typeStr = md->typeString();
stripQualifiers(typeStr);
StringList l;
linkifyText(TextGeneratorSqlite3Impl(l),def,md->getBodyDef(),md,typeStr);
if (typeStr.data())
{
bindTextParameter(i_s_memberdef,":type",typeStr);
}
if (md->definition())
{
bindTextParameter(i_s_memberdef,":definition",md->definition());
}
if (md->argsString())
{
bindTextParameter(i_s_memberdef,":argsstring",md->argsString());
}
}
bindTextParameter(i_s_memberdef,":name",md->name());
if (md->memberType() == MemberType_Property)
{
if (md->isReadable())
{
DBG_CTX(("<read>\n"));
}
if (md->isWritable())
{
DBG_CTX(("<write>\n"));
}
}
#if 0
if (md->memberType()==MemberType_Variable && md->bitfieldString())
{
QCString bitfield = md->bitfieldString();
if (bitfield.at(0)==':') bitfield=bitfield.mid(1);
t << " <bitfield>" << bitfield << "</bitfield>" << endl;
}
MemberDef *rmd = md->reimplements();
if (rmd)
{
t << " <reimplements refid=\""
<< memberOutputFileBase(rmd) << "_1" << rmd->anchor() << "\">"
<< convertToXML(rmd->name()) << "</reimplements>" << endl;
}
LockingPtr<MemberList> rbml = md->reimplementedBy();
if (rbml!=0)
{
MemberListIterator mli(*rbml);
for (mli.toFirst();(rmd=mli.current());++mli)
{
t << " <reimplementedby refid=\""
<< memberOutputFileBase(rmd) << "_1" << rmd->anchor() << "\">"
<< convertToXML(rmd->name()) << "</reimplementedby>" << endl;
}
}
#endif
if (isFunc) //function
{
LockingPtr<ArgumentList> declAl = md->declArgumentList();
LockingPtr<ArgumentList> defAl = md->argumentList();
if (declAl!=0 && declAl->count()>0)
{
ArgumentListIterator declAli(*declAl);
ArgumentListIterator defAli(*defAl);
Argument *a;
for (declAli.toFirst();(a=declAli.current());++declAli)
{
Argument *defArg = defAli.current();
DBG_CTX(("<param>\n"));
if (!a->attrib.isEmpty())
{
DBG_CTX(("<attributes>:%s\n",a->attrib.data()));
}
if (!a->type.isEmpty())
{
StringList l;
linkifyText(TextGeneratorSqlite3Impl(l),def,md->getBodyDef(),md,a->type);
QCString *s=l.first();
while (s)
{
insertMemberReference(db,md->anchor().data(),s->data(),def->getDefFileName().data(),md->getDefLine(),1);
s=l.next();
}
}
if (!a->name.isEmpty())
{
DBG_CTX(("<declname>%s\n",a->name.data()));
}
if (defArg && !defArg->name.isEmpty() && defArg->name!=a->name)
{
DBG_CTX(("<defname>%s\n",defArg->name.data()));
}
if (!a->array.isEmpty())
{
DBG_CTX(("<array>%s",a->array.data()));
}
if (!a->defval.isEmpty())
{
StringList l;
linkifyText(TextGeneratorSqlite3Impl(l),def,md->getBodyDef(),md,a->defval);
}
if (defArg) ++defAli;
}
}
}
else if (md->memberType()==MemberType_Define &&
md->argsString()) // define
{
if (md->argumentList()->count()==0) // special case for "foo()" to
// disguish it from "foo".
{
DBG_CTX(("no params\n"));
}
else
{
ArgumentListIterator ali(*md->argumentList());
Argument *a;
for (ali.toFirst();(a=ali.current());++ali)
{
DBG_CTX(("<param><defname>%s\n",a->type.data()));
}
}
}
// Extract references from initializer
// avoid that extremely large tables are written to the output.
// todo: it's better to adhere to MAX_INITIALIZER_LINES.
// drm_mod_register_buffer,
if (!md->initializer().isEmpty() && md->initializer().length()<2000)
{
StringList l;
linkifyText(TextGeneratorSqlite3Impl(l),def,md->getBodyDef(),md,md->initializer());
QCString *s=l.first();
while (s)
{
DBG_CTX(("initializer:%s %s %s %d\n",
md->anchor().data(),
s->data(),
md->getBodyDef()->getDefFileName().data(),
md->getStartBodyLine()));
insertMemberReference(db,md->anchor().data(),s->data(),md->getBodyDef()->getDefFileName().data(),md->getStartBodyLine(),1);
s=l.next();
}
}
#if 0
if (md->excpString())
{
linkifyText(TextGeneratorXMLImpl(t),def,md->getBodyDef(),md,md->excpString());
}
#endif
if ( md->getScopeString() )
{
bindTextParameter(i_s_memberdef,":scope",md->getScopeString());
}
// File location
if (md->getDefLine() != -1)
{
int id_file = insertFile(db,md->getDefFileName());
if (id_file!=-1)
{
bindIntParameter(i_s_memberdef,":id_file",id_file);
bindIntParameter(i_s_memberdef,":line",md->getDefLine());
bindIntParameter(i_s_memberdef,":column",md->getDefColumn());
if (md->getStartBodyLine()!=-1)
{
int id_bfile = insertFile(db,md->getBodyDef()->absFilePath());
if (id_bfile == -1) exit(-1);
bindIntParameter(i_s_memberdef,":id_ibfile",id_bfile);
bindIntParameter(i_s_memberdef,":bline",md->getStartBodyLine());
// XXX implement getStartBodyColumn
bindIntParameter(i_s_memberdef,":bcolumn",1);
}
}
}
step(db,i_s_memberdef);
/*int id_src =*/ sqlite3_last_insert_rowid(db);
// + cross-references
// The cross-references in initializers only work when both the src and dst
// are defined.
LockingPtr<MemberSDict> mdict = md->getReferencesMembers();
// references
if (mdict!=0)
{
MemberSDict::IteratorDict mdi(*mdict);
MemberDef *rmd;
for (mdi.toFirst();(rmd=mdi.current());++mdi)
{
insertMemberReference(db,md,rmd,mdi.currentKey());
}
}
mdict = md->getReferencedByMembers();
// referencedby
if (mdict!=0)
{
MemberSDict::IteratorDict mdi(*mdict);
MemberDef *rmd;
for (mdi.toFirst();(rmd=mdi.current());++mdi)
{
insertMemberReference(db,rmd,md,mdi.currentKey());
}
}
}
int
main (int argc, char *argv[])
{
#if !defined(WIN32)
install_sig_handler(SIGPIPE, (SIG_HANDLER)SIG_IGN );
#endif
// initialize to read from config file
myDistro->Init( argc, argv );
myName = argv[0];
config();
dprintf_config_tool_on_error(0);
// The arguments take two passes to process --- the first pass
// figures out the mode, after which we can instantiate the required
// query object. We add implied constraints from the command line in
// the second pass.
firstPass (argc, argv);
// if the mode has not been set, it is STARTD_NORMAL
if (mode == MODE_NOTSET) {
setMode (MODE_STARTD_NORMAL, 0, DEFAULT);
}
// instantiate query object
if (!(query = new CondorQuery (type))) {
dprintf_WriteOnErrorBuffer(stderr, true);
fprintf (stderr, "Error: Out of memory\n");
exit (1);
}
// if a first-pass setMode set a mode_constraint, apply it now to the query object
if (mode_constraint && ! explicit_format) {
query->addANDConstraint(mode_constraint);
}
// set pretty print style implied by the type of entity being queried
// but do it with default priority, so that explicitly requested options
// can override it
switch (type)
{
#ifdef HAVE_EXT_POSTGRESQL
case QUILL_AD:
setPPstyle(PP_QUILL_NORMAL, 0, DEFAULT);
break;
#endif /* HAVE_EXT_POSTGRESQL */
case DEFRAG_AD:
setPPstyle(PP_GENERIC_NORMAL, 0, DEFAULT);
break;
case STARTD_AD:
setPPstyle(PP_STARTD_NORMAL, 0, DEFAULT);
break;
case SCHEDD_AD:
setPPstyle(PP_SCHEDD_NORMAL, 0, DEFAULT);
break;
case MASTER_AD:
setPPstyle(PP_MASTER_NORMAL, 0, DEFAULT);
break;
case CKPT_SRVR_AD:
setPPstyle(PP_CKPT_SRVR_NORMAL, 0, DEFAULT);
break;
case COLLECTOR_AD:
setPPstyle(PP_COLLECTOR_NORMAL, 0, DEFAULT);
break;
case STORAGE_AD:
setPPstyle(PP_STORAGE_NORMAL, 0, DEFAULT);
break;
case NEGOTIATOR_AD:
setPPstyle(PP_NEGOTIATOR_NORMAL, 0, DEFAULT);
break;
case GRID_AD:
setPPstyle(PP_GRID_NORMAL, 0, DEFAULT);
break;
case GENERIC_AD:
setPPstyle(PP_GENERIC, 0, DEFAULT);
break;
case ANY_AD:
setPPstyle(PP_ANY_NORMAL, 0, DEFAULT);
break;
default:
setPPstyle(PP_VERBOSE, 0, DEFAULT);
}
// set the constraints implied by the mode
switch (mode) {
#ifdef HAVE_EXT_POSTGRESQL
case MODE_QUILL_NORMAL:
#endif /* HAVE_EXT_POSTGRESQL */
case MODE_DEFRAG_NORMAL:
case MODE_STARTD_NORMAL:
case MODE_MASTER_NORMAL:
case MODE_CKPT_SRVR_NORMAL:
case MODE_SCHEDD_NORMAL:
case MODE_SCHEDD_SUBMITTORS:
case MODE_COLLECTOR_NORMAL:
case MODE_NEGOTIATOR_NORMAL:
case MODE_STORAGE_NORMAL:
case MODE_GENERIC_NORMAL:
case MODE_ANY_NORMAL:
case MODE_GRID_NORMAL:
case MODE_HAD_NORMAL:
break;
case MODE_OTHER:
// tell the query object what the type we're querying is
query->setGenericQueryType(genericType);
free(genericType);
genericType = NULL;
break;
case MODE_STARTD_AVAIL:
// For now, -avail shows you machines avail to anyone.
sprintf (buffer, "%s == \"%s\"", ATTR_STATE,
state_to_string(unclaimed_state));
if (diagnose) {
printf ("Adding constraint [%s]\n", buffer);
}
query->addORConstraint (buffer);
break;
case MODE_STARTD_RUN:
sprintf (buffer, "%s == \"%s\"", ATTR_STATE,
state_to_string(claimed_state));
if (diagnose) {
printf ("Adding constraint [%s]\n", buffer);
}
query->addORConstraint (buffer);
break;
case MODE_STARTD_COD:
sprintf (buffer, "%s > 0", ATTR_NUM_COD_CLAIMS );
if (diagnose) {
printf ("Adding constraint [%s]\n", buffer);
}
query->addORConstraint (buffer);
break;
default:
break;
}
if(javaMode) {
sprintf( buffer, "%s == TRUE", ATTR_HAS_JAVA );
if (diagnose) {
printf ("Adding constraint [%s]\n", buffer);
}
query->addANDConstraint (buffer);
projList.AppendArg(ATTR_HAS_JAVA);
projList.AppendArg(ATTR_JAVA_MFLOPS);
projList.AppendArg(ATTR_JAVA_VENDOR);
projList.AppendArg(ATTR_JAVA_VERSION);
}
if(offlineMode) {
query->addANDConstraint( "size( OfflineUniverses ) != 0" );
projList.AppendArg( "OfflineUniverses" );
//
// Since we can't add a regex to a projection, explicitly list all
// the attributes we know about.
//
projList.AppendArg( "HasVM" );
projList.AppendArg( "VMOfflineReason" );
projList.AppendArg( "VMOfflineTime" );
}
if(absentMode) {
sprintf( buffer, "%s == TRUE", ATTR_ABSENT );
if (diagnose) {
printf( "Adding constraint %s\n", buffer );
}
query->addANDConstraint( buffer );
projList.AppendArg( ATTR_ABSENT );
projList.AppendArg( ATTR_LAST_HEARD_FROM );
projList.AppendArg( ATTR_CLASSAD_LIFETIME );
}
if(vmMode) {
sprintf( buffer, "%s == TRUE", ATTR_HAS_VM);
if (diagnose) {
printf ("Adding constraint [%s]\n", buffer);
}
query->addANDConstraint (buffer);
projList.AppendArg(ATTR_VM_TYPE);
projList.AppendArg(ATTR_VM_MEMORY);
projList.AppendArg(ATTR_VM_NETWORKING);
projList.AppendArg(ATTR_VM_NETWORKING_TYPES);
projList.AppendArg(ATTR_VM_HARDWARE_VT);
projList.AppendArg(ATTR_VM_AVAIL_NUM);
projList.AppendArg(ATTR_VM_ALL_GUEST_MACS);
projList.AppendArg(ATTR_VM_ALL_GUEST_IPS);
projList.AppendArg(ATTR_VM_GUEST_MAC);
projList.AppendArg(ATTR_VM_GUEST_IP);
}
// second pass: add regular parameters and constraints
if (diagnose) {
printf ("----------\n");
}
secondPass (argc, argv);
// initialize the totals object
if (ppStyle == PP_CUSTOM && using_print_format) {
if (pmHeadFoot & HF_NOSUMMARY) ppTotalStyle = PP_CUSTOM;
} else {
ppTotalStyle = ppStyle;
}
TrackTotals totals(ppTotalStyle);
// fetch the query
QueryResult q;
if ((mode == MODE_STARTD_NORMAL) && (ppStyle == PP_STARTD_NORMAL)) {
projList.AppendArg("Name");
projList.AppendArg("Machine");
projList.AppendArg("Opsys");
projList.AppendArg("Arch");
projList.AppendArg("State");
projList.AppendArg("Activity");
projList.AppendArg("LoadAvg");
projList.AppendArg("Memory");
projList.AppendArg("ActvtyTime");
projList.AppendArg("MyCurrentTime");
projList.AppendArg("EnteredCurrentActivity");
} else if( ppStyle == PP_VERBOSE ) {
// Remove everything from the projection list if we're displaying
// the "long form" of the ads.
projList.Clear();
// but if -attributes was supplied, show only those attributes
if ( ! dashAttributes.isEmpty()) {
const char * s;
dashAttributes.rewind();
while ((s = dashAttributes.next())) {
projList.AppendArg(s);
}
}
}
if( projList.Count() > 0 ) {
char **attr_list = projList.GetStringArray();
query->setDesiredAttrs(attr_list);
deleteStringArray(attr_list);
}
// if diagnose was requested, just print the query ad
if (diagnose) {
ClassAd queryAd;
// print diagnostic information about inferred internal state
setMode ((Mode) 0, 0, NULL);
setType (NULL, 0, NULL);
setPPstyle ((ppOption) 0, 0, DEFAULT);
printf ("----------\n");
q = query->getQueryAd (queryAd);
fPrintAd (stdout, queryAd);
printf ("----------\n");
fprintf (stderr, "Result of making query ad was: %d\n", q);
exit (1);
}
// Address (host:port) is taken from requested pool, if given.
char* addr = (NULL != pool) ? pool->addr() : NULL;
Daemon* requested_daemon = pool;
// If we're in "direct" mode, then we attempt to locate the daemon
// associated with the requested subsystem (here encoded by value of mode)
// In this case the host:port of pool (if given) denotes which
// pool is being consulted
if( direct ) {
Daemon *d = NULL;
switch( mode ) {
case MODE_MASTER_NORMAL:
d = new Daemon( DT_MASTER, direct, addr );
break;
case MODE_STARTD_NORMAL:
case MODE_STARTD_AVAIL:
case MODE_STARTD_RUN:
case MODE_STARTD_COD:
d = new Daemon( DT_STARTD, direct, addr );
break;
#ifdef HAVE_EXT_POSTGRESQL
case MODE_QUILL_NORMAL:
d = new Daemon( DT_QUILL, direct, addr );
break;
#endif /* HAVE_EXT_POSTGRESQL */
case MODE_SCHEDD_NORMAL:
case MODE_SCHEDD_SUBMITTORS:
d = new Daemon( DT_SCHEDD, direct, addr );
break;
case MODE_NEGOTIATOR_NORMAL:
d = new Daemon( DT_NEGOTIATOR, direct, addr );
break;
case MODE_CKPT_SRVR_NORMAL:
case MODE_COLLECTOR_NORMAL:
case MODE_LICENSE_NORMAL:
case MODE_STORAGE_NORMAL:
case MODE_GENERIC_NORMAL:
case MODE_ANY_NORMAL:
case MODE_OTHER:
case MODE_GRID_NORMAL:
case MODE_HAD_NORMAL:
// These have to go to the collector, anyway.
break;
default:
fprintf( stderr, "Error: Illegal mode %d\n", mode );
exit( 1 );
break;
}
// Here is where we actually override 'addr', if we can obtain
// address of the requested daemon/subsys. If it can't be
// located, then fail with error msg.
// 'd' will be null (unset) if mode is one of above that must go to
// collector (MODE_ANY_NORMAL, MODE_COLLECTOR_NORMAL, etc)
if (NULL != d) {
if( d->locate() ) {
addr = d->addr();
requested_daemon = d;
} else {
const char* id = d->idStr();
if (NULL == id) id = d->name();
dprintf_WriteOnErrorBuffer(stderr, true);
if (NULL == id) id = "daemon";
fprintf(stderr, "Error: Failed to locate %s\n", id);
fprintf(stderr, "%s\n", d->error());
exit( 1 );
}
}
}
ClassAdList result;
CondorError errstack;
if (NULL != ads_file) {
MyString req; // query requirements
q = query->getRequirements(req);
const char * constraint = req.empty() ? NULL : req.c_str();
if (read_classad_file(ads_file, result, constraint)) {
q = Q_OK;
}
} else if (NULL != addr) {
// this case executes if pool was provided, or if in "direct" mode with
// subsystem that corresponds to a daemon (above).
// Here 'addr' represents either the host:port of requested pool, or
// alternatively the host:port of daemon associated with requested subsystem (direct mode)
q = query->fetchAds (result, addr, &errstack);
} else {
// otherwise obtain list of collectors and submit query that way
CollectorList * collectors = CollectorList::create();
q = collectors->query (*query, result, &errstack);
delete collectors;
}
// if any error was encountered during the query, report it and exit
if (Q_OK != q) {
dprintf_WriteOnErrorBuffer(stderr, true);
// we can always provide these messages:
fprintf( stderr, "Error: %s\n", getStrQueryResult(q) );
fprintf( stderr, "%s\n", errstack.getFullText(true).c_str() );
if ((NULL != requested_daemon) && ((Q_NO_COLLECTOR_HOST == q) ||
(requested_daemon->type() == DT_COLLECTOR)))
{
// Specific long message if connection to collector failed.
const char* fullhost = requested_daemon->fullHostname();
if (NULL == fullhost) fullhost = "<unknown_host>";
const char* daddr = requested_daemon->addr();
if (NULL == daddr) daddr = "<unknown>";
char info[1000];
sprintf(info, "%s (%s)", fullhost, daddr);
printNoCollectorContact( stderr, info, !expert );
} else if ((NULL != requested_daemon) && (Q_COMMUNICATION_ERROR == q)) {
// more helpful message for failure to connect to some daemon/subsys
const char* id = requested_daemon->idStr();
if (NULL == id) id = requested_daemon->name();
if (NULL == id) id = "daemon";
const char* daddr = requested_daemon->addr();
if (NULL == daddr) daddr = "<unknown>";
fprintf(stderr, "Error: Failed to contact %s at %s\n", id, daddr);
}
// fail
exit (1);
}
if (noSort) {
// do nothing
} else if (sortSpecs.empty()) {
// default classad sorting
result.Sort((SortFunctionType)lessThanFunc);
} else {
// User requested custom sorting expressions:
// insert attributes related to custom sorting
result.Open();
while (ClassAd* ad = result.Next()) {
for (vector<SortSpec>::iterator ss(sortSpecs.begin()); ss != sortSpecs.end(); ++ss) {
ss->expr->SetParentScope(ad);
classad::Value v;
ss->expr->Evaluate(v);
stringstream vs;
// This will properly render all supported value types,
// including undefined and error, although current semantic
// pre-filters classads where sort expressions are undef/err:
vs << ((v.IsStringValue())?"\"":"") << v << ((v.IsStringValue())?"\"":"");
ad->AssignExpr(ss->keyAttr.c_str(), vs.str().c_str());
// Save the full expr in case user wants to examine on output:
ad->AssignExpr(ss->keyExprAttr.c_str(), ss->arg.c_str());
}
}
result.Open();
result.Sort((SortFunctionType)customLessThanFunc);
}
// output result
prettyPrint (result, &totals);
delete query;
return 0;
}
void
secondPass (int argc, char *argv[])
{
const char * pcolon = NULL;
char *daemonname;
for (int i = 1; i < argc; i++) {
// omit parameters which qualify switches
if( matchPrefix(argv[i],"-pool", 2) || matchPrefix(argv[i],"-direct", 4) ) {
i++;
continue;
}
if( matchPrefix(argv[i],"-subsystem", 5) ) {
i++;
continue;
}
if (matchPrefix (argv[i], "-format", 2)) {
pm.registerFormat (argv[i+1], argv[i+2]);
StringList attributes;
ClassAd ad;
if(!ad.GetExprReferences(argv[i+2],NULL,&attributes)){
fprintf( stderr, "Error: Parse error of: %s\n", argv[i+2]);
exit(1);
}
attributes.rewind();
char const *s;
while( (s=attributes.next()) ) {
projList.AppendArg(s);
}
if (diagnose) {
printf ("Arg %d --- register format [%s] for [%s]\n",
i, argv[i+1], argv[i+2]);
}
i += 2;
continue;
}
if (*argv[i] == '-' &&
(is_arg_colon_prefix(argv[i]+1, "autoformat", &pcolon, 5) ||
is_arg_colon_prefix(argv[i]+1, "af", &pcolon, 2)) ) {
// make sure we have at least one more argument
if ( !argv[i+1] || *(argv[i+1]) == '-') {
fprintf( stderr, "Error: Argument %s requires "
"at last one attribute parameter\n", argv[i] );
fprintf( stderr, "Use \"%s -help\" for details\n", myName );
exit( 1 );
}
bool flabel = false;
bool fCapV = false;
bool fRaw = false;
bool fheadings = false;
const char * prowpre = NULL;
const char * pcolpre = " ";
const char * pcolsux = NULL;
if (pcolon) {
++pcolon;
while (*pcolon) {
switch (*pcolon)
{
case ',': pcolsux = ","; break;
case 'n': pcolsux = "\n"; break;
case 'g': pcolpre = NULL; prowpre = "\n"; break;
case 't': pcolpre = "\t"; break;
case 'l': flabel = true; break;
case 'V': fCapV = true; break;
case 'r': case 'o': fRaw = true; break;
case 'h': fheadings = true; break;
}
++pcolon;
}
}
pm.SetAutoSep(prowpre, pcolpre, pcolsux, "\n");
while (argv[i+1] && *(argv[i+1]) != '-') {
++i;
ClassAd ad;
StringList attributes;
if(!ad.GetExprReferences(argv[i],NULL,&attributes)){
fprintf( stderr, "Error: Parse error of: %s\n", argv[i]);
exit(1);
}
attributes.rewind();
char const *s;
while ((s = attributes.next())) {
projList.AppendArg(s);
}
MyString lbl = "";
int wid = 0;
int opts = FormatOptionNoTruncate;
if (fheadings || pm_head.Length() > 0) {
const char * hd = fheadings ? argv[i] : "(expr)";
wid = 0 - (int)strlen(hd);
opts = FormatOptionAutoWidth | FormatOptionNoTruncate;
pm_head.Append(hd);
}
else if (flabel) { lbl.formatstr("%s = ", argv[i]); wid = 0; opts = 0; }
lbl += fRaw ? "%r" : (fCapV ? "%V" : "%v");
if (diagnose) {
printf ("Arg %d --- register format [%s] width=%d, opt=0x%x for [%s]\n",
i, lbl.Value(), wid, opts, argv[i]);
}
pm.registerFormat(lbl.Value(), wid, opts, argv[i]);
}
// if autoformat list ends in a '-' without any characters after it, just eat the arg and keep going.
if (i+1 < argc && '-' == (argv[i+1])[0] && 0 == (argv[i+1])[1]) {
++i;
}
continue;
}
if (is_dash_arg_colon_prefix(argv[i], "print-format", &pcolon, 2)) {
if ( (i+1 >= argc) || (*(argv[i+1]) == '-' && (argv[i+1])[1] != 0)) {
fprintf( stderr, "Error: Argument -print-format requires a filename argument\n");
exit( 1 );
}
// hack allow -pr ! to disable use of user-default print format files.
if (MATCH == strcmp(argv[i+1], "!")) {
++i;
disable_user_print_files = true;
continue;
}
ppTotalStyle = ppStyle;
setPPstyle (PP_CUSTOM, i, argv[i]);
setPPwidth();
++i; // skip to the next argument.
if (set_status_print_mask_from_stream(argv[i], true, &mode_constraint) < 0) {
fprintf(stderr, "Error: invalid select file %s\n", argv[i]);
exit (1);
}
if (mode_constraint) {
query->addANDConstraint(mode_constraint);
}
using_print_format = true; // so we can hack totals.
continue;
}
if (matchPrefix (argv[i], "-target", 5)) {
i++;
continue;
}
if (is_dash_arg_prefix(argv[i], "ads", 2)) {
++i;
continue;
}
if( matchPrefix(argv[i], "-sort", 3) ) {
i++;
if ( ! noSort) {
sprintf( buffer, "%s =!= UNDEFINED", argv[i] );
query->addANDConstraint( buffer );
}
continue;
}
if (matchPrefix (argv[i], "-statistics", 6)) {
i += 2;
sprintf(buffer,"STATISTICS_TO_PUBLISH = \"%s\"", statistics);
if (diagnose) {
printf ("[%s]\n", buffer);
}
query->addExtraAttribute(buffer);
continue;
}
if (matchPrefix (argv[i], "-attributes", 3) ) {
// parse attributes to be selected and split them along ","
StringList more_attrs(argv[i+1],",");
char const *s;
more_attrs.rewind();
while( (s=more_attrs.next()) ) {
projList.AppendArg(s);
dashAttributes.append(s);
}
i++;
continue;
}
// figure out what the other parameters should do
if (*argv[i] != '-') {
// display extra information for diagnosis
if (diagnose) {
printf ("Arg %d (%s) --- adding constraint", i, argv[i]);
}
if( !(daemonname = get_daemon_name(argv[i])) ) {
if ( (mode==MODE_SCHEDD_SUBMITTORS) && strchr(argv[i],'@') ) {
// For a submittor query, it is possible that the
// hostname is really a UID_DOMAIN. And there is
// no requirement that UID_DOMAIN actually have
// an inverse lookup in DNS... so if get_daemon_name()
// fails with a fully qualified submittor lookup, just
// use what we are given and do not flag an error.
daemonname = strnewp(argv[i]);
} else {
dprintf_WriteOnErrorBuffer(stderr, true);
fprintf( stderr, "%s: unknown host %s\n",
argv[0], get_host_part(argv[i]) );
exit(1);
}
}
switch (mode) {
case MODE_DEFRAG_NORMAL:
case MODE_STARTD_NORMAL:
case MODE_STARTD_COD:
#ifdef HAVE_EXT_POSTGRESQL
case MODE_QUILL_NORMAL:
#endif /* HAVE_EXT_POSTGRESQL */
case MODE_SCHEDD_NORMAL:
case MODE_SCHEDD_SUBMITTORS:
case MODE_MASTER_NORMAL:
case MODE_COLLECTOR_NORMAL:
case MODE_CKPT_SRVR_NORMAL:
case MODE_NEGOTIATOR_NORMAL:
case MODE_STORAGE_NORMAL:
case MODE_ANY_NORMAL:
case MODE_GENERIC_NORMAL:
case MODE_STARTD_AVAIL:
case MODE_OTHER:
case MODE_GRID_NORMAL:
case MODE_HAD_NORMAL:
sprintf(buffer,"(%s==\"%s\") || (%s==\"%s\")",
ATTR_NAME, daemonname, ATTR_MACHINE, daemonname );
if (diagnose) {
printf ("[%s]\n", buffer);
}
query->addORConstraint (buffer);
break;
case MODE_STARTD_RUN:
sprintf (buffer,"%s == \"%s\"",ATTR_REMOTE_USER,argv[i]);
if (diagnose) {
printf ("[%s]\n", buffer);
}
query->addORConstraint (buffer);
break;
default:
fprintf(stderr,"Error: Don't know how to process %s\n",argv[i]);
}
delete [] daemonname;
daemonname = NULL;
} else
if (matchPrefix (argv[i], "-constraint", 4)) {
if (diagnose) {
printf ("[%s]\n", argv[i+1]);
}
query->addANDConstraint (argv[i+1]);
i++;
}
}
}
///////////////////////////////////////////////////////////////////////////////
// Note: this method should get speeded up (see Gnats PR 846).
MyString
MultiLogFiles::loadLogFileNameFromSubFile(const MyString &strSubFilename,
const MyString &directory, bool &isXml, bool usingDefaultNode)
{
dprintf( D_FULLDEBUG, "MultiLogFiles::loadLogFileNameFromSubFile(%s, %s)\n",
strSubFilename.Value(), directory.Value() );
TmpDir td;
if ( directory != "" ) {
MyString errMsg;
if ( !td.Cd2TmpDir(directory.Value(), errMsg) ) {
dprintf(D_ALWAYS, "Error from Cd2TmpDir: %s\n", errMsg.Value());
return "";
}
}
StringList logicalLines;
if ( fileNameToLogicalLines( strSubFilename, logicalLines ) != "" ) {
return "";
}
MyString logFileName("");
MyString initialDir("");
MyString isXmlLogStr("");
// Now look through the submit file logical lines to find the
// log file and initial directory (if specified) and combine
// them into a path to the log file that's either absolute or
// relative to the DAG submit directory. Also look for log_xml.
const char *logicalLine;
while( (logicalLine = logicalLines.next()) != NULL ) {
MyString submitLine(logicalLine);
MyString tmpLogName = getParamFromSubmitLine(submitLine, "log");
if ( tmpLogName != "" ) {
logFileName = tmpLogName;
}
// If we are using the default node log, we don't care
// about these
if( !usingDefaultNode ) {
MyString tmpInitialDir = getParamFromSubmitLine(submitLine,
"initialdir");
if ( tmpInitialDir != "" ) {
initialDir = tmpInitialDir;
}
MyString tmpLogXml = getParamFromSubmitLine(submitLine, "log_xml");
if ( tmpLogXml != "" ) {
isXmlLogStr = tmpLogXml;
}
}
}
if ( !usingDefaultNode ) {
//
// Check for macros in the log file name -- we currently don't
// handle those.
//
// If we are using the default node, we don't need to check this
if ( logFileName != "" ) {
if ( strstr(logFileName.Value(), "$(") ) {
dprintf(D_ALWAYS, "MultiLogFiles: macros ('$(...') not allowed "
"in log file name (%s) in DAG node submit files\n",
logFileName.Value());
logFileName = "";
}
}
// Do not need to prepend initialdir if we are using the
// default node log
if ( logFileName != "" ) {
// Prepend initialdir to log file name if log file name is not
// an absolute path.
if ( initialDir != "" && !fullpath(logFileName.Value()) ) {
logFileName = initialDir + DIR_DELIM_STRING + logFileName;
}
// We do this in case the same log file is specified with a
// relative and an absolute path.
// Note: we now do further checking that doesn't rely on
// comparing paths to the log files. wenger 2004-05-27.
CondorError errstack;
if ( !makePathAbsolute( logFileName, errstack ) ) {
dprintf(D_ALWAYS, "%s\n", errstack.getFullText().c_str());
return "";
}
}
isXmlLogStr.lower_case();
isXml = (isXmlLogStr == "true");
if ( directory != "" ) {
MyString errMsg;
if ( !td.Cd2MainDir(errMsg) ) {
dprintf(D_ALWAYS, "Error from Cd2MainDir: %s\n", errMsg.Value());
return "";
}
}
}
return logFileName;
}
// parse a configuration string in the form "ALL:opt, CAT:opt, ALT:opt"
// where opt can be one or more of
// 0-3 verbosity level, 0 is least and 3 is most. default is usually 1
// NONE disable all
// ALL enable all
// R enable Recent (default)
// !R disable Recent
// D enable Debug
// !D disable Debug (default)
// Z don't publish values markerd IF_NONZERO when their value is 0
// !Z ignore IF_NONZERO publishing flag
//
// return value is the PublishFlags that should be passed in to StatisticsPool::Publish
// for this category.
//
int generic_stats_ParseConfigString(
const char * config, // name of the string parameter to read from the config file
const char * pool_name, // name of the stats pool/category of stats to look for
const char * pool_alt, // alternate name of the category to look for
int flags_def) // default value for publish flags for this pool
{
// special case, if there is no string, or the string is just "default", then
// return the default flags
if ( ! config || MATCH == strcasecmp(config,"DEFAULT"))
return flags_def;
// special case, if the string is empty, or the string is just "none", then
// return 0 (disable all)
if ( ! config[0] || MATCH == strcasecmp(config,"NONE"))
return 0;
// tokenize the list on , or space
StringList items;
items.initializeFromString(config);
// if the config string is non-trivial, then it must contain either our pool_name
// or pool_alt or "DEFAULT" or "ALL" or we do not publish this pool.
int PublishFlags = 0;
// walk the list, looking for items that match our pool name or the keyword DEFAULT or ALL
//
items.rewind();
while (const char * p = items.next()) {
int flags = PublishFlags;
const char * psep = strchr(p,':');
if (psep) {
size_t cch = psep - p;
char sz[64];
if (cch >= COUNTOF(sz))
continue;
strncpy(sz, p, cch);
sz[cch] = 0;
if (strcasecmp(sz,pool_name) && strcasecmp(sz,pool_alt) && strcasecmp(sz,"DEFAULT") && strcasecmp(sz,"ALL"))
continue;
} else {
if (strcasecmp(p,pool_name) && strcasecmp(p,pool_alt) && strcasecmp(p,"DEFAULT") && strcasecmp(p,"ALL"))
continue;
}
// if we get to here, we found our pool name or "DEFAULT" or "ALL"
// so we begin with our default flags
flags = flags_def;
// if there are any options, then parse them and modify the flags
if (psep) {
const char * popt = psep+1;
if (MATCH == strcasecmp(popt,"NONE")) {
flags = 0;
} else {
bool bang = false;
const char * parse_error = NULL;
while (popt[0]) {
char ch = popt[0];
if (ch >= '0' && ch <= '3') {
int level = (atoi(popt) * IF_BASICPUB) & IF_PUBLEVEL;
flags = (flags & ~IF_PUBLEVEL) | level;
} else if (ch == '!') {
bang = true;
} else if (ch == 'd' || ch == 'D') {
flags = bang ? (flags & ~IF_DEBUGPUB) : (flags | IF_DEBUGPUB);
} else if (ch == 'r' || ch == 'R') {
flags = bang ? (flags & ~IF_RECENTPUB) : (flags | IF_RECENTPUB);
} else if (ch == 'z' || ch == 'Z') {
flags = bang ? (flags & ~IF_NONZERO) : (flags | IF_NONZERO);
} else if (ch == 'l' || ch == 'L') {
flags = bang ? (flags | IF_NOLIFETIME) : (flags & ~IF_NOLIFETIME);
} else {
if ( ! parse_error) parse_error = popt;
}
++popt;
}
if (parse_error) {
dprintf(D_ALWAYS, "Option '%s' invalid in '%s' when parsing statistics to publish. effect is %08X\n",
parse_error, p, flags);
}
}
}
PublishFlags = flags;
dprintf(D_FULLDEBUG, "'%s' gives flags %08X for %s statistics\n", p, PublishFlags, pool_name);
}
return PublishFlags;
}
int main(int argc, char **argv)
{
int result = 0;
if ( argc <= 1 || (argc >= 2 && !strcmp("-usage", argv[1])) ) {
printf("Usage: condor_check_userlogs <log file 1> "
"[log file 2] ... [log file n]\n");
exit(0);
}
// Set up dprintf.
dprintf_set_tool_debug("condor_check_userlogs", 0);
set_debug_flags(NULL, D_ALWAYS);
StringList logFiles;
for ( int argnum = 1; argnum < argc; ++argnum ) {
logFiles.append(argv[argnum]);
}
logFiles.rewind();
ReadMultipleUserLogs ru;
char *filename;
while ( (filename = logFiles.next()) ) {
MyString filestring( filename );
CondorError errstack;
if ( !ru.monitorLogFile( filestring, false, errstack ) ) {
fprintf( stderr, "Error monitoring log file %s: %s\n", filename,
errstack.getFullText().c_str() );
result = 1;
}
}
bool logsMissing = false;
CheckEvents ce;
int totalSubmitted = 0;
int netSubmitted = 0;
bool done = false;
while( !done ) {
ULogEvent* e = NULL;
MyString errorMsg;
ULogEventOutcome outcome = ru.readEvent( e );
switch (outcome) {
case ULOG_RD_ERROR:
case ULOG_UNK_ERROR:
logsMissing = true;
case ULOG_NO_EVENT:
printf( "Log outcome: %s\n", ULogEventOutcomeNames[outcome] );
done = true;
break;
case ULOG_OK:
printf( "Log event: %s (%d.%d.%d)",
ULogEventNumberNames[e->eventNumber],
e->cluster, e->proc, e->subproc );
if ( ce.CheckAnEvent(e, errorMsg) != CheckEvents::EVENT_OKAY ) {
fprintf(stderr, "%s\n", errorMsg.Value());
result = 1;
}
if( e->eventNumber == ULOG_SUBMIT ) {
SubmitEvent* ee = (SubmitEvent*) e;
printf( " (\"%s\")", ee->submitEventLogNotes );
++totalSubmitted;
++netSubmitted;
printf( "\n Total submitted: %d; net submitted: %d\n",
totalSubmitted, netSubmitted );
}
if( e->eventNumber == ULOG_JOB_HELD ) {
JobHeldEvent* ee = (JobHeldEvent*) e;
printf( " (code=%d subcode=%d)", ee->getReasonCode(),
ee->getReasonSubCode());
}
if( e->eventNumber == ULOG_JOB_TERMINATED ) {
--netSubmitted;
printf( "\n Total submitted: %d; net submitted: %d\n",
totalSubmitted, netSubmitted );
}
if( e->eventNumber == ULOG_JOB_ABORTED ) {
--netSubmitted;
printf( "\n Total submitted: %d; net submitted: %d\n",
totalSubmitted, netSubmitted );
}
if( e->eventNumber == ULOG_EXECUTABLE_ERROR ) {
--netSubmitted;
printf( "\n Total submitted: %d; net submitted: %d\n",
totalSubmitted, netSubmitted );
}
printf( "\n" );
break;
default:
fprintf(stderr, "Unexpected read event outcome!\n");
result = 1;
break;
}
}
logFiles.rewind();
while ( (filename = logFiles.next()) ) {
MyString filestring( filename );
CondorError errstack;
if ( !ru.unmonitorLogFile( filestring, errstack ) ) {
fprintf( stderr, "Error unmonitoring log file %s: %s\n", filename,
errstack.getFullText().c_str() );
result = 1;
}
}
MyString errorMsg;
CheckEvents::check_event_result_t checkAllResult =
ce.CheckAllJobs(errorMsg);
if ( checkAllResult != CheckEvents::EVENT_OKAY ) {
fprintf(stderr, "%s\n", errorMsg.Value());
fprintf(stderr, "CheckAllJobs() result: %s\n",
CheckEvents::ResultToString(checkAllResult));
result = 1;
}
if ( result == 0 ) {
if ( !logsMissing ) {
printf("Log(s) are okay\n");
} else {
printf("Log(s) may be okay\n");
printf( "Some logs cannot be read\n");
}
} else {
printf("Log(s) have error(s)\n");
}
return result;
}
//-------------------------------------------------------------------------
bool
GetConfigAndAttrs( /* const */ StringList &dagFiles, bool useDagDir,
MyString &configFile, StringList &attrLines, MyString &errMsg )
{
bool result = true;
// Note: destructor will change back to original directory.
TmpDir dagDir;
dagFiles.rewind();
char *dagFile;
while ( (dagFile = dagFiles.next()) != NULL ) {
//
// Change to the DAG file's directory if necessary, and
// get the filename we need to use for it from that directory.
//
const char * newDagFile;
if ( useDagDir ) {
MyString tmpErrMsg;
if ( !dagDir.Cd2TmpDirFile( dagFile, tmpErrMsg ) ) {
AppendError( errMsg,
MyString("Unable to change to DAG directory ") +
tmpErrMsg );
return false;
}
newDagFile = condor_basename( dagFile );
} else {
newDagFile = dagFile;
}
StringList configFiles;
// Note: destructor will close file.
MultiLogFiles::FileReader reader;
errMsg = reader.Open( newDagFile );
if ( errMsg != "" ) {
return false;
}
MyString logicalLine;
while ( reader.NextLogicalLine( logicalLine ) ) {
if ( logicalLine != "" ) {
// Note: StringList constructor removes leading
// whitespace from lines.
StringList tokens( logicalLine.Value(), " \t" );
tokens.rewind();
const char *firstToken = tokens.next();
if ( !strcasecmp( firstToken, "config" ) ) {
// Get the value.
const char *newValue = tokens.next();
if ( !newValue || !strcmp( newValue, "" ) ) {
AppendError( errMsg, "Improperly-formatted "
"file: value missing after keyword "
"CONFIG" );
result = false;
} else {
// Add the value we just found to the config
// files list (if it's not already in the
// list -- we don't want duplicates).
configFiles.rewind();
char *existingValue;
bool alreadyInList = false;
while ( ( existingValue = configFiles.next() ) ) {
if ( !strcmp( existingValue, newValue ) ) {
alreadyInList = true;
}
}
if ( !alreadyInList ) {
// Note: append copies the string here.
configFiles.append( newValue );
}
}
//some DAG commands are needed for condor_submit_dag, too...
} else if ( !strcasecmp( firstToken, "SET_JOB_ATTR" ) ) {
// Strip of DAGMan-specific command name; the
// rest we pass to the submit file.
logicalLine.replaceString( "SET_JOB_ATTR", "" );
logicalLine.trim();
if ( logicalLine == "" ) {
AppendError( errMsg, "Improperly-formatted "
"file: value missing after keyword "
"SET_JOB_ATTR" );
result = false;
} else {
attrLines.append( logicalLine.Value() );
}
}
}
}
reader.Close();
//
// Check the specified config file(s) against whatever we
// currently have, setting the config file if it hasn't
// been set yet, flagging an error if config files conflict.
//
configFiles.rewind();
char * cfgFile;
while ( (cfgFile = configFiles.next()) ) {
MyString cfgFileMS = cfgFile;
MyString tmpErrMsg;
if ( MakePathAbsolute( cfgFileMS, tmpErrMsg ) ) {
if ( configFile == "" ) {
configFile = cfgFileMS;
} else if ( configFile != cfgFileMS ) {
AppendError( errMsg, MyString("Conflicting DAGMan ") +
"config files specified: " + configFile +
" and " + cfgFileMS );
result = false;
}
} else {
AppendError( errMsg, tmpErrMsg );
result = false;
}
}
//
// Go back to our original directory.
//
MyString tmpErrMsg;
if ( !dagDir.Cd2MainDir( tmpErrMsg ) ) {
AppendError( errMsg,
MyString("Unable to change to original directory ") +
tmpErrMsg );
result = false;
}
}
return result;
}
// Gets the log files from a Stork submit file.
MyString
MultiLogFiles::loadLogFileNamesFromStorkSubFile(
const MyString &strSubFilename,
const MyString &directory,
StringList &listLogFilenames)
{
MyString rtnVal;
MyString path;
std::string adBuf;
classad::ClassAdParser parser;
classad::PrettyPrint unparser;
std::string unparsed;
dprintf( D_FULLDEBUG,
"MultiLogFiles::loadLogFileNamesFromStorkSubFile(%s, %s)\n",
strSubFilename.Value(), directory.Value() );
// Construct fully qualified path from directory and log file.
if ( directory.Length() > 0 ) {
path = directory + DIR_DELIM_STRING;
}
path += strSubFilename;
// Read submit file into std::string buffer, the native input buffer for
// the [new] ClassAds parser.
rtnVal = MultiLogFiles::readFile( path.Value(), adBuf);
if (rtnVal.Length() > 0 ) {
return rtnVal;
}
// read all classads out of the input file
int offset = 0;
classad::ClassAd ad;
// Loop through the Stork submit file, parsing out one submit job [ClassAd]
// at a time.
skip_whitespace(adBuf,offset); // until the parser can do this itself
while (parser.ParseClassAd(adBuf, ad, offset) ) {
std::string logfile;
// ad now contains the next Stork job ClassAd. Extract log file, if
// found.
if ( ! ad.EvaluateAttrString("log", logfile) ) {
// no log file specified
continue;
}
// reject empty log file names
if ( logfile.empty() ) {
unparser.Unparse( unparsed, &ad);
rtnVal.formatstr("Stork job specifies null log file:%s",
unparsed.c_str() );
return rtnVal;
}
// reject log file names with embedded macros
if ( logfile.find('$') != std::string::npos) {
unparser.Unparse( unparsed, &ad);
rtnVal.formatstr("macros not allowed in Stork log file names:%s",
unparsed.c_str() );
return rtnVal;
}
// All logfile must be fully qualified paths. Prepend the current
// working directory if logfile not a fully qualified path.
if ( ! fullpath(logfile.c_str() ) ) {
MyString currentDir;
if ( ! condor_getcwd(currentDir) ) {
rtnVal.formatstr("condor_getcwd() failed with errno %d (%s)",
errno, strerror(errno));
dprintf(D_ALWAYS, "ERROR: %s at %s:%d\n", rtnVal.Value(),
__FILE__, __LINE__);
return rtnVal;
}
std::string tmp = currentDir.Value();
tmp += DIR_DELIM_STRING;
tmp += logfile;
logfile = tmp;
}
// Add the log file we just found to the log file list
// (if it's not already in the list -- we don't want
// duplicates).
listLogFilenames.rewind();
char *psLogFilename;
bool bAlreadyInList = false;
while ( (psLogFilename = listLogFilenames.next()) ) {
if (logfile == psLogFilename) {
bAlreadyInList = true;
}
}
if (!bAlreadyInList) {
// Note: append copies the string here.
listLogFilenames.append(logfile.c_str() );
}
skip_whitespace(adBuf,offset); // until the parser can do this itself
}
return rtnVal;
}
int
stdin_pipe_handler(Service*, int) {
std::string* line;
while ((line = stdin_buffer.GetNextLine()) != NULL) {
const char * command = line->c_str();
dprintf (D_ALWAYS, "got stdin: %s\n", command);
Gahp_Args args;
if (parse_gahp_command (command, &args) &&
verify_gahp_command (args.argv, args.argc)) {
// Catch "special commands first
if (strcasecmp (args.argv[0], GAHP_COMMAND_RESULTS) == 0) {
// Print number of results
std::string rn_buff;
formatstr( rn_buff, "%d", result_list.number() );
const char * commands [] = {
GAHP_RESULT_SUCCESS,
rn_buff.c_str() };
gahp_output_return (commands, 2);
// Print each result line
char * next;
result_list.rewind();
while ((next = result_list.next()) != NULL) {
printf ("%s\n", next);
fflush(stdout);
dprintf(D_FULLDEBUG,"put stdout: %s\n",next);
result_list.deleteCurrent();
}
new_results_signaled = FALSE;
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_VERSION) == 0) {
printf ("S %s\n", version);
fflush (stdout);
dprintf(D_FULLDEBUG,"put stdout: S %s\n",version);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_QUIT) == 0) {
gahp_output_return_success();
DC_Exit(0);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_ASYNC_MODE_ON) == 0) {
async_mode = TRUE;
new_results_signaled = FALSE;
gahp_output_return_success();
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_ASYNC_MODE_OFF) == 0) {
async_mode = FALSE;
gahp_output_return_success();
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_QUIT) == 0) {
gahp_output_return_success();
return 0; // exit
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_COMMANDS) == 0) {
const char * commands [] = {
GAHP_RESULT_SUCCESS,
GAHP_COMMAND_DOWNLOAD_SANDBOX,
GAHP_COMMAND_UPLOAD_SANDBOX,
GAHP_COMMAND_DESTROY_SANDBOX,
GAHP_COMMAND_ASYNC_MODE_ON,
GAHP_COMMAND_ASYNC_MODE_OFF,
GAHP_COMMAND_RESULTS,
GAHP_COMMAND_QUIT,
GAHP_COMMAND_VERSION,
GAHP_COMMAND_COMMANDS};
gahp_output_return (commands, 10);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_DOWNLOAD_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!\n" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_download_sandbox, (void*)strdup(command), NULL, download_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created download_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
close( fds[0] );
}
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_UPLOAD_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!\n" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_upload_sandbox, (void*)strdup(command), NULL, upload_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created upload_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
close( fds[0] );
}
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_DESTROY_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!\n" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_destroy_sandbox, (void*)strdup(command), NULL, destroy_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created destroy_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
close( fds[0] );
}
} else {
// should never get here if verify does its job
dprintf(D_ALWAYS, "FTGAHP: got bad command: %s\n", args.argv[0]);
gahp_output_return_error();
}
} else {
gahp_output_return_error();
}
delete line;
}
// check if GetNextLine() returned NULL because of an error or EOF
if (stdin_buffer.IsError() || stdin_buffer.IsEOF()) {
dprintf (D_ALWAYS, "stdin buffer closed, exiting\n");
DC_Exit (1);
}
return TRUE;
}
//---------------------------------------------------------------------------
bool
Job::MonitorLogFile( ReadMultipleUserLogs &condorLogReader,
ReadMultipleUserLogs &storkLogReader, bool nfsIsError,
bool recovery, const char *defaultNodeLog, bool usingDefault )
{
debug_printf( DEBUG_DEBUG_2,
"Attempting to monitor log file for node %s\n",
GetJobName() );
if ( _logIsMonitored ) {
debug_printf( DEBUG_DEBUG_1, "Warning: log file for node "
"%s is already monitored\n", GetJobName() );
return true;
}
ReadMultipleUserLogs &logReader = (_jobType == TYPE_CONDOR) ?
condorLogReader : storkLogReader;
std::string logFileStr;
if ( _jobType == TYPE_CONDOR ) {
// We check to see if the user has specified a log file
// If not, we give him a default
MyString templogFileStr = MultiLogFiles::loadLogFileNameFromSubFile( _cmdFile,
_directory, _logFileIsXml, usingDefault);
logFileStr = templogFileStr.Value();
} else {
StringList logFiles;
MyString tmpResult = MultiLogFiles::loadLogFileNamesFromStorkSubFile(
_cmdFile, _directory, logFiles );
if ( tmpResult != "" ) {
debug_printf( DEBUG_QUIET, "Error getting Stork log file: %s\n",
tmpResult.Value() );
LogMonitorFailed();
return false;
} else if ( logFiles.number() != 1 ) {
debug_printf( DEBUG_QUIET, "Error: %d Stork log files found "
"in submit file %s; we want 1\n",
logFiles.number(), _cmdFile );
LogMonitorFailed();
return false;
} else {
logFiles.rewind();
logFileStr = logFiles.next();
}
}
// Warn the user if the node's log file is in /tmp.
if ( logFileStr.find( "/tmp" ) == 0 ) {
debug_printf( DEBUG_QUIET, "Warning: "
"Log file %s for node %s is in /tmp\n",
logFileStr.c_str(), GetJobName() );
check_warning_strictness( usingDefault ? DAG_STRICT_2 : DAG_STRICT_1 );
}
if ( logFileStr == "" ) {
logFileStr = defaultNodeLog;
_useDefaultLog = true;
// Default User log is never XML
// This could be specified in the submit file and should be
// ignored.
_logFileIsXml = false;
debug_printf( DEBUG_NORMAL, "Unable to get log file from "
"submit file %s (node %s); using default (%s)\n",
_cmdFile, GetJobName(), logFileStr.c_str() );
append_default_log = false;
} else {
append_default_log = usingDefault;
if( append_default_log ) {
// DAGman is not going to look at the user-specified log.
// It will look at the defaultNode log.
logFileStr = defaultNodeLog;
_useDefaultLog = false;
_logFileIsXml = false;
}
}
// This function returns true if the log file is on NFS and
// that is an error. If the log file is on NFS, but nfsIsError
// is false, it prints a warning but returns false.
if ( MultiLogFiles::logFileNFSError( logFileStr.c_str(),
nfsIsError ) ) {
debug_printf( DEBUG_QUIET, "Error: log file %s on NFS\n",
logFileStr.c_str() );
LogMonitorFailed();
return false;
}
delete [] _logFile;
// Saving log file here in case submit file gets changed.
_logFile = strnewp( logFileStr.c_str() );
debug_printf( DEBUG_DEBUG_2, "Monitoring log file <%s> for node %s\n",
GetLogFile(), GetJobName() );
CondorError errstack;
if ( !logReader.monitorLogFile( GetLogFile(), !recovery, errstack ) ) {
errstack.pushf( "DAGMan::Job", DAGMAN_ERR_LOG_FILE,
"ERROR: Unable to monitor log file for node %s",
GetJobName() );
debug_printf( DEBUG_QUIET, "%s\n", errstack.getFullText().c_str() );
LogMonitorFailed();
EXCEPT( "Fatal log file monitoring error!\n" );
return false;
}
_logIsMonitored = true;
return true;
}
int
stdin_pipe_handler(Service*, int) {
std::string* line;
while ((line = stdin_buffer.GetNextLine()) != NULL) {
const char * command = line->c_str();
// CREATE_CONDOR_SECURITY_SESSION contains sensitive data that
// normally shouldn't be written to a publically-readable log.
// We should conceal it unless GAHP_DEBUG_HIDE_SENSITIVE_DATA
// says not to.
if ( param_boolean( "GAHP_DEBUG_HIDE_SENSITIVE_DATA", true ) &&
strncmp( command, GAHP_COMMAND_CREATE_CONDOR_SECURITY_SESSION,
strlen( GAHP_COMMAND_CREATE_CONDOR_SECURITY_SESSION ) ) == 0 ) {
dprintf( D_ALWAYS, "got stdin: %s XXXXXXXX\n",
GAHP_COMMAND_CREATE_CONDOR_SECURITY_SESSION );
} else {
dprintf (D_ALWAYS, "got stdin: %s\n", command);
}
Gahp_Args args;
if (parse_gahp_command (command, &args) &&
verify_gahp_command (args.argv, args.argc)) {
// Catch "special commands first
if (strcasecmp (args.argv[0], GAHP_COMMAND_RESULTS) == 0) {
// Print number of results
std::string rn_buff;
formatstr( rn_buff, "%d", result_list.number() );
const char * commands [] = {
GAHP_RESULT_SUCCESS,
rn_buff.c_str() };
gahp_output_return (commands, 2);
// Print each result line
char * next;
result_list.rewind();
while ((next = result_list.next()) != NULL) {
printf ("%s\n", next);
fflush(stdout);
dprintf(D_FULLDEBUG,"put stdout: %s\n",next);
result_list.deleteCurrent();
}
new_results_signaled = FALSE;
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_VERSION) == 0) {
printf ("S %s\n", version);
fflush (stdout);
dprintf(D_FULLDEBUG,"put stdout: S %s\n",version);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_QUIT) == 0) {
gahp_output_return_success();
DC_Exit(0);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_ASYNC_MODE_ON) == 0) {
async_mode = TRUE;
new_results_signaled = FALSE;
gahp_output_return_success();
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_ASYNC_MODE_OFF) == 0) {
async_mode = FALSE;
gahp_output_return_success();
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_QUIT) == 0) {
gahp_output_return_success();
return 0; // exit
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_COMMANDS) == 0) {
const char * commands [] = {
GAHP_RESULT_SUCCESS,
GAHP_COMMAND_DOWNLOAD_SANDBOX,
GAHP_COMMAND_UPLOAD_SANDBOX,
GAHP_COMMAND_DESTROY_SANDBOX,
GAHP_COMMAND_CREATE_CONDOR_SECURITY_SESSION,
GAHP_COMMAND_CONDOR_VERSION,
GAHP_COMMAND_ASYNC_MODE_ON,
GAHP_COMMAND_ASYNC_MODE_OFF,
GAHP_COMMAND_RESULTS,
GAHP_COMMAND_QUIT,
GAHP_COMMAND_VERSION,
GAHP_COMMAND_COMMANDS};
gahp_output_return (commands, 12);
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_CREATE_CONDOR_SECURITY_SESSION) == 0) {
ClaimIdParser claimid( args.argv[1] );
if ( !daemonCore->getSecMan()->CreateNonNegotiatedSecuritySession(
DAEMON,
claimid.secSessionId(),
claimid.secSessionKey(),
claimid.secSessionInfo(),
CONDOR_PARENT_FQU,
NULL,
0 ) ) {
gahp_output_return_error();
} else {
sec_session_id = claimid.secSessionId();
gahp_output_return_success();
}
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_CONDOR_VERSION) == 0) {
peer_condor_version = args.argv[1];
const char *reply [] = { GAHP_RESULT_SUCCESS,
escapeGahpString( CondorVersion() ) };
gahp_output_return( reply, 2 );
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_DOWNLOAD_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_download_sandbox, (void*)strdup(command), NULL, download_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created download_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
const char * res[2] = {
"Worker thread failed",
"NULL"
};
enqueue_result(args.argv[1], res, 2);
close( fds[0] );
}
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_UPLOAD_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_upload_sandbox, (void*)strdup(command), NULL, upload_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created upload_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
const char * res[1] = {
"Worker thread failed"
};
enqueue_result(args.argv[1], res, 1);
close( fds[0] );
}
} else if (strcasecmp (args.argv[0], GAHP_COMMAND_DESTROY_SANDBOX) == 0) {
int fds[2];
if ( pipe( fds ) < 0 ) {
EXCEPT( "Failed to create pipe!" );
}
ChildErrorPipe = fds[1];
int tid = daemonCore->Create_Thread(do_command_destroy_sandbox, (void*)strdup(command), NULL, destroy_sandbox_reaper_id);
close( fds[1] );
if( tid ) {
dprintf (D_ALWAYS, "BOSCO: created destroy_sandbox thread, id: %i\n", tid);
// this is a "success" in the sense that the gahp command was
// well-formatted. whether or not the file transfer works or
// not is not what we are reporting here.
gahp_output_return_success();
SandboxEnt e;
e.pid = tid;
e.request_id = args.argv[1];
e.sandbox_id = args.argv[2];
e.error_pipe = fds[0];
// transfer started, record the entry in the map
std::pair<int, struct SandboxEnt> p(tid, e);
sandbox_map.insert(p);
} else {
dprintf (D_ALWAYS, "BOSCO: Create_Thread FAILED!\n");
gahp_output_return_success();
const char * res[1] = {
"Worker thread failed"
};
enqueue_result(args.argv[1], res, 1);
close( fds[0] );
}
} else {
// should never get here if verify does its job
dprintf(D_ALWAYS, "FTGAHP: got bad command: %s\n", args.argv[0]);
gahp_output_return_error();
}
} else {
gahp_output_return_error();
}
delete line;
}
// check if GetNextLine() returned NULL because of an error or EOF
if (stdin_buffer.IsError() || stdin_buffer.IsEOF()) {
dprintf (D_ALWAYS, "stdin buffer closed, exiting\n");
DC_Exit (1);
}
return TRUE;
}
MyString
MultiLogFiles::getValuesFromFile(const MyString &fileName,
const MyString &keyword, StringList &values, int skipTokens)
{
MyString errorMsg;
StringList logicalLines;
if ( (errorMsg = fileNameToLogicalLines( fileName,
logicalLines )) != "" ) {
return errorMsg;
}
const char * logicalLine;
while ( (logicalLine = logicalLines.next()) ) {
if ( strcmp(logicalLine, "") ) {
// Note: StringList constructor removes leading
// whitespace from lines.
StringList tokens(logicalLine, " \t");
tokens.rewind();
if ( !strcasecmp(tokens.next(), keyword.Value()) ) {
// Skip over unwanted tokens.
for ( int skipped = 0; skipped < skipTokens; skipped++ ) {
if ( !tokens.next() ) {
MyString result = MyString( "Improperly-formatted DAG "
"file: value missing after keyword <" ) +
keyword + ">";
return result;
}
}
// Get the value.
const char *newValue = tokens.next();
if ( !newValue || !strcmp( newValue, "") ) {
MyString result = MyString( "Improperly-formatted DAG "
"file: value missing after keyword <" ) +
keyword + ">";
return result;
}
// Add the value we just found to the values list
// (if it's not already in the list -- we don't want
// duplicates).
values.rewind();
char *existingValue;
bool alreadyInList = false;
while ( (existingValue = values.next()) ) {
if (!strcmp( existingValue, newValue ) ) {
alreadyInList = true;
}
}
if (!alreadyInList) {
// Note: append copies the string here.
values.append(newValue);
}
}
}
}
return "";
}
