bool
BaseCodec::addAttributeToMap (ClassAd& ad, const char* name, AttributeMapType& _map)
{
ExprTree *expr;
// All these extra lookups are horrible. They have to
// be there because the ClassAd may have multiple
// copies of the same attribute name! This means that
// the last attribute with a given name will set the
// value, but the last attribute tends to be the
// attribute with the oldest (wrong) value. How
// annoying is that!
if (!(expr = ad.Lookup(name))) {
dprintf(D_FULLDEBUG, "Warning: failed to lookup attribute '%s' from ad\n", name);
return false;
}
classad::Value value;
ad.EvaluateExpr(expr,value);
std::string key = name;
switch (value.GetType()) {
// seems this covers expressions also
case classad::Value::ERROR_VALUE:
case classad::Value::UNDEFINED_VALUE:
case classad::Value::BOOLEAN_VALUE:
_map[key] = new AviaryAttribute(AviaryAttribute::EXPR_TYPE,trimQuotes(ExprTreeToString(expr)).c_str());
break;
case classad::Value::INTEGER_VALUE:
{
int i;
value.IsIntegerValue (i);
string i_str;
sprintf(i_str,"%d",i);
_map[key] = new AviaryAttribute(AviaryAttribute::INTEGER_TYPE,i_str.c_str());
break;
}
case classad::Value::REAL_VALUE:
{
double d;
value.IsRealValue(d);
string d_str;
sprintf(d_str,"%f",d);
_map[key] = new AviaryAttribute(AviaryAttribute::FLOAT_TYPE,d_str.c_str());
break;
}
case classad::Value::STRING_VALUE:
default:
_map[key] = new AviaryAttribute(AviaryAttribute::STRING_TYPE,trimQuotes(ExprTreeToString(expr)).c_str());
}
return true;
}
bool
Compress( ClassAdCollectionServer *server, LocalCollectionQuery *query,
const References &refs, CompressedAds &comp, list<ClassAd*> &rest )
{
string key, sig;
References::const_iterator ritr;
ClassAd *ad;
CompressedAds::iterator citr;
ClassAdBin *bin;
query->ToFirst( );
query->Current( key );
while( !query->IsAfterLast( ) ) {
// get ad
if( !( ad = server->GetClassAd( key ) ) ) {
return( false );
}
// get signature of current ad
if( !MakeSignature( ad, refs, sig ) ) {
// can't make signature --- can't compress
rest.push_back( (ClassAd*) ad->Copy( ) );
}
// get bin
if( ( citr = comp.find( sig ) ) == comp.end( ) ) {
// no bin ... make one
bin = new ClassAdBin;
bin->count = 1;
// make a projected classad
for( ritr=refs.begin( ); ritr!=refs.end( ); ritr++ ) {
bin->ad->Insert( *ritr, ad->Lookup( *ritr )->Copy( ), false );
}
// insert bin into container
comp[sig] = bin;
} else {
// increment membership in bin
bin = citr->second;
bin->count++;
}
// process next ad
query->Next( key );
}
return( true );
}
ExprTree *View::
GetRankExpr( )
{
ClassAd *lAd;
ExprTree *tree;
// get rank expression from ad in left context
if( !(lAd = evalEnviron.GetLeftAd( ) ) ) {
CLASSAD_EXCEPT( "internal error: no view info in view" );
}
if( !(tree = lAd->Lookup( ATTR_RANK ) ) ) {
CondorErrno = ERR_NO_RANK_EXPR;
CondorErrMsg = "no 'Rank' expression in view info";
return( (ExprTree*) NULL );
}
return( tree );
}
ExprTree *View::
GetConstraintExpr( )
{
ClassAd *lAd;
ExprTree *tree;
// get requirements expression from ad in left context
if( !(lAd = evalEnviron.GetLeftAd( ) ) ) {
CLASSAD_EXCEPT( "internal error: no view info in view" );
}
if( !(tree = lAd->Lookup( ATTR_REQUIREMENTS ) ) ) {
CondorErrno = ERR_NO_REQUIREMENTS_EXPR;
CondorErrMsg = "no 'Requirements' expression in view info";
return( (ExprTree*) NULL );
}
return( tree );
}
bool
OfflineCollectorPlugin::expire (
ClassAd &ad )
{
classad::Value result;
bool val;
dprintf (
D_FULLDEBUG,
"In OfflineCollectorPlugin::expire()\n" );
/* bail out if the plug-in is not enabled, or if no ABSENT_REQUIREMENTS
have been defined */
if ( !enabled() || !AbsentReq ) {
return false; // return false tells collector to delete this ad
}
/* for now, if the ad is of any type other than a startd ad, bail out. currently
absent ads only supported for ads of type Machine, because our offline storage
assumes that. */
if ( strcmp(GetMyTypeName(ad),STARTD_ADTYPE) ) {
return false; // return false tells collector to delete this ad
}
/* The ad may be a STARTD_PVT_ADTYPE, even though GetMyTypeName() claims
it is a STARTD_ADTYPE. Sigh. This is because the startd sends private
ads w/ the wrong type, because the query object queries private ads w/ the
wrong type. If I were to fix the startd to label private ads with the proper
type, an incompatibility between startd/negotiator would have to be dealt with.
So here we try to distinguish if this ad is really a STARTD_PVT_ADTYPE by seeing
if a Capability attr is present and a State attr is not present. */
if ( ad.Lookup(ATTR_CAPABILITY) && !ad.Lookup(ATTR_STATE) ) {
// looks like a private ad, we don't want to store these
return false; // return false tells collector to delete this ad
}
/* If the ad is alraedy has ABSENT=True and it is expiring, then
let it be deleted as in this case it already sat around absent
for the absent lifetime. */
bool already_absent = false;
ad.LookupBool(ATTR_ABSENT,already_absent);
if (already_absent) {
MyString s;
const char *key = makeOfflineKey(ad,s);
if (key) {
persistentRemoveAd(key);
}
return false; // return false tells collector to delete this ad
}
/* Test is ad against the absent requirements expression, and
mark the ad absent if true */
if (EvalExprTree(AbsentReq,&ad,NULL,result) &&
result.IsBooleanValue(val) && val)
{
int lifetime, timestamp;
lifetime = param_integer (
"ABSENT_EXPIRE_ADS_AFTER",
60 * 60 * 24 * 30 ); // default expire absent ads in a month
if ( lifetime == 0 ) lifetime = INT_MAX; // 0 means forever
ad.Assign ( ATTR_ABSENT, true );
ad.Assign ( ATTR_CLASSAD_LIFETIME, lifetime );
timestamp = time(NULL);
ad.Assign(ATTR_LAST_HEARD_FROM, timestamp);
ad.Assign ( ATTR_MY_CURRENT_TIME, timestamp );
persistentStoreAd(NULL,ad);
// if we marked this ad as absent, we want to keep it in the collector
return true; // return true tells the collector to KEEP this ad
}
return false; // return false tells collector to delete this ad
}
bool
SchedulerObject::submit(AttributeMapType &jobAdMap, std::string &id, std::string &text)
{
int cluster;
int proc;
if (!m_codec) {
text = "Codec has not been initialized";
return false;
}
// our mandatory set of attributes for a submit
const char* required[] = {
ATTR_JOB_CMD,
ATTR_REQUIREMENTS,
ATTR_OWNER,
ATTR_JOB_IWD,
NULL
};
// 1. Create transaction
BeginTransaction();
// 2. Create cluster
if (-1 == (cluster = NewCluster())) {
AbortTransaction();
text = "Failed to create new cluster";
return false;
}
// 3. Create proc
if (-1 == (proc = NewProc(cluster))) {
AbortTransaction();
text = "Failed to create new proc";
return false;
}
// 4. Submit job ad
// Schema: (vanilla job)
// Schedd demands - Owner, JobUniverse
// To run - JobStatus, Requirements
// Schedd excepts if no Owner
// Schedd prunes on startup if no Owner or JobUniverse
// Schedd won't run job without JobStatus
// Job cannot match without Requirements
// Shadow rejects jobs without an Iwd
// Shadow: Job has no CondorVersion, assuming pre version 6.3.3
// Shadow: Unix Vanilla job is pre version 6.3.3, setting 'TransferFiles = "NEVER"'
// Starter won't run job without Cmd
// Starter needs a valid Owner (local account name) if not using nobody
// condor_q requires ClusterId (int), ProcId (int), QDate (int), RemoteUserCpu (float), JobStatus (int), JobPrio (int), ImageSize (int), Owner (str) and Cmd (str)
// Schema: (vm job)
// ShouldTransferFiles - unset by default, must be set
ClassAd ad;
int universe;
// ShouldTransferFiles - unset by default, must be set
// shadow will try to setup local transfer sandbox otherwise
// without good priv
ad.Assign(ATTR_SHOULD_TRANSFER_FILES, "NO");
if (!m_codec->mapToClassAd(jobAdMap, ad, text)) {
AbortTransaction();
return false;
}
std::string missing;
if (!checkRequiredAttrs(ad, required, missing)) {
AbortTransaction();
text = "Job ad is missing required attributes: " + missing;
return false;
}
// EARLY SET: These attribute are set early so the incoming ad
// has a change to override them.
::SetAttribute(cluster, proc, ATTR_JOB_STATUS, "1"); // 1 = idle
// Junk that condor_q wants, but really shouldn't be necessary
::SetAttribute(cluster, proc, ATTR_JOB_REMOTE_USER_CPU, "0.0"); // float
::SetAttribute(cluster, proc, ATTR_JOB_PRIO, "0"); // int
::SetAttribute(cluster, proc, ATTR_IMAGE_SIZE, "0"); // int
if (!ad.LookupInteger(ATTR_JOB_UNIVERSE, universe)) {
char* uni_str = param("DEFAULT_UNIVERSE");
if (!uni_str) {
universe = CONDOR_UNIVERSE_VANILLA;
}
else {
universe = CondorUniverseNumber(uni_str);
}
::SetAttributeInt(cluster, proc, ATTR_JOB_UNIVERSE, universe );
}
// more stuff - without these our idle stats are whack
if ( universe != CONDOR_UNIVERSE_MPI && universe != CONDOR_UNIVERSE_PVM ) {
::SetAttribute(cluster, proc, ATTR_MAX_HOSTS, "1"); // int
::SetAttribute(cluster, proc, ATTR_MIN_HOSTS, "1"); // int
}
::SetAttribute(cluster, proc, ATTR_CURRENT_HOSTS, "0"); // int
ExprTree *expr;
const char *name;
std::string value;
ad.ResetExpr();
while (ad.NextExpr(name,expr)) {
// All these extra lookups are horrible. They have to
// be there because the ClassAd may have multiple
// copies of the same attribute name! This means that
// the last attribute with a given name will set the
// value, but the last attribute tends to be the
// attribute with the oldest (wrong) value. How
// annoying is that!
if (!(expr = ad.Lookup(name))) {
dprintf(D_ALWAYS, "Failed to lookup %s\n", name);
AbortTransaction();
text = "Failed to parse job ad attribute";
return false;
}
value = ExprTreeToString(expr);
::SetAttribute(cluster, proc, name, value.c_str());
}
// LATE SET: These attributes are set late, after the incoming
// ad, so they override whatever the incoming ad set.
char buf[22]; // 22 is max size for an id, 2^32 + . + 2^32 + \0
snprintf(buf, 22, "%d", cluster);
::SetAttribute(cluster, proc, ATTR_CLUSTER_ID, buf);
snprintf(buf, 22, "%d", proc);
::SetAttribute(cluster, proc, ATTR_PROC_ID, buf);
snprintf(buf, 22, "%ld", time(NULL));
::SetAttribute(cluster, proc, ATTR_Q_DATE, buf);
// Could check for some invalid attributes, e.g
// JobUniverse <= CONDOR_UNIVERSE_MIN or >= CONDOR_UNIVERSE_MAX
// 5. Commit transaction
CommitTransaction();
// 6. Reschedule
scheduler.needReschedule();
// 7. Return identifier
// TODO: dag ids?
string tmp;
//tmp.sprintf("%s#%d.%d", Name, cluster, proc);
// we have other API compositions for job id and submission id
// so let's return raw cluster.proc
aviUtilFmt(tmp,"%d.%d", cluster, proc);
id = tmp.c_str();
return true;
}
int main( int argc, char *argv[] )
{
const char *filename=0;
char *pool=0;
int command=-1;
int i;
bool use_tcp = false;
bool with_ack = false;
bool allow_multiple = false;
param_functions *p_funcs = NULL;
myDistro->Init( argc, argv );
config();
p_funcs = get_param_functions();
for( i=1; i<argc; i++ ) {
if(!strcmp(argv[i],"-help")) {
usage(argv[0]);
exit(0);
} else if(!strcmp(argv[i],"-pool")) {
i++;
if(!argv[i]) {
fprintf(stderr,"-pool requires an argument.\n\n");
usage(argv[0]);
exit(1);
}
pool = argv[i];
} else if(!strncmp(argv[i],"-tcp",strlen(argv[i]))) {
use_tcp = true;
} else if(!strncmp(argv[i],"-multiple",strlen(argv[i]))) {
// We don't set allow_multiple=true by default, because
// existing users (e.g. glideinWMS) have stray blank lines
// in the input file.
allow_multiple = true;
} else if(!strcmp(argv[i],"-version")) {
version();
exit(0);
} else if(!strcmp(argv[i],"-debug")) {
// dprintf to console
Termlog = 1;
p_funcs = get_param_functions();
dprintf_config ("TOOL", p_funcs);
} else if(argv[i][0]!='-' || !strcmp(argv[i],"-")) {
if(command==-1) {
command = getCollectorCommandNum(argv[i]);
if(command==-1) {
fprintf(stderr,"Unknown command name %s\n\n",argv[i]);
usage(argv[0]);
exit(1);
}
} else if(!filename) {
filename = argv[i];
} else {
fprintf(stderr,"Extra argument: %s\n\n",argv[i]);
usage(argv[0]);
exit(1);
}
} else {
fprintf(stderr,"Unknown argument: %s\n\n",argv[i]);
usage(argv[0]);
exit(1);
}
}
FILE *file;
ClassAdList ads;
Daemon *collector;
Sock *sock;
switch( command ) {
case UPDATE_STARTD_AD_WITH_ACK:
with_ack = true;
break;
}
if( with_ack ) {
use_tcp = true;
}
if(!filename || !strcmp(filename,"-")) {
file = stdin;
filename = "(stdin)";
} else {
file = safe_fopen_wrapper_follow(filename,"r");
}
if(!file) {
fprintf(stderr,"couldn't open %s: %s\n",filename,strerror(errno));
return 1;
}
while(!feof(file)) {
int eof=0,error=0,empty=0;
char const *delim = "\n";
if( !allow_multiple ) {
delim = "***";
}
ClassAd *ad = new ClassAd(file,const_cast<char *>(delim),eof,error,empty);
if(error) {
fprintf(stderr,"couldn't parse ClassAd in %s\n",filename);
delete ad;
return 1;
}
if( empty ) {
delete ad;
break;
}
if( !allow_multiple && ads.Length() > 0 ) {
fprintf(stderr,"ERROR: failed to parse '%s' as a ClassAd attribute\n",delim);
delete ad;
return 1;
}
ads.Insert(ad);
}
if(ads.Length() == 0) {
fprintf(stderr,"%s is empty\n",filename);
return 1;
}
CollectorList * collectors;
if ( pool ) {
collector = new Daemon( DT_COLLECTOR, pool, 0 );
collectors = new CollectorList();
collectors->append (collector);
} else {
collectors = CollectorList::create();
}
bool had_error = false;
collectors->rewind();
while (collectors->next(collector)) {
dprintf(D_FULLDEBUG,"locating collector %s...\n", collector->name());
if(!collector->locate()) {
fprintf(stderr,"couldn't locate collector: %s\n",collector->error());
had_error = true;
continue;
}
dprintf(D_FULLDEBUG,"collector is %s located at %s\n",
collector->hostname(),collector->addr());
sock = NULL;
ClassAd *ad;
int success_count = 0;
int failure_count = 0;
ads.Rewind();
while( (ad=ads.Next()) ) {
// If there's no "MyAddress", generate one..
if( !ad->Lookup( ATTR_MY_ADDRESS ) ) {
MyString tmp;
tmp.formatstr( "<%s:0>", my_ip_string() );
ad->Assign( ATTR_MY_ADDRESS, tmp.Value() );
}
if ( use_tcp ) {
if( !sock ) {
sock = collector->startCommand(command,Stream::reli_sock,20);
}
else {
// Use existing connection.
sock->encode();
sock->put(command);
}
} else {
// We must open a new UDP socket each time.
delete sock;
sock = collector->startCommand(command,Stream::safe_sock,20);
}
int result = 0;
if ( sock ) {
result += ad->put( *sock );
result += sock->end_of_message();
}
if ( result != 2 ) {
fprintf(stderr,"failed to send classad to %s\n",collector->addr());
had_error = true;
failure_count++;
delete sock;
sock = NULL;
continue;
}
if( with_ack ) {
sock->decode();
int ok = 0;
if( !sock->get(ok) || !sock->end_of_message() ) {
fprintf(stderr,"failed to get ack from %s\n",collector->addr());
had_error = true;
failure_count++;
delete sock;
sock = NULL;
continue;
}
// ack protocol does not allow for multiple updates,
// so close the socket now
delete sock;
sock = NULL;
}
success_count++;
}
if( sock ) {
CondorVersionInfo const *ver = sock->get_peer_version();
if( !ver || ver->built_since_version(7,7,3) ) {
// graceful hangup so the collector knows we are done
sock->encode();
command = DC_NOP;
sock->put(command);
sock->end_of_message();
}
delete sock;
sock = NULL;
}
printf("Sent %d of %d ad%s to %s.\n",
success_count,
success_count + failure_count,
success_count+failure_count == 1 ? "" : "s",
collector->name());
}
delete collectors;
return (had_error)?1:0;
}
int
unwind_attributes(classad_context cad, char *attribute_name, char ***results)
{
if (cad == NULL) return C_CLASSAD_INVALID_CONTEXT;
if ((results == NULL) || (attribute_name == NULL))
return C_CLASSAD_INVALID_ARG;
ClassAd *ad = (ClassAd *)cad;
ExprTree *et;
bool need_to_delete_et = false;
et = ad->Lookup(attribute_name);
if (et == NULL)
{
return C_CLASSAD_VALUE_NOT_FOUND;
}
if (et->GetKind() == ExprTree::LITERAL_NODE)
{
// The attribute was probably stringified. Try to parse it.
Value v;
EvalState state;
state.SetScopes( ad );
et->Evaluate(state,v);
std::string strres;
if (v.IsStringValue( strres ))
{
ClassAdParser parser;
et=NULL;
parser.ParseExpression(strres,et);
need_to_delete_et = true;
}
}
BinaryOpUnwind res_unp;
std::string result;
res_unp.Unparse(result, et);
int n_results;
if (*results == NULL)
{
n_results = 0;
(*results) = (char **)malloc(sizeof(char **));
if ((*results) == NULL) return C_CLASSAD_OUT_OF_MEMORY;
(*results)[0] = NULL;
}
else
{
for (n_results = 0; (*results)[n_results] != NULL; n_results++) /*NOP*/ ;
}
std::vector<std::string>::const_iterator it;
for (it = res_unp.m_unwind_output.begin();
it != res_unp.m_unwind_output.end(); ++it)
{
n_results++;
char **new_results;
new_results = (char **)realloc(*results, (n_results+1)*sizeof(char *));
if (new_results == NULL)
{
if (need_to_delete_et) delete et;
return C_CLASSAD_OUT_OF_MEMORY;
}
(*results) = new_results;
(*results)[n_results] = NULL;
(*results)[n_results-1] = strdup(it->c_str());
if (((*results)[n_results-1]) == NULL)
{
if (need_to_delete_et) delete et;
return C_CLASSAD_OUT_OF_MEMORY;
}
}
if (need_to_delete_et) delete et;
return C_CLASSAD_NO_ERROR;
}