Exemplo n.º 1
0
bool
BaseCodec::addAttributeToMap (ClassAd& ad, const char* name, AttributeMapType& _map)
{
    ExprTree *expr;

    // All these extra lookups are horrible. They have to
    // be there because the ClassAd may have multiple
    // copies of the same attribute name! This means that
    // the last attribute with a given name will set the
    // value, but the last attribute tends to be the
    // attribute with the oldest (wrong) value. How
    // annoying is that!
    if (!(expr = ad.Lookup(name))) {
        dprintf(D_FULLDEBUG, "Warning: failed to lookup attribute '%s' from ad\n", name);
        return false;
    }

    classad::Value value;
    ad.EvaluateExpr(expr,value);
	std::string key = name;
    switch (value.GetType()) {
        // seems this covers expressions also
        case classad::Value::ERROR_VALUE:
        case classad::Value::UNDEFINED_VALUE:
        case classad::Value::BOOLEAN_VALUE:
            _map[key] = new AviaryAttribute(AviaryAttribute::EXPR_TYPE,trimQuotes(ExprTreeToString(expr)).c_str());
            break;
        case classad::Value::INTEGER_VALUE:
        {
            int i;
            value.IsIntegerValue (i);
            string i_str;
            sprintf(i_str,"%d",i);
            _map[key] = new AviaryAttribute(AviaryAttribute::INTEGER_TYPE,i_str.c_str());
            break;
        }
        case classad::Value::REAL_VALUE:
        {
            double d;
            value.IsRealValue(d);
            string d_str;
            sprintf(d_str,"%f",d);
            _map[key] = new AviaryAttribute(AviaryAttribute::FLOAT_TYPE,d_str.c_str());
            break;
        }
        case classad::Value::STRING_VALUE:
        default:
            _map[key] = new AviaryAttribute(AviaryAttribute::STRING_TYPE,trimQuotes(ExprTreeToString(expr)).c_str());
    }

    return true;
}
Exemplo n.º 2
0
bool
Compress( ClassAdCollectionServer *server, LocalCollectionQuery *query,
          const References &refs, CompressedAds &comp, list<ClassAd*> &rest )
{
    string                      key, sig;
    References::const_iterator  ritr;
    ClassAd                     *ad;
    CompressedAds::iterator     citr;
    ClassAdBin                  *bin;

    query->ToFirst( );
    query->Current( key );
    while( !query->IsAfterLast( ) ) {
        // get ad
        if( !( ad = server->GetClassAd( key ) ) ) {
            return( false );
        }

        // get signature of current ad
        if( !MakeSignature( ad, refs, sig ) ) {
            // can't make signature --- can't compress
            rest.push_back( (ClassAd*) ad->Copy( ) );
        }

        // get bin
        if( ( citr = comp.find( sig ) ) == comp.end( ) ) {
            // no bin ... make one
            bin = new ClassAdBin;
            bin->count = 1;

            // make a projected classad
            for( ritr=refs.begin( ); ritr!=refs.end( ); ritr++ ) {
                bin->ad->Insert( *ritr, ad->Lookup( *ritr )->Copy( ), false );
            }

            // insert bin into container
            comp[sig] = bin;
        } else {
            // increment membership in bin
            bin = citr->second;
            bin->count++;
        }

        // process next ad
        query->Next( key );
    }

    return( true );
}
Exemplo n.º 3
0
ExprTree *View::
GetRankExpr( )
{
	ClassAd		*lAd;
	ExprTree	*tree;
		// get rank expression from ad in left context
	if( !(lAd = evalEnviron.GetLeftAd( ) ) ) {
		CLASSAD_EXCEPT( "internal error:  no view info in view" );
	}
	if( !(tree = lAd->Lookup( ATTR_RANK ) ) ) {
		CondorErrno = ERR_NO_RANK_EXPR;
		CondorErrMsg = "no 'Rank' expression in view info";
		return( (ExprTree*) NULL );
	}
	return( tree );
}
Exemplo n.º 4
0
ExprTree *View::
GetConstraintExpr( )
{
	ClassAd		*lAd;
	ExprTree	*tree;
		// get requirements expression from ad in left context
	if( !(lAd = evalEnviron.GetLeftAd( ) ) ) {
		CLASSAD_EXCEPT( "internal error:  no view info in view" );
	}
	if( !(tree = lAd->Lookup( ATTR_REQUIREMENTS ) ) ) {
		CondorErrno = ERR_NO_REQUIREMENTS_EXPR;
		CondorErrMsg = "no 'Requirements' expression in view info";
		return( (ExprTree*) NULL );
	}
	return( tree );
}
Exemplo n.º 5
0
bool 
OfflineCollectorPlugin::expire ( 
	ClassAd &ad )
{
	classad::Value result;
	bool val;

	dprintf (
		D_FULLDEBUG,
		"In OfflineCollectorPlugin::expire()\n" );

	/* bail out if the plug-in is not enabled, or if no ABSENT_REQUIREMENTS
	   have been defined */
	if ( !enabled() || !AbsentReq ) {
		return false;	// return false tells collector to delete this ad
	}

	/* for now, if the ad is of any type other than a startd ad, bail out. currently
	   absent ads only supported for ads of type Machine, because our offline storage
	   assumes that. */
	if ( strcmp(GetMyTypeName(ad),STARTD_ADTYPE) ) {
		return false;	// return false tells collector to delete this ad
	}
	/*	The ad may be a STARTD_PVT_ADTYPE, even though GetMyTypeName() claims 
		it is a STARTD_ADTYPE. Sigh. This is because the startd sends private 
		ads w/ the wrong type, because the query object queries private ads w/ the 
		wrong type. If I were to fix the startd to label private ads with the proper
		type, an incompatibility between startd/negotiator would have to be dealt with.
		So here we try to distinguish if this ad is really a STARTD_PVT_ADTYPE by seeing
		if a Capability attr is present and a State attr is not present. */
	if ( ad.Lookup(ATTR_CAPABILITY) && !ad.Lookup(ATTR_STATE) ) {
		// looks like a private ad, we don't want to store these
		return false;	// return false tells collector to delete this ad
	}


	/* If the ad is alraedy has ABSENT=True and it is expiring, then
	   let it be deleted as in this case it already sat around absent
	   for the absent lifetime. */
	bool already_absent = false;
	ad.LookupBool(ATTR_ABSENT,already_absent);
	if (already_absent) {
		MyString s;
		const char *key = makeOfflineKey(ad,s);
		if (key) {
			persistentRemoveAd(key);
		}
		return false; // return false tells collector to delete this ad
	}

	/* Test is ad against the absent requirements expression, and
	   mark the ad absent if true */
	if (EvalExprTree(AbsentReq,&ad,NULL,result) &&
		result.IsBooleanValue(val) && val) 
	{
		int lifetime, timestamp;

		lifetime = param_integer ( 
			"ABSENT_EXPIRE_ADS_AFTER",
			60 * 60 * 24 * 30 );	// default expire absent ads in a month		
		if ( lifetime == 0 ) lifetime = INT_MAX; // 0 means forever

		ad.Assign ( ATTR_ABSENT, true );
		ad.Assign ( ATTR_CLASSAD_LIFETIME, lifetime );
		timestamp = time(NULL);
		ad.Assign(ATTR_LAST_HEARD_FROM, timestamp);
		ad.Assign ( ATTR_MY_CURRENT_TIME, timestamp );
		persistentStoreAd(NULL,ad);
		// if we marked this ad as absent, we want to keep it in the collector
		return true;	// return true tells the collector to KEEP this ad
	}

	return false;	// return false tells collector to delete this ad
}
Exemplo n.º 6
0
bool
SchedulerObject::submit(AttributeMapType &jobAdMap, std::string &id, std::string &text)
{
	int cluster;
	int proc;

    if (!m_codec) {
        text = "Codec has not been initialized";
        return false;
    }

	// our mandatory set of attributes for a submit
	const char* required[] = {
				ATTR_JOB_CMD,
				ATTR_REQUIREMENTS,
				ATTR_OWNER,
				ATTR_JOB_IWD,
				NULL
				};

		// 1. Create transaction
	BeginTransaction();

		// 2. Create cluster
	if (-1 == (cluster = NewCluster())) {
		AbortTransaction();
		text = "Failed to create new cluster";
		return false;
	}

		// 3. Create proc
	if (-1 == (proc = NewProc(cluster))) {
		AbortTransaction();
		text = "Failed to create new proc";
		return false;
	}

		// 4. Submit job ad

		// Schema: (vanilla job)
		// Schedd demands - Owner, JobUniverse
		// To run - JobStatus, Requirements

		// Schedd excepts if no Owner
		// Schedd prunes on startup if no Owner or JobUniverse
		// Schedd won't run job without JobStatus
		// Job cannot match without Requirements
		// Shadow rejects jobs without an Iwd
		// Shadow: Job has no CondorVersion, assuming pre version 6.3.3
		// Shadow: Unix Vanilla job is pre version 6.3.3, setting 'TransferFiles = "NEVER"'
		// Starter won't run job without Cmd
		// Starter needs a valid Owner (local account name) if not using nobody
		// condor_q requires ClusterId (int), ProcId (int), QDate (int), RemoteUserCpu (float), JobStatus (int), JobPrio (int), ImageSize (int), Owner (str) and Cmd (str)

		// Schema: (vm job)
		// ShouldTransferFiles - unset by default, must be set

	ClassAd ad;
	int universe;

    // ShouldTransferFiles - unset by default, must be set
    // shadow will try to setup local transfer sandbox otherwise
    // without good priv
    ad.Assign(ATTR_SHOULD_TRANSFER_FILES, "NO");

	if (!m_codec->mapToClassAd(jobAdMap, ad, text)) {
		AbortTransaction();
		return false;
	}

	std::string missing;
	if (!checkRequiredAttrs(ad, required, missing)) {
		AbortTransaction();
		text = "Job ad is missing required attributes: " + missing;
		return false;
	}

		// EARLY SET: These attribute are set early so the incoming ad
		// has a change to override them.
	::SetAttribute(cluster, proc, ATTR_JOB_STATUS, "1"); // 1 = idle

		// Junk that condor_q wants, but really shouldn't be necessary
	::SetAttribute(cluster, proc, ATTR_JOB_REMOTE_USER_CPU, "0.0"); // float
	::SetAttribute(cluster, proc, ATTR_JOB_PRIO, "0");              // int
	::SetAttribute(cluster, proc, ATTR_IMAGE_SIZE, "0");            // int

	if (!ad.LookupInteger(ATTR_JOB_UNIVERSE, universe)) {
		char* uni_str = param("DEFAULT_UNIVERSE");
		if (!uni_str) {
			universe = CONDOR_UNIVERSE_VANILLA;
		}
		else {
			universe = CondorUniverseNumber(uni_str);
		}
		::SetAttributeInt(cluster, proc, ATTR_JOB_UNIVERSE, universe );
	}
	// more stuff - without these our idle stats are whack
	if ( universe != CONDOR_UNIVERSE_MPI && universe != CONDOR_UNIVERSE_PVM ) {
		::SetAttribute(cluster, proc, ATTR_MAX_HOSTS, "1");              // int
		::SetAttribute(cluster, proc, ATTR_MIN_HOSTS, "1");            // int
	}
	::SetAttribute(cluster, proc, ATTR_CURRENT_HOSTS, "0"); // int

	ExprTree *expr;
	const char *name;
	std::string value;
	ad.ResetExpr();
	while (ad.NextExpr(name,expr)) {

			// All these extra lookups are horrible. They have to
			// be there because the ClassAd may have multiple
			// copies of the same attribute name! This means that
			// the last attribute with a given name will set the
			// value, but the last attribute tends to be the
			// attribute with the oldest (wrong) value. How
			// annoying is that!
		if (!(expr = ad.Lookup(name))) {
			dprintf(D_ALWAYS, "Failed to lookup %s\n", name);

			AbortTransaction();
			text = "Failed to parse job ad attribute";
			return false;
		}

        value = ExprTreeToString(expr);
        ::SetAttribute(cluster, proc, name, value.c_str());
	}

		// LATE SET: These attributes are set late, after the incoming
		// ad, so they override whatever the incoming ad set.
	char buf[22]; // 22 is max size for an id, 2^32 + . + 2^32 + \0
	snprintf(buf, 22, "%d", cluster);
	::SetAttribute(cluster, proc, ATTR_CLUSTER_ID, buf);
	snprintf(buf, 22, "%d", proc);
	::SetAttribute(cluster, proc, ATTR_PROC_ID, buf);
	snprintf(buf, 22, "%ld", time(NULL));
	::SetAttribute(cluster, proc, ATTR_Q_DATE, buf);

		// Could check for some invalid attributes, e.g
		//  JobUniverse <= CONDOR_UNIVERSE_MIN or >= CONDOR_UNIVERSE_MAX

		// 5. Commit transaction
	CommitTransaction();


		// 6. Reschedule
	scheduler.needReschedule();


		// 7. Return identifier
		// TODO: dag ids?
	string tmp;
	//tmp.sprintf("%s#%d.%d", Name, cluster, proc);
	// we have other API compositions for job id and submission id
	// so let's return raw cluster.proc
	aviUtilFmt(tmp,"%d.%d", cluster, proc);
	id = tmp.c_str();

	return true;
}
Exemplo n.º 7
0
int main( int argc, char *argv[] )
{
	const char *filename=0;
	char *pool=0;
	int command=-1;
	int i;
	bool use_tcp = false;
	bool with_ack = false;
	bool allow_multiple = false;
	param_functions *p_funcs = NULL;


	myDistro->Init( argc, argv );
	config();
	p_funcs = get_param_functions();

	for( i=1; i<argc; i++ ) {
		if(!strcmp(argv[i],"-help")) {
			usage(argv[0]);
			exit(0);
		} else if(!strcmp(argv[i],"-pool")) {	
			i++;
			if(!argv[i]) {
				fprintf(stderr,"-pool requires an argument.\n\n");
				usage(argv[0]);
				exit(1);
			}
			pool = argv[i];
		} else if(!strncmp(argv[i],"-tcp",strlen(argv[i]))) {
			use_tcp = true;
		} else if(!strncmp(argv[i],"-multiple",strlen(argv[i]))) {
				// We don't set allow_multiple=true by default, because
				// existing users (e.g. glideinWMS) have stray blank lines
				// in the input file.
			allow_multiple = true;
		} else if(!strcmp(argv[i],"-version")) {
			version();
			exit(0);
		} else if(!strcmp(argv[i],"-debug")) {
				// dprintf to console
			Termlog = 1;
			p_funcs = get_param_functions();
			dprintf_config ("TOOL", p_funcs);
		} else if(argv[i][0]!='-' || !strcmp(argv[i],"-")) {
			if(command==-1) {
				command = getCollectorCommandNum(argv[i]);
				if(command==-1) {
					fprintf(stderr,"Unknown command name %s\n\n",argv[i]);
					usage(argv[0]);
					exit(1);
				}
			} else if(!filename) {
				filename = argv[i];
			} else {
				fprintf(stderr,"Extra argument: %s\n\n",argv[i]);
				usage(argv[0]);
				exit(1);
			}
		} else {
			fprintf(stderr,"Unknown argument: %s\n\n",argv[i]);
			usage(argv[0]);
			exit(1);
		}
	}

	FILE *file;
	ClassAdList ads;
	Daemon *collector;
	Sock *sock;

	switch( command ) {
	case UPDATE_STARTD_AD_WITH_ACK:
		with_ack = true;
		break;
	}

	if( with_ack ) {
		use_tcp =  true;
	}

	if(!filename || !strcmp(filename,"-")) {
		file = stdin;
		filename = "(stdin)";
	} else {
		file = safe_fopen_wrapper_follow(filename,"r");
	}
	if(!file) {
		fprintf(stderr,"couldn't open %s: %s\n",filename,strerror(errno));
		return 1;
	}

	while(!feof(file)) {
		int eof=0,error=0,empty=0;
		char const *delim = "\n";
		if( !allow_multiple ) {
			delim = "***";
		}
		ClassAd *ad = new ClassAd(file,const_cast<char *>(delim),eof,error,empty);
		if(error) {
			fprintf(stderr,"couldn't parse ClassAd in %s\n",filename);
			delete ad;
			return 1;
		}
		if( empty ) {
			delete ad;
			break;
		}
		if( !allow_multiple && ads.Length() > 0 ) {
			fprintf(stderr,"ERROR: failed to parse '%s' as a ClassAd attribute\n",delim);
			delete ad;
			return 1;
		}
		ads.Insert(ad);
	}

	if(ads.Length() == 0) {
		fprintf(stderr,"%s is empty\n",filename);
		return 1;
	}

	CollectorList * collectors;
	if ( pool ) {
		collector = new Daemon( DT_COLLECTOR, pool, 0 );
		collectors = new CollectorList();
		collectors->append (collector);
	} else {
		collectors = CollectorList::create();
	}

	bool had_error = false;

	collectors->rewind();
	while (collectors->next(collector)) {
		
		dprintf(D_FULLDEBUG,"locating collector %s...\n", collector->name());

		if(!collector->locate()) {
			fprintf(stderr,"couldn't locate collector: %s\n",collector->error());
			had_error = true;
			continue;
		}

		dprintf(D_FULLDEBUG,"collector is %s located at %s\n",
				collector->hostname(),collector->addr());

		sock = NULL;

		ClassAd *ad;
		int success_count = 0;
		int failure_count = 0;
		ads.Rewind();
		while( (ad=ads.Next()) ) {

				// If there's no "MyAddress", generate one..
			if( !ad->Lookup( ATTR_MY_ADDRESS ) ) {
				MyString tmp;
				tmp.formatstr( "<%s:0>", my_ip_string() );
				ad->Assign( ATTR_MY_ADDRESS, tmp.Value() );
			}

			if ( use_tcp ) {
				if( !sock ) {
					sock = collector->startCommand(command,Stream::reli_sock,20);
				}
				else {
						// Use existing connection.
					sock->encode();
					sock->put(command);
				}
			} else {
					// We must open a new UDP socket each time.
				delete sock;
				sock = collector->startCommand(command,Stream::safe_sock,20);
			}

			int result = 0;
			if ( sock ) {
				result += ad->put( *sock );
				result += sock->end_of_message();
			}
			if ( result != 2 ) {
				fprintf(stderr,"failed to send classad to %s\n",collector->addr());
				had_error = true;
				failure_count++;
				delete sock;
				sock = NULL;
				continue;
			}

			if( with_ack ) {
				sock->decode();
				int ok = 0;
				if( !sock->get(ok) || !sock->end_of_message() ) {
					fprintf(stderr,"failed to get ack from %s\n",collector->addr());
					had_error = true;
					failure_count++;
					delete sock;
					sock = NULL;
					continue;
				}

					// ack protocol does not allow for multiple updates,
					// so close the socket now
				delete sock;
				sock = NULL;
			}

			success_count++;
		}
		if( sock ) {
			CondorVersionInfo const *ver = sock->get_peer_version();
			if( !ver || ver->built_since_version(7,7,3) ) {
					// graceful hangup so the collector knows we are done
				sock->encode();
				command = DC_NOP;
				sock->put(command);
				sock->end_of_message();
			}

			delete sock;
			sock = NULL;
		}

		printf("Sent %d of %d ad%s to %s.\n",
			   success_count,
			   success_count + failure_count,
			   success_count+failure_count == 1 ? "" : "s",
			   collector->name());
	}

	delete collectors;

	return (had_error)?1:0;
}
Exemplo n.º 8
0
  int
  unwind_attributes(classad_context cad, char *attribute_name, char ***results) 
   {
    if (cad == NULL) return C_CLASSAD_INVALID_CONTEXT;
    if ((results == NULL) || (attribute_name == NULL))
      return C_CLASSAD_INVALID_ARG;

    ClassAd *ad = (ClassAd *)cad;

    ExprTree *et;
    bool need_to_delete_et = false;

    et = ad->Lookup(attribute_name);
    if (et == NULL)
     {
      return C_CLASSAD_VALUE_NOT_FOUND;
     }

    if (et->GetKind() == ExprTree::LITERAL_NODE)
     {
      // The attribute was probably stringified. Try to parse it.
      Value v;
      EvalState state;
      state.SetScopes( ad );

      et->Evaluate(state,v);
      std::string strres;

      if (v.IsStringValue( strres ))
       {
        ClassAdParser parser;
        et=NULL;
        parser.ParseExpression(strres,et);
        need_to_delete_et = true;
       }
     }

    BinaryOpUnwind res_unp; 
    std::string result;
    res_unp.Unparse(result, et);
    int n_results;
    if (*results == NULL)
     {
      n_results = 0;
      (*results) = (char **)malloc(sizeof(char **));
      if ((*results) == NULL) return C_CLASSAD_OUT_OF_MEMORY;
      (*results)[0] = NULL;
     }
    else
     {
      for (n_results = 0; (*results)[n_results] != NULL; n_results++) /*NOP*/ ;
     }

    std::vector<std::string>::const_iterator it;
    for (it = res_unp.m_unwind_output.begin(); 
         it != res_unp.m_unwind_output.end(); ++it)
     {
      n_results++;
      char **new_results;
      new_results = (char **)realloc(*results, (n_results+1)*sizeof(char *));
      if (new_results == NULL)
       {
        if (need_to_delete_et) delete et;
        return C_CLASSAD_OUT_OF_MEMORY;
       }
      (*results) = new_results;
      (*results)[n_results] = NULL;
      (*results)[n_results-1] = strdup(it->c_str());
      if (((*results)[n_results-1]) == NULL)
       {
        if (need_to_delete_et) delete et;
        return C_CLASSAD_OUT_OF_MEMORY;
       }
     }

    if (need_to_delete_et) delete et;
    return C_CLASSAD_NO_ERROR;
   }