void BaseResource::UpdateLeases()
{
dprintf(D_FULLDEBUG,"*** UpdateLeases called\n");
if ( hasLeases == false ) {
dprintf(D_FULLDEBUG," Leases not supported, cancelling timer\n" );
daemonCore->Cancel_Timer( updateLeasesTimerId );
updateLeasesTimerId = TIMER_UNSET;
return;
}
// Don't start a new lease update too soon after the previous one.
int delay;
delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL);
if ( delay > 0 ) {
daemonCore->Reset_Timer( updateLeasesTimerId, delay );
dprintf(D_FULLDEBUG," UpdateLeases: last update too recent, delaying %d secs\n",delay);
return;
}
daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER );
if ( updateLeasesActive == false ) {
BaseJob *curr_job;
int new_lease_duration = INT_MAX;
dprintf(D_FULLDEBUG," UpdateLeases: calc'ing new leases\n");
registeredJobs.Rewind();
while ( registeredJobs.Next( curr_job ) ) {
int job_lease_duration = m_defaultLeaseDuration;
curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, job_lease_duration );
if ( job_lease_duration > 0 && job_lease_duration < new_lease_duration ) {
new_lease_duration = job_lease_duration;
}
}
dprintf(D_FULLDEBUG," UpdateLeases: new shared lease duration: %d\n", new_lease_duration );
// This is how close to the lease expiration time we want to be
// when we try a renewal.
int renew_threshold = ( new_lease_duration * 2 / 3 ) + 10;
if ( new_lease_duration == INT_MAX ||
m_sharedLeaseExpiration > time(NULL) + renew_threshold ) {
// Lease doesn't need renewal, yet.
time_t next_renew_time = m_sharedLeaseExpiration - renew_threshold;
if ( new_lease_duration == INT_MAX ||
next_renew_time > time(NULL) + 3600 ) {
next_renew_time = time(NULL) + 3600;
}
dprintf(D_FULLDEBUG," UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL));
lastUpdateLeases = time(NULL);
daemonCore->Reset_Timer( updateLeasesTimerId,
next_renew_time - time(NULL) );
return;
} else {
// Time to renew the lease
m_sharedLeaseExpiration = time(NULL) + new_lease_duration;
if ( !m_hasSharedLeases ) {
registeredJobs.Rewind();
while ( registeredJobs.Next( curr_job ) ) {
std::string job_id;
int tmp;
if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) &&
curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, tmp )
) {
leaseUpdates.Append( curr_job );
}
}
}
dprintf(D_FULLDEBUG," new shared lease expiration at %ld, performing renewal...\n",m_sharedLeaseExpiration);
requestScheddUpdateNotification( updateLeasesTimerId );
updateLeasesActive = true;
}
}
unsigned update_delay = 0;
bool update_complete;
SimpleList<PROC_ID> update_succeeded;
bool update_success;
dprintf(D_FULLDEBUG," UpdateLeases: calling DoUpdateLeases\n");
if ( m_hasSharedLeases ) {
DoUpdateSharedLease( update_delay, update_complete, update_success );
} else {
DoUpdateLeases( update_delay, update_complete, update_succeeded );
}
if ( update_delay ) {
daemonCore->Reset_Timer( updateLeasesTimerId, update_delay );
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases wants delay of %u secs\n",update_delay);
return;
}
if ( !update_complete ) {
updateLeasesCmdActive = true;
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases in progress\n");
return;
}
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases complete, processing results\n");
bool first_update = lastUpdateLeases == 0;
updateLeasesCmdActive = false;
lastUpdateLeases = time(NULL);
if ( m_hasSharedLeases ) {
BaseJob *curr_job;
std::string tmp;
registeredJobs.Rewind();
while ( registeredJobs.Next( curr_job ) ) {
if ( first_update ) {
// New jobs may be waiting for the lease be to established
// before they proceed with submission.
curr_job->SetEvaluateState();
}
if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) {
curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration );
}
}
} else {
std::string msg = " update_succeeded:";
BaseJob *curr_job;
PROC_ID curr_id;
update_succeeded.Rewind();
while ( update_succeeded.Next( curr_id ) ) {
formatstr_cat(msg, " %d.%d", curr_id.cluster, curr_id.proc);
if ( BaseJob::JobsByProcId.lookup( curr_id, curr_job ) == 0 ) {
curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration );
}
}
dprintf(D_FULLDEBUG,"%s\n",msg.c_str());
leaseUpdates.Clear();
}
updateLeasesActive = false;
dprintf(D_FULLDEBUG," UpdateLeases: lease update complete, resetting timer for 30 secs\n");
daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY );
}
void BaseResource::UpdateLeases()
{
dprintf(D_FULLDEBUG,"*** UpdateLeases called\n");
if ( hasLeases == false ) {
dprintf(D_FULLDEBUG," Leases not supported, cancelling timer\n" );
daemonCore->Cancel_Timer( updateLeasesTimerId );
updateLeasesTimerId = TIMER_UNSET;
return;
}
// Don't start a new lease update too soon after the previous one.
int delay;
delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL);
if ( delay > 0 ) {
daemonCore->Reset_Timer( updateLeasesTimerId, delay );
dprintf(D_FULLDEBUG," UpdateLeases: last update too recent, delaying %d secs\n",delay);
return;
}
daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER );
if ( updateLeasesActive == false ) {
BaseJob *curr_job;
time_t next_renew_time = INT_MAX;
time_t job_renew_time;
int min_new_expire = INT_MAX;
dprintf(D_FULLDEBUG," UpdateLeases: calc'ing new leases\n");
registeredJobs.Rewind();
dprintf(D_FULLDEBUG," starting min_new_expire=%d next_renew_time=%ld\n",min_new_expire,next_renew_time);
while ( registeredJobs.Next( curr_job ) ) {
int new_expire;
std::string job_id;
job_renew_time = next_renew_time;
// Don't update the lease for a job that isn't submitted
// anywhere. The Job object will start the lease when it
// submits the job.
if ( ( m_hasSharedLeases || curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) &&
CalculateJobLease( curr_job->jobAd, new_expire,
m_defaultLeaseDuration,
&job_renew_time ) ) {
if ( new_expire < min_new_expire ) {
min_new_expire = new_expire;
}
if ( !m_hasSharedLeases ) {
curr_job->UpdateJobLeaseSent( new_expire );
leaseUpdates.Append( curr_job );
}
} else if ( job_renew_time < next_renew_time ) {
next_renew_time = job_renew_time;
}
dprintf(D_FULLDEBUG," after %d.%d: min_new_expire=%d next_renew_time=%ld job_renew_time=%ld\n",curr_job->procID.cluster,curr_job->procID.proc,min_new_expire,next_renew_time,job_renew_time);
}
if ( min_new_expire == INT_MAX ||
( m_hasSharedLeases && next_renew_time < INT_MAX &&
m_sharedLeaseExpiration != 0 ) ) {
if ( next_renew_time > time(NULL) + 3600 ) {
next_renew_time = time(NULL) + 3600;
}
dprintf(D_FULLDEBUG," UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL));
lastUpdateLeases = time(NULL);
daemonCore->Reset_Timer( updateLeasesTimerId,
next_renew_time - time(NULL) );
} else {
if ( m_hasSharedLeases ) {
registeredJobs.Rewind();
while ( registeredJobs.Next( curr_job ) ) {
std::string job_id;
if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) {
curr_job->UpdateJobLeaseSent( min_new_expire );
}
}
m_sharedLeaseExpiration = min_new_expire;
dprintf(D_FULLDEBUG," new shared lease expiration at %ld, updating job ads...\n",m_sharedLeaseExpiration);
}
requestScheddUpdateNotification( updateLeasesTimerId );
updateLeasesActive = true;
leaseAttrsSynched = false;
}
return;
}
if ( leaseAttrsSynched == false ) {
bool still_dirty = false;
BaseJob *curr_job;
leaseUpdates.Rewind();
while ( leaseUpdates.Next( curr_job ) ) {
bool exists, dirty;
curr_job->jobAd->GetDirtyFlag( ATTR_JOB_LEASE_EXPIRATION,
&exists, &dirty );
if ( !exists ) {
// What!? The attribute disappeared? Forget about renewing
// the lease then
dprintf( D_ALWAYS, "Lease attribute disappeared for job %d.%d, ignoring it\n",
curr_job->procID.cluster, curr_job->procID.proc );
leaseUpdates.DeleteCurrent();
}
if ( dirty ) {
still_dirty = true;
requestScheddUpdate( curr_job, false );
}
}
if ( still_dirty ) {
requestScheddUpdateNotification( updateLeasesTimerId );
dprintf(D_FULLDEBUG," UpdateLeases: waiting for schedd synch\n");
return;
}
else dprintf(D_FULLDEBUG," UpdateLeases: leases synched\n");
}
leaseAttrsSynched = true;
unsigned update_delay = 0;
bool update_complete;
SimpleList<PROC_ID> update_succeeded;
bool update_success;
dprintf(D_FULLDEBUG," UpdateLeases: calling DoUpdateLeases\n");
if ( m_hasSharedLeases ) {
DoUpdateSharedLease( update_delay, update_complete, update_success );
} else {
DoUpdateLeases( update_delay, update_complete, update_succeeded );
}
if ( update_delay ) {
daemonCore->Reset_Timer( updateLeasesTimerId, update_delay );
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases wants delay of %uld secs\n",update_delay);
return;
}
if ( !update_complete ) {
updateLeasesCmdActive = true;
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases in progress\n");
return;
}
dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases complete, processing results\n");
bool first_update = lastUpdateLeases == 0;
updateLeasesCmdActive = false;
lastUpdateLeases = time(NULL);
if ( m_hasSharedLeases ) {
BaseJob *curr_job;
std::string tmp;
registeredJobs.Rewind();
while ( registeredJobs.Next( curr_job ) ) {
if ( first_update ) {
// New jobs may be waiting for the lease be to established
// before they proceed with submission.
curr_job->SetEvaluateState();
}
if ( !curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) {
continue;
}
bool curr_renewal_failed = !update_success;
bool last_renewal_failed = false;
curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
last_renewal_failed );
if ( curr_renewal_failed != last_renewal_failed ) {
curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
curr_renewal_failed );
requestScheddUpdate( curr_job, false );
}
}
} else {
update_succeeded.Rewind();
PROC_ID id;
std::string msg = " update_succeeded:";
while(update_succeeded.Next(id)) formatstr_cat(msg, " %d.%d", id.cluster, id.proc);
dprintf(D_FULLDEBUG,"%s\n",msg.c_str());
BaseJob *curr_job;
leaseUpdates.Rewind();
while ( leaseUpdates.Next( curr_job ) ) {
bool curr_renewal_failed;
bool last_renewal_failed = false;
if ( update_succeeded.IsMember( curr_job->procID ) ) {
dprintf(D_FULLDEBUG," %d.%d is in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc);
curr_renewal_failed = false;
} else {
dprintf(D_FULLDEBUG," %d.%d is not in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc);
curr_renewal_failed = true;
}
curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
last_renewal_failed );
if ( curr_renewal_failed != last_renewal_failed ) {
curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
curr_renewal_failed );
requestScheddUpdate( curr_job, false );
}
leaseUpdates.DeleteCurrent();
}
}
updateLeasesActive = false;
dprintf(D_FULLDEBUG," UpdateLeases: lease update complete, resetting timer for 30 secs\n");
daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY );
}