void BaseResource::UpdateLeases() { dprintf(D_FULLDEBUG,"*** UpdateLeases called\n"); if ( hasLeases == false ) { dprintf(D_FULLDEBUG," Leases not supported, cancelling timer\n" ); daemonCore->Cancel_Timer( updateLeasesTimerId ); updateLeasesTimerId = TIMER_UNSET; return; } // Don't start a new lease update too soon after the previous one. int delay; delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL); if ( delay > 0 ) { daemonCore->Reset_Timer( updateLeasesTimerId, delay ); dprintf(D_FULLDEBUG," UpdateLeases: last update too recent, delaying %d secs\n",delay); return; } daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER ); if ( updateLeasesActive == false ) { BaseJob *curr_job; int new_lease_duration = INT_MAX; dprintf(D_FULLDEBUG," UpdateLeases: calc'ing new leases\n"); registeredJobs.Rewind(); while ( registeredJobs.Next( curr_job ) ) { int job_lease_duration = m_defaultLeaseDuration; curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, job_lease_duration ); if ( job_lease_duration > 0 && job_lease_duration < new_lease_duration ) { new_lease_duration = job_lease_duration; } } dprintf(D_FULLDEBUG," UpdateLeases: new shared lease duration: %d\n", new_lease_duration ); // This is how close to the lease expiration time we want to be // when we try a renewal. int renew_threshold = ( new_lease_duration * 2 / 3 ) + 10; if ( new_lease_duration == INT_MAX || m_sharedLeaseExpiration > time(NULL) + renew_threshold ) { // Lease doesn't need renewal, yet. time_t next_renew_time = m_sharedLeaseExpiration - renew_threshold; if ( new_lease_duration == INT_MAX || next_renew_time > time(NULL) + 3600 ) { next_renew_time = time(NULL) + 3600; } dprintf(D_FULLDEBUG," UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL)); lastUpdateLeases = time(NULL); daemonCore->Reset_Timer( updateLeasesTimerId, next_renew_time - time(NULL) ); return; } else { // Time to renew the lease m_sharedLeaseExpiration = time(NULL) + new_lease_duration; if ( !m_hasSharedLeases ) { registeredJobs.Rewind(); while ( registeredJobs.Next( curr_job ) ) { std::string job_id; int tmp; if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) && curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, tmp ) ) { leaseUpdates.Append( curr_job ); } } } dprintf(D_FULLDEBUG," new shared lease expiration at %ld, performing renewal...\n",m_sharedLeaseExpiration); requestScheddUpdateNotification( updateLeasesTimerId ); updateLeasesActive = true; } } unsigned update_delay = 0; bool update_complete; SimpleList<PROC_ID> update_succeeded; bool update_success; dprintf(D_FULLDEBUG," UpdateLeases: calling DoUpdateLeases\n"); if ( m_hasSharedLeases ) { DoUpdateSharedLease( update_delay, update_complete, update_success ); } else { DoUpdateLeases( update_delay, update_complete, update_succeeded ); } if ( update_delay ) { daemonCore->Reset_Timer( updateLeasesTimerId, update_delay ); dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases wants delay of %u secs\n",update_delay); return; } if ( !update_complete ) { updateLeasesCmdActive = true; dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases in progress\n"); return; } dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases complete, processing results\n"); bool first_update = lastUpdateLeases == 0; updateLeasesCmdActive = false; lastUpdateLeases = time(NULL); if ( m_hasSharedLeases ) { BaseJob *curr_job; std::string tmp; registeredJobs.Rewind(); while ( registeredJobs.Next( curr_job ) ) { if ( first_update ) { // New jobs may be waiting for the lease be to established // before they proceed with submission. curr_job->SetEvaluateState(); } if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) { curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration ); } } } else { std::string msg = " update_succeeded:"; BaseJob *curr_job; PROC_ID curr_id; update_succeeded.Rewind(); while ( update_succeeded.Next( curr_id ) ) { formatstr_cat(msg, " %d.%d", curr_id.cluster, curr_id.proc); if ( BaseJob::JobsByProcId.lookup( curr_id, curr_job ) == 0 ) { curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration ); } } dprintf(D_FULLDEBUG,"%s\n",msg.c_str()); leaseUpdates.Clear(); } updateLeasesActive = false; dprintf(D_FULLDEBUG," UpdateLeases: lease update complete, resetting timer for 30 secs\n"); daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY ); }
void BaseResource::UpdateLeases() { dprintf(D_FULLDEBUG,"*** UpdateLeases called\n"); if ( hasLeases == false ) { dprintf(D_FULLDEBUG," Leases not supported, cancelling timer\n" ); daemonCore->Cancel_Timer( updateLeasesTimerId ); updateLeasesTimerId = TIMER_UNSET; return; } // Don't start a new lease update too soon after the previous one. int delay; delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL); if ( delay > 0 ) { daemonCore->Reset_Timer( updateLeasesTimerId, delay ); dprintf(D_FULLDEBUG," UpdateLeases: last update too recent, delaying %d secs\n",delay); return; } daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER ); if ( updateLeasesActive == false ) { BaseJob *curr_job; time_t next_renew_time = INT_MAX; time_t job_renew_time; int min_new_expire = INT_MAX; dprintf(D_FULLDEBUG," UpdateLeases: calc'ing new leases\n"); registeredJobs.Rewind(); dprintf(D_FULLDEBUG," starting min_new_expire=%d next_renew_time=%ld\n",min_new_expire,next_renew_time); while ( registeredJobs.Next( curr_job ) ) { int new_expire; std::string job_id; job_renew_time = next_renew_time; // Don't update the lease for a job that isn't submitted // anywhere. The Job object will start the lease when it // submits the job. if ( ( m_hasSharedLeases || curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) && CalculateJobLease( curr_job->jobAd, new_expire, m_defaultLeaseDuration, &job_renew_time ) ) { if ( new_expire < min_new_expire ) { min_new_expire = new_expire; } if ( !m_hasSharedLeases ) { curr_job->UpdateJobLeaseSent( new_expire ); leaseUpdates.Append( curr_job ); } } else if ( job_renew_time < next_renew_time ) { next_renew_time = job_renew_time; } dprintf(D_FULLDEBUG," after %d.%d: min_new_expire=%d next_renew_time=%ld job_renew_time=%ld\n",curr_job->procID.cluster,curr_job->procID.proc,min_new_expire,next_renew_time,job_renew_time); } if ( min_new_expire == INT_MAX || ( m_hasSharedLeases && next_renew_time < INT_MAX && m_sharedLeaseExpiration != 0 ) ) { if ( next_renew_time > time(NULL) + 3600 ) { next_renew_time = time(NULL) + 3600; } dprintf(D_FULLDEBUG," UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL)); lastUpdateLeases = time(NULL); daemonCore->Reset_Timer( updateLeasesTimerId, next_renew_time - time(NULL) ); } else { if ( m_hasSharedLeases ) { registeredJobs.Rewind(); while ( registeredJobs.Next( curr_job ) ) { std::string job_id; if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) { curr_job->UpdateJobLeaseSent( min_new_expire ); } } m_sharedLeaseExpiration = min_new_expire; dprintf(D_FULLDEBUG," new shared lease expiration at %ld, updating job ads...\n",m_sharedLeaseExpiration); } requestScheddUpdateNotification( updateLeasesTimerId ); updateLeasesActive = true; leaseAttrsSynched = false; } return; } if ( leaseAttrsSynched == false ) { bool still_dirty = false; BaseJob *curr_job; leaseUpdates.Rewind(); while ( leaseUpdates.Next( curr_job ) ) { bool exists, dirty; curr_job->jobAd->GetDirtyFlag( ATTR_JOB_LEASE_EXPIRATION, &exists, &dirty ); if ( !exists ) { // What!? The attribute disappeared? Forget about renewing // the lease then dprintf( D_ALWAYS, "Lease attribute disappeared for job %d.%d, ignoring it\n", curr_job->procID.cluster, curr_job->procID.proc ); leaseUpdates.DeleteCurrent(); } if ( dirty ) { still_dirty = true; requestScheddUpdate( curr_job, false ); } } if ( still_dirty ) { requestScheddUpdateNotification( updateLeasesTimerId ); dprintf(D_FULLDEBUG," UpdateLeases: waiting for schedd synch\n"); return; } else dprintf(D_FULLDEBUG," UpdateLeases: leases synched\n"); } leaseAttrsSynched = true; unsigned update_delay = 0; bool update_complete; SimpleList<PROC_ID> update_succeeded; bool update_success; dprintf(D_FULLDEBUG," UpdateLeases: calling DoUpdateLeases\n"); if ( m_hasSharedLeases ) { DoUpdateSharedLease( update_delay, update_complete, update_success ); } else { DoUpdateLeases( update_delay, update_complete, update_succeeded ); } if ( update_delay ) { daemonCore->Reset_Timer( updateLeasesTimerId, update_delay ); dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases wants delay of %uld secs\n",update_delay); return; } if ( !update_complete ) { updateLeasesCmdActive = true; dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases in progress\n"); return; } dprintf(D_FULLDEBUG," UpdateLeases: DoUpdateLeases complete, processing results\n"); bool first_update = lastUpdateLeases == 0; updateLeasesCmdActive = false; lastUpdateLeases = time(NULL); if ( m_hasSharedLeases ) { BaseJob *curr_job; std::string tmp; registeredJobs.Rewind(); while ( registeredJobs.Next( curr_job ) ) { if ( first_update ) { // New jobs may be waiting for the lease be to established // before they proceed with submission. curr_job->SetEvaluateState(); } if ( !curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) { continue; } bool curr_renewal_failed = !update_success; bool last_renewal_failed = false; curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED, last_renewal_failed ); if ( curr_renewal_failed != last_renewal_failed ) { curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED, curr_renewal_failed ); requestScheddUpdate( curr_job, false ); } } } else { update_succeeded.Rewind(); PROC_ID id; std::string msg = " update_succeeded:"; while(update_succeeded.Next(id)) formatstr_cat(msg, " %d.%d", id.cluster, id.proc); dprintf(D_FULLDEBUG,"%s\n",msg.c_str()); BaseJob *curr_job; leaseUpdates.Rewind(); while ( leaseUpdates.Next( curr_job ) ) { bool curr_renewal_failed; bool last_renewal_failed = false; if ( update_succeeded.IsMember( curr_job->procID ) ) { dprintf(D_FULLDEBUG," %d.%d is in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc); curr_renewal_failed = false; } else { dprintf(D_FULLDEBUG," %d.%d is not in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc); curr_renewal_failed = true; } curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED, last_renewal_failed ); if ( curr_renewal_failed != last_renewal_failed ) { curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED, curr_renewal_failed ); requestScheddUpdate( curr_job, false ); } leaseUpdates.DeleteCurrent(); } } updateLeasesActive = false; dprintf(D_FULLDEBUG," UpdateLeases: lease update complete, resetting timer for 30 secs\n"); daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY ); }