Ejemplo n.º 1
0
void BaseResource::UpdateLeases()
{
dprintf(D_FULLDEBUG,"*** UpdateLeases called\n");
	if ( hasLeases == false ) {
dprintf(D_FULLDEBUG,"    Leases not supported, cancelling timer\n" );
		daemonCore->Cancel_Timer( updateLeasesTimerId );
		updateLeasesTimerId = TIMER_UNSET;
		return;
	}

	// Don't start a new lease update too soon after the previous one.
	int delay;
	delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL);
	if ( delay > 0 ) {
		daemonCore->Reset_Timer( updateLeasesTimerId, delay );
dprintf(D_FULLDEBUG,"    UpdateLeases: last update too recent, delaying %d secs\n",delay);
		return;
	}

	daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER );

    if ( updateLeasesActive == false ) {
		BaseJob *curr_job;
		int new_lease_duration = INT_MAX;
dprintf(D_FULLDEBUG,"    UpdateLeases: calc'ing new leases\n");
		registeredJobs.Rewind();
		while ( registeredJobs.Next( curr_job ) ) {
			int job_lease_duration = m_defaultLeaseDuration;
			curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, job_lease_duration );
			if ( job_lease_duration > 0 && job_lease_duration < new_lease_duration ) {
				new_lease_duration = job_lease_duration;
			}
		}
dprintf(D_FULLDEBUG,"    UpdateLeases: new shared lease duration: %d\n", new_lease_duration );
		// This is how close to the lease expiration time we want to be
		// when we try a renewal.
		int renew_threshold = ( new_lease_duration * 2 / 3 ) + 10;
		if ( new_lease_duration == INT_MAX ||
			 m_sharedLeaseExpiration > time(NULL) + renew_threshold ) {
			// Lease doesn't need renewal, yet.
			time_t next_renew_time = m_sharedLeaseExpiration - renew_threshold;
			if ( new_lease_duration == INT_MAX ||
				 next_renew_time > time(NULL) + 3600 ) {
				next_renew_time = time(NULL) + 3600;
			}
dprintf(D_FULLDEBUG,"    UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL));
			lastUpdateLeases = time(NULL);
			daemonCore->Reset_Timer( updateLeasesTimerId,
									 next_renew_time - time(NULL) );
			return;
		} else {
			// Time to renew the lease
			m_sharedLeaseExpiration = time(NULL) + new_lease_duration;
			if ( !m_hasSharedLeases ) {
				registeredJobs.Rewind();
				while ( registeredJobs.Next( curr_job ) ) {
					std::string job_id;
					int tmp;
					if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) &&
						 curr_job->jobAd->LookupInteger( ATTR_JOB_LEASE_DURATION, tmp )
					) {
						leaseUpdates.Append( curr_job );
					}
				}
			}
dprintf(D_FULLDEBUG,"    new shared lease expiration at %ld, performing renewal...\n",m_sharedLeaseExpiration);
			requestScheddUpdateNotification( updateLeasesTimerId );
			updateLeasesActive = true;
		}
	}

	unsigned update_delay = 0;
	bool update_complete;
	SimpleList<PROC_ID> update_succeeded;
	bool update_success;
dprintf(D_FULLDEBUG,"    UpdateLeases: calling DoUpdateLeases\n");
	if ( m_hasSharedLeases ) {
		DoUpdateSharedLease( update_delay, update_complete, update_success );
	} else {
		DoUpdateLeases( update_delay, update_complete, update_succeeded );
	}

	if ( update_delay ) {
		daemonCore->Reset_Timer( updateLeasesTimerId, update_delay );
dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases wants delay of %u secs\n",update_delay);
		return;
	}

	if ( !update_complete ) {
		updateLeasesCmdActive = true;
dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases in progress\n");
		return;
	}

dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases complete, processing results\n");
	bool first_update = lastUpdateLeases == 0;

	updateLeasesCmdActive = false;
	lastUpdateLeases = time(NULL);

	if ( m_hasSharedLeases ) {
		BaseJob *curr_job;
		std::string tmp;
		registeredJobs.Rewind();
		while ( registeredJobs.Next( curr_job ) ) {
			if ( first_update ) {
				// New jobs may be waiting for the lease be to established
				// before they proceed with submission.
				curr_job->SetEvaluateState();
			}
			if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) {
				curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration );
			}
		}
	} else {
std::string msg = "    update_succeeded:";
		BaseJob *curr_job;
		PROC_ID curr_id;
		update_succeeded.Rewind();
		while ( update_succeeded.Next( curr_id ) ) {
formatstr_cat(msg, " %d.%d", curr_id.cluster, curr_id.proc);
			if ( BaseJob::JobsByProcId.lookup( curr_id, curr_job ) == 0 ) {
				curr_job->UpdateJobLeaseSent( m_sharedLeaseExpiration );
			}
		}
dprintf(D_FULLDEBUG,"%s\n",msg.c_str());
		leaseUpdates.Clear();
	}

	updateLeasesActive = false;

dprintf(D_FULLDEBUG,"    UpdateLeases: lease update complete, resetting timer for 30 secs\n");
	daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY );
}
Ejemplo n.º 2
0
void BaseResource::UpdateLeases()
{
dprintf(D_FULLDEBUG,"*** UpdateLeases called\n");
	if ( hasLeases == false ) {
dprintf(D_FULLDEBUG,"    Leases not supported, cancelling timer\n" );
		daemonCore->Cancel_Timer( updateLeasesTimerId );
		updateLeasesTimerId = TIMER_UNSET;
		return;
	}

	// Don't start a new lease update too soon after the previous one.
	int delay;
	delay = (lastUpdateLeases + UPDATE_LEASE_DELAY) - time(NULL);
	if ( delay > 0 ) {
		daemonCore->Reset_Timer( updateLeasesTimerId, delay );
dprintf(D_FULLDEBUG,"    UpdateLeases: last update too recent, delaying %d secs\n",delay);
		return;
	}

	daemonCore->Reset_Timer( updateLeasesTimerId, TIMER_NEVER );

    if ( updateLeasesActive == false ) {
		BaseJob *curr_job;
		time_t next_renew_time = INT_MAX;
		time_t job_renew_time;
		int min_new_expire = INT_MAX;
dprintf(D_FULLDEBUG,"    UpdateLeases: calc'ing new leases\n");
		registeredJobs.Rewind();
dprintf(D_FULLDEBUG,"    starting min_new_expire=%d next_renew_time=%ld\n",min_new_expire,next_renew_time);
		while ( registeredJobs.Next( curr_job ) ) {
			int new_expire;
			std::string  job_id;
			job_renew_time = next_renew_time;
				// Don't update the lease for a job that isn't submitted
				// anywhere. The Job object will start the lease when it
				// submits the job.
			if ( ( m_hasSharedLeases || curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) &&
				 CalculateJobLease( curr_job->jobAd, new_expire,
									m_defaultLeaseDuration,
									&job_renew_time ) ) {

				if ( new_expire < min_new_expire ) {
					min_new_expire = new_expire;
				}
				if ( !m_hasSharedLeases ) {
					curr_job->UpdateJobLeaseSent( new_expire );
					leaseUpdates.Append( curr_job );
				}
			} else if ( job_renew_time < next_renew_time ) {
				next_renew_time = job_renew_time;
			}
dprintf(D_FULLDEBUG,"    after %d.%d: min_new_expire=%d next_renew_time=%ld job_renew_time=%ld\n",curr_job->procID.cluster,curr_job->procID.proc,min_new_expire,next_renew_time,job_renew_time);
		}
		if ( min_new_expire == INT_MAX ||
			 ( m_hasSharedLeases && next_renew_time < INT_MAX &&
			   m_sharedLeaseExpiration != 0 ) ) {
			if ( next_renew_time > time(NULL) + 3600 ) {
				next_renew_time = time(NULL) + 3600;
			}
dprintf(D_FULLDEBUG,"    UpdateLeases: nothing to renew, resetting timer for %ld secs\n",next_renew_time - time(NULL));
			lastUpdateLeases = time(NULL);
			daemonCore->Reset_Timer( updateLeasesTimerId,
									 next_renew_time - time(NULL) );
		} else {
			if ( m_hasSharedLeases ) {
				registeredJobs.Rewind();
				while ( registeredJobs.Next( curr_job ) ) {
					std::string job_id;
					if ( curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, job_id ) ) {
						curr_job->UpdateJobLeaseSent( min_new_expire );
					}
				}
				m_sharedLeaseExpiration = min_new_expire;
dprintf(D_FULLDEBUG,"    new shared lease expiration at %ld, updating job ads...\n",m_sharedLeaseExpiration);
			}
			requestScheddUpdateNotification( updateLeasesTimerId );
			updateLeasesActive = true;
			leaseAttrsSynched = false;
		}
		return;
	}

	if ( leaseAttrsSynched == false ) {
		bool still_dirty = false;
		BaseJob *curr_job;
		leaseUpdates.Rewind();
		while ( leaseUpdates.Next( curr_job ) ) {
			bool exists, dirty;
			curr_job->jobAd->GetDirtyFlag( ATTR_JOB_LEASE_EXPIRATION,
										   &exists, &dirty );
			if ( !exists ) {
					// What!? The attribute disappeared? Forget about renewing
					// the lease then
				dprintf( D_ALWAYS, "Lease attribute disappeared for job %d.%d, ignoring it\n",
						 curr_job->procID.cluster, curr_job->procID.proc );
				leaseUpdates.DeleteCurrent();
			}
			if ( dirty ) {
				still_dirty = true;
				requestScheddUpdate( curr_job, false );
			}
		}
		if ( still_dirty ) {
			requestScheddUpdateNotification( updateLeasesTimerId );
dprintf(D_FULLDEBUG,"    UpdateLeases: waiting for schedd synch\n");
			return;
		}
else dprintf(D_FULLDEBUG,"    UpdateLeases: leases synched\n");
	}

	leaseAttrsSynched = true;

	unsigned update_delay = 0;
	bool update_complete;
	SimpleList<PROC_ID> update_succeeded;
	bool update_success;
dprintf(D_FULLDEBUG,"    UpdateLeases: calling DoUpdateLeases\n");
	if ( m_hasSharedLeases ) {
		DoUpdateSharedLease( update_delay, update_complete, update_success );
	} else {
		DoUpdateLeases( update_delay, update_complete, update_succeeded );
	}

	if ( update_delay ) {
		daemonCore->Reset_Timer( updateLeasesTimerId, update_delay );
dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases wants delay of %uld secs\n",update_delay);
		return;
	}

	if ( !update_complete ) {
		updateLeasesCmdActive = true;
dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases in progress\n");
		return;
	}

dprintf(D_FULLDEBUG,"    UpdateLeases: DoUpdateLeases complete, processing results\n");
	bool first_update = lastUpdateLeases == 0;

	updateLeasesCmdActive = false;
	lastUpdateLeases = time(NULL);

	if ( m_hasSharedLeases ) {
		BaseJob *curr_job;
		std::string tmp;
		registeredJobs.Rewind();
		while ( registeredJobs.Next( curr_job ) ) {
			if ( first_update ) {
				// New jobs may be waiting for the lease be to established
				// before they proceed with submission.
				curr_job->SetEvaluateState();
			}
			if ( !curr_job->jobAd->LookupString( ATTR_GRID_JOB_ID, tmp ) ) {
				continue;
			}
			bool curr_renewal_failed = !update_success;
			bool last_renewal_failed = false;
			curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
										 last_renewal_failed );
			if ( curr_renewal_failed != last_renewal_failed ) {
				curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
										 curr_renewal_failed );
				requestScheddUpdate( curr_job, false );
			}
		}
	} else {
update_succeeded.Rewind();
PROC_ID id;
std::string msg = "    update_succeeded:";
 while(update_succeeded.Next(id)) formatstr_cat(msg, " %d.%d", id.cluster, id.proc);
dprintf(D_FULLDEBUG,"%s\n",msg.c_str());
		BaseJob *curr_job;
		leaseUpdates.Rewind();
		while ( leaseUpdates.Next( curr_job ) ) {
			bool curr_renewal_failed;
			bool last_renewal_failed = false;
			if ( update_succeeded.IsMember( curr_job->procID ) ) {
dprintf(D_FULLDEBUG,"    %d.%d is in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc);
				curr_renewal_failed = false;
			} else {
dprintf(D_FULLDEBUG,"    %d.%d is not in succeeded list\n",curr_job->procID.cluster,curr_job->procID.proc);
				curr_renewal_failed = true;
			}
			curr_job->jobAd->LookupBool( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
										 last_renewal_failed );
			if ( curr_renewal_failed != last_renewal_failed ) {
				curr_job->jobAd->Assign( ATTR_LAST_JOB_LEASE_RENEWAL_FAILED,
										 curr_renewal_failed );
				requestScheddUpdate( curr_job, false );
			}
			leaseUpdates.DeleteCurrent();
		}
	}

	updateLeasesActive = false;

dprintf(D_FULLDEBUG,"    UpdateLeases: lease update complete, resetting timer for 30 secs\n");
	daemonCore->Reset_Timer( updateLeasesTimerId, UPDATE_LEASE_DELAY );
}