// FinalizeCompletedJobs
//------------------------------------------------------------------------------
void Server::FinalizeCompletedJobs()
{
PROFILE_FUNCTION
JobQueueRemote & jcr = JobQueueRemote::Get();
while ( Job * job = jcr.GetCompletedJob() )
{
// get associated connection
ClientState * cs = (ClientState *)job->GetUserData();
MutexHolder mh( m_ClientListMutex );
bool connectionStillActive = ( m_ClientList.Find( cs ) != nullptr );
if ( connectionStillActive )
{
Node::State result = job->GetNode()->GetState();
ASSERT( ( result == Node::UP_TO_DATE ) || ( result == Node::FAILED ) );
MemoryStream ms;
ms.Write( job->GetJobId() );
ms.Write( job->GetNode()->GetName() );
ms.Write( result == Node::UP_TO_DATE );
ms.Write( job->GetSystemErrorCount() > 0 );
ms.Write( job->GetMessages() );
ms.Write( job->GetNode()->GetLastBuildTime() );
// write the data - build result for success, or output+errors for failure
ms.Write( (uint32_t)job->GetDataSize() );
ms.WriteBuffer( job->GetData(), job->GetDataSize() );
MutexHolder mh2( cs->m_Mutex );
ASSERT( cs->m_NumJobsActive );
cs->m_NumJobsActive--;
Protocol::MsgJobResult msg;
msg.Send( cs->m_Connection, ms );
}
else
{
// we might get here without finding the connection
// (if the connection was lost before we completed)
}
FDELETE job;
}
}
// SendServerStatus
//------------------------------------------------------------------------------
void Server::SendServerStatus()
{
PROFILE_FUNCTION
MutexHolder mh( m_ClientListMutex );
const ClientState * const * end = m_ClientList.End();
for ( ClientState ** it = m_ClientList.Begin(); it != end; ++it )
{
ClientState * cs = *it;
MutexHolder mh2( cs->m_Mutex );
if ( cs->m_StatusTimer.GetElapsedMS() < Protocol::SERVER_STATUS_FREQUENCY_MS )
{
continue;
}
cs->m_StatusTimer.Start();
Protocol::MsgServerStatus msg;
msg.Send( cs->m_Connection );
}
}
// CheckWaitingJobs
//------------------------------------------------------------------------------
void Server::CheckWaitingJobs( const ToolManifest * manifest )
{
// queue for start any jobs that may now be ready
#ifdef ASSERTS_ENABLED
bool atLeastOneJobStarted = false;
#endif
MutexHolder mhC( m_ClientListMutex );
const ClientState * const * end = m_ClientList.End();
for ( ClientState ** it = m_ClientList.Begin(); it!=end; ++it )
{
// For each connected client...
ClientState * cs = *it;
MutexHolder mh2( cs->m_Mutex );
// .. check all jobs waiting for ToolManifests
int32_t numJobs = (int32_t)cs->m_WaitingJobs.GetSize();
for ( int32_t i=( numJobs -1 ); i >= 0; --i )
{
Job * job = cs->m_WaitingJobs[ i ];
ToolManifest * manifestForThisJob = job->GetToolManifest();
ASSERT( manifestForThisJob );
if ( manifestForThisJob == manifest )
{
cs->m_WaitingJobs.EraseIndex( i );
JobQueueRemote::Get().QueueJob( job );
PROTOCOL_DEBUG( "Server: Job %x can now be started\n", job );
#ifdef ASSERTS_ENABLED
atLeastOneJobStarted = true;
#endif
}
}
}
// We should only have called this function when a ToolChain sync was complete
// so at least 1 job should have been waiting for it
ASSERT( atLeastOneJobStarted );
}
// FindNeedyClients
//------------------------------------------------------------------------------
void Server::FindNeedyClients()
{
if ( m_ShouldExit )
{
return;
}
PROFILE_FUNCTION
MutexHolder mh( m_ClientListMutex );
// determine job availability
int availableJobs = (int)WorkerThreadRemote::GetNumCPUsToUse();
if ( availableJobs == 0 )
{
return;
}
++availableJobs; // over request to parallelize building/network transfers
ClientState ** iter = m_ClientList.Begin();
const ClientState * const * end = m_ClientList.End();
for ( ; iter != end; ++iter )
{
ClientState * cs = *iter;
MutexHolder mh2( cs->m_Mutex );
// any jobs requested or in progress reduce the available count
int reservedJobs = cs->m_NumJobsRequested +
cs->m_NumJobsActive;
availableJobs -= reservedJobs;
if ( availableJobs <= 0 )
{
return;
}
}
// we have some jobs available
// sort clients to find neediest first
m_ClientList.SortDeref();
Protocol::MsgRequestJob msg;
while ( availableJobs > 0 )
{
bool anyJobsRequested = false;
iter = m_ClientList.Begin();
for ( ; iter != end; ++iter )
{
ClientState * cs = *iter;
MutexHolder mh2( cs->m_Mutex );
size_t reservedJobs = cs->m_NumJobsRequested;
if ( reservedJobs >= cs->m_NumJobsAvailable )
{
continue; // we've maxed out the requests to this worker
}
// request job from this client
msg.Send( cs->m_Connection );
cs->m_NumJobsRequested++;
availableJobs--;
anyJobsRequested = true;
}
// if we did a pass and couldn't request any more jobs, then bail out
if ( anyJobsRequested == false )
{
break;
}
}
}
//------------------------------------------------------------------------------
/*virtual*/ void Server::OnDisconnected( const ConnectionInfo * connection )
{
ASSERT( connection );
ClientState * cs = (ClientState *)connection->GetUserData();
ASSERT( cs );
// Unhook any jobs which are queued or in progress for this client
// - deletes the queued jobs
// - unhooks the UserData for in-progress jobs so the result is discarded on completion
JobQueueRemote & jqr = JobQueueRemote::Get();
jqr.CancelJobsWithUserData( cs );
// check if any tool chain was being sync'd from this Client
Array< ToolManifest * > cancelledManifests( 0, true );
{
MutexHolder manifestMH( m_ToolManifestsMutex );
const ToolManifest * const * end = m_Tools.End();
ToolManifest ** it = m_Tools.Begin();
while ( it != end )
{
// if synchronizing from connection that was just disconnected...
ToolManifest * tm = *it;
if ( ( tm->IsSynchronized() == false ) &&
( tm->GetUserData() == connection ) )
{
// ...flag any expected files as not synching
tm->CancelSynchronizingFiles();
tm->SetUserData( nullptr );
cancelledManifests.Append( tm );
}
++it;
}
}
// free the serverstate structure
MutexHolder mh( m_ClientListMutex );
ClientState ** iter = m_ClientList.Find( cs );
ASSERT( iter );
m_ClientList.Erase( iter );
// because we cancelled manifest syncrhonization, we need to check if other
// connections are waiting for the same manifest
{
ClientState ** it = m_ClientList.Begin();
const ClientState * const * end = m_ClientList.End();
for ( ; it != end; ++it )
{
ClientState * otherCS = *it;
MutexHolder mh2( otherCS->m_Mutex );
const Job * const * jEnd = otherCS->m_WaitingJobs.End();
for ( Job ** jIt = otherCS->m_WaitingJobs.Begin(); jIt != jEnd; ++jIt )
{
Job * j = *jIt;
ToolManifest * jMan = j->GetToolManifest();
if ( cancelledManifests.Find( jMan ) )
{
RequestMissingFiles( otherCS->m_Connection, jMan );
}
}
}
}
// This is usually null here, but might need to be freed if
// we had the connection drop between message and payload
FREE( (void *)( cs->m_CurrentMessage ) );
// delete any jobs where we were waiting on Tool synchronization
const Job * const * end = cs->m_WaitingJobs.End();
for ( Job ** it=cs->m_WaitingJobs.Begin(); it!=end; ++it )
{
delete *it;
}
FDELETE cs;
}
// LookForWorkers
//------------------------------------------------------------------------------
void Client::LookForWorkers()
{
PROFILE_FUNCTION
MutexHolder mh( m_ServerListMutex );
const size_t numWorkers( m_ServerList.GetSize() );
// find out how many connections we have now
size_t numConnections = 0;
for ( size_t i=0; i<numWorkers; i++ )
{
if ( m_ServerList[ i ].m_Connection )
{
numConnections++;
}
}
// limit maximum concurrent connections
if ( numConnections >= CONNECTION_LIMIT )
{
return;
}
// if we're connected to every possible worker already
if ( numConnections == numWorkers )
{
return;
}
// randomize the start index to better distribute workers when there
// are many workers/clients - otherwise all clients will attempt to connect
// to the first CONNECTION_LIMIT workers
Random r;
size_t startIndex = r.GetRandIndex( (uint32_t)numWorkers );
// find someone to connect to
for ( size_t j=0; j<numWorkers; j++ )
{
const size_t i( ( j + startIndex ) % numWorkers );
ServerState & ss = m_ServerList[ i ];
if ( ss.m_Connection )
{
continue;
}
// ignore blacklisted workers
if ( ss.m_Blacklisted )
{
continue;
}
// lock the server state
MutexHolder mhSS( ss.m_Mutex );
ASSERT( ss.m_Jobs.IsEmpty() );
if ( ss.m_DelayTimer.GetElapsed() < CONNECTION_REATTEMPT_DELAY_TIME )
{
continue;
}
const ConnectionInfo * ci = Connect( m_WorkerList[ i ], Protocol::PROTOCOL_PORT );
if ( ci == nullptr )
{
ss.m_DelayTimer.Start(); // reset connection attempt delay
}
else
{
const uint32_t numJobsAvailable( JobQueue::IsValid() ? (uint32_t)JobQueue::Get().GetNumDistributableJobsAvailable() : 0 );
ci->SetUserData( &ss );
ss.m_Connection = ci; // success!
ss.m_NumJobsAvailable = numJobsAvailable;
ss.m_StatusTimer.Start();
// send connection msg
Protocol::MsgConnection msg( numJobsAvailable );
MutexHolder mh2( ss.m_Mutex );
msg.Send( ci );
}
// limit to one connection attempt per iteration
return;
}
}
bool palMinHeapTest() {
palMinHeap<int> mh;
mh.SetAllocator(g_DefaultHeapAllocator);
mh.Insert(3);
mh.Insert(999912);
mh.Insert(66);
mh.Insert(-3);
mh.Insert(27);
mh.Insert(0);
mh.Insert(1);
mh.Insert(1000000);
int correct_order[] = {-3, 0, 1, 3, 27, 66, 999912, 1000000};
int answer_index = 0;
while (mh.IsEmpty() == false) {
int answer = mh.FindMin();
if (correct_order[answer_index] != answer) {
palBreakHere();
}
mh.DeleteMin();
answer_index++;
}
mh.Insert(3);
mh.Insert(999912);
mh.Insert(66);
mh.Insert(-3);
mh.Insert(27);
mh.Insert(0);
mh.Insert(1);
mh.Insert(1000000);
palMinHeap<int> mh2(mh);
answer_index = 0;
while (mh2.IsEmpty() == false) {
int answer = mh2.FindMin();
if (correct_order[answer_index] != answer) {
palBreakHere();
}
mh2.DeleteMin();
answer_index++;
}
palMinHeap<int> mh3;
mh3.SetAllocator(g_DefaultHeapAllocator);
mh3 = mh;
answer_index = 0;
while (mh3.IsEmpty() == false) {
int answer = mh3.FindMin();
if (correct_order[answer_index] != answer) {
palBreakHere();
}
mh3.DeleteMin();
answer_index++;
}
answer_index = 0;
while (mh.IsEmpty() == false) {
int answer = mh.FindMin();
if (correct_order[answer_index] != answer) {
palBreakHere();
}
mh.DeleteMin();
answer_index++;
}
return true;
}
