void DrDynamicBroadcastManager::RegisterVertex(DrVertexPtr vertex, bool splitting)
{
DrAssert(!splitting);
if (m_baseTee == DrNull)
{
m_baseTee = dynamic_cast<DrTeeVertexPtr>(vertex);
DrAssert(m_baseTee != DrNull);
}
}
void DrCohortProcess::NotifyVertexCompletion()
{
DrLogI("Enter with m_numberOfVerticesLeftToComplete %d", m_numberOfVerticesLeftToComplete);
DrAssert(m_numberOfVerticesLeftToComplete > 0);
--m_numberOfVerticesLeftToComplete;
if (m_numberOfVerticesLeftToComplete > 0)
{
DrLogI("Still have %d vertices to complete", m_numberOfVerticesLeftToComplete);
return;
}
bool scheduleTermination = false;
if (m_process.IsEmpty() == false)
{
DrLockBoxKey<DrProcess> process(m_process);
if (process->GetInfo()->m_state->m_state <= DPS_Running)
{
/* the process hasn't exited yet even though all the vertices are done. It will
probably exit soon of its own accord, but in case it doesn't we'll schedule
a message that will terminate it after a while */
DrLogI("Process %s has not exited, scheduling termination", process->GetName().GetChars());
scheduleTermination = true;
}
else
{
/* The process completed and we're done with it, call Terminate to clean up */
DrLogI("Process %s completed, calling Terminate to clean up", process->GetName().GetChars());
DrAssert(process->GetInfo()->m_state->m_process != DrNull);
process->Terminate();
}
}
if (scheduleTermination)
{
/* The listener for the state message (DrProcess::ReceiveMessage(DrProcessState message))
will call Terminate to clean up in response to this message. */
DrPStateMessageRef message = DrNew DrPStateMessage(m_process, DPS_Failed);
m_messagePump->EnQueueDelayed(m_timeout, message);
}
DrLogI("Notifying cohort of vertex completion");
m_parent->NotifyProcessComplete(m_version);
DrLogI("Discarding cohort process");
Discard();
}
void DrVertexProcessStatus::SetOutputChannelCount(int nOutputChannels)
{
DrAssert(nOutputChannels >= 0);
m_outputChannel = DrNew DrOutputChannelArray(nOutputChannels);
int i;
for (i=0; i<nOutputChannels; ++i)
{
m_outputChannel[i] = DrNew DrOutputChannelDescription();
}
m_nextOutputChannelToRead = 0;
}
void DrOutputStreamManager::AddDynamicSplitVertex(DrOutputVertexPtr newVertex)
{
if (m_startedSplitting == false)
{
DrAssert(m_vertices->Size() == 1);
SetNumberOfParts(0);
m_startedSplitting = true;
}
m_vertices->Add(newVertex);
}
void DrChannelDescription::CopyFrom(DrChannelDescriptionPtr src, bool includeLengths)
{
DrAssert(m_isInputChannel == src->m_isInputChannel);
SetChannelURI(src->GetChannelURI());
SetChannelState(src->GetChannelState());
SetChannelMetaData(src->GetChannelMetaData());
if (includeLengths)
{
SetChannelProcessedLength(src->GetChannelProcessedLength());
SetChannelTotalLength(src->GetChannelTotalLength());
}
}
void DrDynamicBroadcastManager::NotifyUpstreamVertexCompleted(DrActiveVertexPtr vertex, int outputPort,
int /* unused executionVersion */,
DrResourcePtr machine,
DrVertexExecutionStatisticsPtr statistics)
// a node upstream of a tee has terminated, expand the tee into a tree
{
DrEdge oe = vertex->GetOutputs()->GetEdge(outputPort);
DrTeeVertexPtr sourceTee = dynamic_cast<DrTeeVertexPtr>((DrVertexPtr) oe.m_remoteVertex);
DrAssert(sourceTee != DrNull);
UINT64 dataWritten = statistics->m_outputData[0]->m_dataWritten;
ExpandTee(sourceTee, dataWritten, machine);
}
void DrDynamicBroadcastManager::MaybeMakeRoundRobinPodMachines()
// reorder the machines and return a list
{
if (m_roundRobinMachines == DrNull)
{
m_roundRobinMachines = DrNew DrResourceList();
/* we have to look down a long chain to find who's in the cluster, but it's there somewhere... */
DrUniversePtr universe = m_copyVertex->GetStageManager()->GetGraph()->GetCluster()->GetUniverse();
{
DrAutoCriticalSection acs(universe->GetResourceLock());
DrResourceListRef pods = universe->GetResources(DRL_Rack);
DrIntArrayListRef podIndex = DrNew DrIntArrayList();
int i;
for (i=0; i<pods->Size(); ++i)
{
podIndex->Add(0);
}
int podsToFinish = pods->Size();
DrAssert(podsToFinish > 0);
int currentPod = 0;
do
{
int currentIndex = podIndex[currentPod];
if (currentIndex == -1)
{
/* we have already previously exhausted this pod, so just keep going */
}
else
{
DrResourceListRef podChildren = pods[currentPod]->GetChildren();
if (currentIndex == podChildren->Size())
{
/* we have used all the machines from this pod */
DrAssert(podsToFinish > 0);
--podsToFinish;
}
else
{
m_roundRobinMachines->Add(podChildren[currentIndex]);
podIndex[currentPod] = currentIndex+1;
}
}
++currentPod;
if (currentPod == pods->Size())
{
currentPod = 0;
}
}
while (podsToFinish > 0);
DrAssert(m_roundRobinMachines->Size() == universe->GetResources(DRL_Computer)->Size());
}
}
}
void DrDynamicBroadcastManager::ExpandTee(DrTeeVertexPtr sourceTee, UINT64 dataWritten, DrResourcePtr machine)
{
// how many nodes to expand this stage to
int destinations = sourceTee->GetOutputs()->GetNumberOfEdges();
if (destinations < s_minConsumers)
{
return;
}
int copies = (int)(sqrt((double)destinations));
// find the pods lazily
MaybeMakeRoundRobinPodMachines();
int machines = m_roundRobinMachines->Size();
DrAssert(machines > 0);
// If there is only one machine don't ExpandTee
if (machines == 1)
{
return;
}
if (copies > machines)
{
copies = machines;
}
int currentMachine;
for (currentMachine=0; currentMachine<machines; ++currentMachine)
{
if (m_roundRobinMachines[currentMachine] == machine)
{
/* this is the machine the upstream vertex just ran on */
break;
}
}
DrAssert(currentMachine < machines);
DrLogI("Inserting dynamic broadcast tree source: %d size: %d\n", sourceTee->GetId(), copies);
int edgesPerNode = destinations / copies;
int nodesWithExtraDestination = destinations % copies;
DrTeeVertexRef newTee;
DrVertexListRef newVertices = DrNew DrVertexList();
int currentDestination = 0;
int copy;
// insert 'copies' broadcast nodes
for (copy=0; copy<copies; ++copy)
{
DrVertexPtr downstream = DrNull;
DrVertexRef t = m_baseTee->DrVertex::MakeCopy(m_teeNumber);
DrTeeVertexPtr tee = dynamic_cast<DrTeeVertexPtr>((DrVertexPtr) t);
tee->GetStageManager()->RegisterVertex(tee);
tee->GetInputs()->SetNumberOfEdges(1);
int edges = edgesPerNode + (copy < nodesWithExtraDestination);
tee->GetOutputs()->SetNumberOfEdges(edges);
if (copy == 0)
{
/* the first 'copy' is just another tee without a copier, since the data is already on this machine */
downstream = tee;
newTee = tee;
}
else
{
DrVertexRef v = m_copyVertex->DrVertex::MakeCopy(m_teeNumber);
DrActiveVertexPtr newVertex = dynamic_cast<DrActiveVertexPtr>((DrVertexPtr) v);
DrAssert(newVertex != DrNull);
newVertex->GetStageManager()->RegisterVertex(newVertex);
newVertex->GetInputs()->SetNumberOfEdges(1);
newVertex->GetOutputs()->SetNumberOfEdges(1);
/* make it prefer the new machine more than the one where the data lives */
newVertex->GetAffinity()->AddLocality(m_roundRobinMachines[currentMachine]);
newVertex->GetAffinity()->SetWeight(10 * dataWritten);
newVertex->ConnectOutput(0, tee, 0, DCT_File);
downstream = newVertex;
}
newVertices->Add(downstream);
int i;
for (i=0; i<edges; ++i, ++currentDestination)
{
DrAssert(currentDestination < destinations);
DrEdge e = sourceTee->GetOutputs()->GetEdge(currentDestination);
sourceTee->DisconnectOutput(currentDestination, true);
tee->ConnectOutput(i, e.m_remoteVertex, e.m_remotePort, DCT_File);
}
sourceTee->ConnectOutput(copy, downstream, 0, DCT_File);
++m_teeNumber;
}
DrAssert(currentDestination == destinations);
sourceTee->GetOutputs()->Compact(DrNull);
/* kick all the copy vertices to start them going */
int kick;
for (kick=0; kick<newVertices->Size(); ++kick)
{
newVertices[kick]->InitializeForGraphExecution();
newVertices[kick]->KickStateMachine();
}
/* now recurse down with the new tee we just created. Since there are a logarithmic number of levels,
I'm not worried about exhausting the stack unless somebody builds a *really* big cluster */
ExpandTee(newTee, dataWritten, machine);
}
void DrVertexProcessStatus::SetMaxOpenOutputChannelCount(int channelCount)
{
DrAssert(channelCount >= 0);
m_maxOutputChannels = channelCount;
}
void DrCohortProcess::ReceiveMessage(DrProcessInfoRef message)
{
if (m_receivedProcess == false)
{
/* when the CohortStarter helper below actually scheduled the process, it
wasn't holding our lock. So the first thing it does is send us a
message containing our DrProcess. Because of the ordering guarantees
of the message queue, that is the first message we will receive. There
are two cases: either it sent the process to be scheduled, in which
case we will hear from a subsequent message whether it succeeded or not,
or there was an unsatisfiable hard constraint in which case we hear with
an error right now. */
m_receivedProcess = true;
/* this can only be the fake message from the Cohort Starter */
DrAssert(message->m_state->m_state == DPS_NotStarted);
if (message->m_process.IsNull())
{
/* there was an error scheduling */
DrAssert(message->m_state->m_status != DrNull);
DrString msg = DrError::ToShortText(message->m_state->m_status);
DrLogI("Cohort %s v.%d got scheduling error message on cohort startup %s",
m_parent->GetDescription().GetChars(), m_version, msg.GetChars());
m_parent->GetGang()->CancelVersion(m_version, message->m_state->m_status);
}
else
{
DrAssert(message->m_state->m_status == DrNull);
m_process = message->m_process;
DrLogI("Cohort %s v.%d got startup message",
m_parent->GetDescription().GetChars(), m_version);
}
return;
}
if (m_process.IsNull())
{
/* we have already finished so do nothing */
return;
}
DrLogI("Cohort %s v.%d got message state %d",
m_parent->GetDescription().GetChars(), m_version,
message->m_state->m_state);
DrProcessStateRecordPtr state = message->m_state;
if (state->m_state > DPS_Running)
{
/* in the normal course of affairs, we should have already seen the process
start running, in which case the vertices have all initiated their own
message sends to the process which will also return informing them that
it has finished, at which point we will be notified cleanly via NotifyVertexCompletion.
The DrProcess machinery is supposed to have delayed the message we are now
receiving in order to give the vertex messages a chance to arrive. So if we
ever get here, something has gone wrong: either the process never started or
the vertex messages didn't get sent.
We are going to call Cancel below on the gang, which will result in all our vertices
calling NotifyVertexCompletion and eventually us cleaning up once they all report.
*/
DrErrorRef error;
if (state->m_state == DPS_Completed)
{
DrString reason;
if (m_processHandle == DrNull)
{
if (state->m_status == DrNull)
{
reason.SetF("Process completed with no error without starting");
error = DrNew DrError(DrError_VertexError, "DrCohortProcess", reason);
}
else
{
reason.SetF("Process completed with code %s without starting",
DRERRORSTRING(state->m_status->m_code));
error = DrNew DrError(state->m_status->m_code, "DrCohortProcess", reason);
error->AddProvenance(state->m_status);
}
}
else
{
if (state->m_status == DrNull)
{
reason.SetF("Process completed with no error but vertex message was never delivered");
error = DrNew DrError(DrError_VertexError, "DrCohortProcess", reason);
}
else
{
reason.SetF("Process completed with code %s but vertex message was never delivered",
DRERRORSTRING(state->m_status->m_code));
error = DrNew DrError(state->m_status->m_code, "DrCohortProcess", reason);
error->AddProvenance(state->m_status);
}
}
}
else
{
if (state->m_status == DrNull)
{
DrLogW("Empty status delivered with process info state %u", state->m_state);
DrString reason;
reason.SetF("Empty status with failed process state %u", state->m_state);
error = DrNew DrError(DrError_Unexpected, "DrCohortProcess", reason);
}
else
{
DrString reason;
reason.SetF("%s process code %s",
(state->m_state == DPS_Failed) ? "Failed" : "Zombie",
DRERRORSTRING(state->m_status->m_code));
error = DrNew DrError(state->m_status->m_code, "DrCohortProcess", reason);
error->AddProvenance(state->m_status);
}
}
DrString eString = DrError::ToShortText(error);
DrLogI("Cohort %s v.%d cancelling gang %s %s",
m_parent->GetDescription().GetChars(), m_version, eString.GetChars(),
error->m_explanation.GetChars());
m_parent->GetGang()->CancelVersion(m_version, error);
}
else if (m_processHandle == DrNull && state->m_state == DPS_Running)
{
/* the process has started so tell everyone about it */
m_processHandle = state->m_process;
DrAssert(m_processHandle != DrNull);
DrLogI("Cohort %s v.%d starting process",
m_parent->GetDescription().GetChars(), m_version);
m_parent->NotifyProcessHasStarted(m_version);
}
}
