//
// Process a set status request
//
void XComputeSetStatusOverlapped::ProcessIO(DrError retval, UInt32 numBytes)
{
//
// If there was an error, log and fail
//
if (retval != DrError_OK)
{
DrLogA( "Completion port returned error. error %s %u bytes", DRERRORSTRING(retval), numBytes);
}
//
// If there is no data, log and fail
//
if (numBytes != 0)
{
DrLogA( "Completion port returned non-zero. %u bytes", numBytes);
}
//
// Handle other outcomes based on the operation state
//
m_request->Process(m_operationState);
delete this;
}
//
// Handle response from xcompute
//
void DVertexXComputeSetStatus::Process(DrError err)
{
//
// If status successfully sent, log success and check on vertex status
//
if (err == DrError_OK)
{
DrLogI( "PN send succeeded. label %s", m_label.GetString());
if (m_exitOnCompletion != DrExitCode_StillActive)
{
//
// If vertex is not still active, report that it is exiting
// this may kill this process if all verticies are complete
//
m_parentOuter->VertexExiting(m_exitOnCompletion);
}
return;
}
//
// If there was a communication failure, retry up to 4 times
//
if (err == DrError_RemoteDisconnected ||
err == DrError_LocalDisconnected ||
err == DrError_ConnectionFailed)
{
if (m_sendCount < 4)
{
DrLogW( "Retrying PN send. error %s", DRERRORSTRING(err));
m_parent->SendSetStatusRequest(this);
return;
}
}
//
// If m_isAssert, just report warning, otherwise log and fail
// todo: this seems backwards. I don't understand how m_isAssert is set.
//
if (m_isAssert)
{
DrLogW(
"Send to PN failed: not asserting again. done %u sends, error %s",
m_sendCount, DRERRORSTRING(err));
}
else
{
DrLogA(
"Send to PN failed. done %u sends, error %s",
m_sendCount, DRERRORSTRING(err));
}
}
bool DoInitialize()
{
if (m_valid) {
// We don't want perf counter files
Counters::SetInitNoPerfFiles();
}
// Register ourselves as the singleton config manager
if (!m_valid || !Configuration::PreInitialize(this)) {
DrLogA( "DryadConfigurationManager",
"Failed to preinitialize dryad configuration manager");
m_valid = false;
}
if (m_valid) {
if (!CommonInit(m_strIniFileName.GetString(), 0, -1 )) {
DrLogA( "DryadConfigurationManager",
"Failed to initialize dryad configuration manager");
m_valid = false;
}
}
return m_valid;
}
unsigned __stdcall WorkQueue::ThreadFunc(void* arg)
{
WorkQueue* self = (WorkQueue *) arg;
LogAssert(self->m_completionPort != INVALID_HANDLE_VALUE);
DrLogI("WorkQueue::ThreadFunc starting thread");
bool finished = false;
do
{
DWORD numBytes;
ULONG_PTR completionKey;
LPOVERLAPPED overlapped;
// DrLogD(
// "WorkQueue::ThreadFunc waiting for completion event");
BOOL retval = ::GetQueuedCompletionStatus(self->m_completionPort,
&numBytes,
&completionKey,
&overlapped,
INFINITE);
// DrLogD(
// "WorkQueue::ThreadFunc received completion event",
// "retval: %d", retval);
if (retval != 0)
{
finished = (numBytes == DWORKQUEUE_EXIT);
if (finished)
{
DrLogI("WorkQueue::ThreadFunc received shutdown event");
}
}
else
{
DWORD errCode = GetLastError();
DrLogA("WorkQueue::GetQueuedCompletionStatus. error code: 0x%08x", HRESULT_FROM_WIN32(errCode));
}
bool queueDrained = false;
bool decrementedCount = false;
do
{
WorkRequest* request = NULL;
{
AutoCriticalSection acs(&(self->m_baseCS));
if (!decrementedCount)
{
decrementedCount = true;
LogAssert(self->m_numQueuedWakeUps > 0);
--(self->m_numQueuedWakeUps);
// DrLogD(
// "WorkQueue::ThreadFunc decremented queued wakeups",
// "new val: %d", self->m_numQueuedWakeUps);
}
if (self->m_list.IsEmpty())
{
queueDrained = true;
// DrLogD(
// "WorkQueue::ThreadFunc found empty work queue");
}
else
{
request = self->m_list.CastOut(self->m_list.RemoveHead());
LogAssert(request != NULL);
// DrLogD(
// "WorkQueue::ThreadFunc removed work item to process");
}
}
if (!queueDrained)
{
request->Process();
delete request;
request = NULL;
}
} while (!queueDrained);
} while (!finished);
DrLogI("WorkQueue::ThreadFunc exiting thread");
return 0;
}
//
// Put a work item in the queue
//
bool WorkQueue::EnQueue(WorkRequest* item)
{
LogAssert(item != NULL);
//
// Enter a critical section to add work item to queue and notify any waiting worker threads
//
{
AutoCriticalSection acs (&m_baseCS);
if (m_state == WQS_Stopping)
{
//
// If stopping, log rejection
//
DrLogI("WorkQueue::EnQueue rejecting stopping item");
return false;
}
else
{
LogAssert(m_state == WQS_Running);
}
//
// If item shouldn't be aborted, add it to list of work and make sure
// worker threads are awake
//
if (!item->ShouldAbort())
{
m_list.InsertAsTail(m_list.CastIn(item));
item = NULL;
if (m_numQueuedWakeUps < m_numWorkerThreads)
{
//
// If additional worker threads are availble, post queued work
//
++m_numQueuedWakeUps;
BOOL retval = ::PostQueuedCompletionStatus(m_completionPort,
DWORKQUEUE_CONTINUE,
NULL,
NULL);
if (retval == 0)
{
//
// Log any failure posting queued work item
//
DWORD errCode = GetLastError();
DrLogA("WorkQueue::EnQueue post completion status. error code:0x%08x", HRESULT_FROM_WIN32(errCode));
}
}
}
}
//
// If item is non-null, then ShouldAbort returned true above.
// In this case, log, abort, and clean up
//
if (item != NULL)
{
DrLogD("WorkQueue::EnQueue processing aborting work item");
item->Process();
delete item;
}
return true;
}
void WorkQueue::Stop()
{
DrLogI("WorkQueue::Stop entered");
{
AutoCriticalSection acs(&m_baseCS);
LogAssert(m_state == WQS_Running);
m_state = WQS_Stopping;
m_numQueuedWakeUps += m_numWorkerThreads;
}
DWORD i;
for (i=0; i<m_numWorkerThreads; ++i)
{
BOOL retval = ::PostQueuedCompletionStatus(m_completionPort,
DWORKQUEUE_EXIT,
NULL,
NULL);
if (retval == 0)
{
DWORD errCode = GetLastError();
DrLogA("WorkQueue::Stop post completion status. error code: 0x%08x", HRESULT_FROM_WIN32(errCode));
}
}
DrLogI("WorkQueue::Stop sent completion events");
DWORD waitRet = ::WaitForMultipleObjects(m_numWorkerThreads,
m_threadHandle,
TRUE,
INFINITE);
LogAssert(/*waitRet >= WAIT_OBJECT_0 &&*/
waitRet < (WAIT_OBJECT_0 + m_numWorkerThreads));
DrLogI("WorkQueue::Stop all threads have terminated");
{
AutoCriticalSection acs(&m_baseCS);
BOOL bRetval;
LogAssert(m_numQueuedWakeUps == 0);
LogAssert(m_list.IsEmpty());
for (i=0; i<m_numWorkerThreads; ++i)
{
bRetval = ::CloseHandle(m_threadHandle[i]);
if (bRetval == 0)
{
DWORD errCode = GetLastError();
DrLogA("WorkQueue::Stop close thread handle. error code: 0x%08x", HRESULT_FROM_WIN32(errCode));
}
m_threadHandle[i] = INVALID_HANDLE_VALUE;
}
bRetval = ::CloseHandle(m_completionPort);
if (bRetval == 0)
{
DWORD errCode = GetLastError();
DrLogA("WorkQueue::Stop close completion port handle. error code: 0x%08x", HRESULT_FROM_WIN32(errCode));
}
m_completionPort = INVALID_HANDLE_VALUE;
m_state = WQS_Stopped;
}
DrLogI("WorkQueue::Stop exiting");
}
//
// Send updated status to vertex service
//
void DVertexXComputePnController::
SendSetStatusRequest(DryadPnProcessPropertyRequest* r)
{
//
// Cast request to required type and make sure it's valid
//
DVertexXComputeSetStatus* request =
dynamic_cast<DVertexXComputeSetStatus*>(r);
LogAssert(request != NULL);
//
// Wrap request in XComputeSetStatusOverlapped
//
XComputeSetStatusOverlapped* overlapped =
new XComputeSetStatusOverlapped(request);
//
// Create asynchronous execution information
//
XC_ASYNC_INFO asyncInfo;
memset(&asyncInfo, 0, sizeof(asyncInfo));
asyncInfo.cbSize = sizeof(asyncInfo);
asyncInfo.pOperationState = overlapped->GetOperationState();
asyncInfo.IOCP = g_dryadNativePort->GetCompletionPort();
asyncInfo.pOverlapped = overlapped->GetOverlapped();
//
// Update request counters
//
request->IncrementSendCount();
g_dryadNativePort->IncrementOutstandingRequests();
//
// Update process info
//
XCERROR err =
XcSetAndGetProcessInfo(NULL,//GetProcessHandle(),
request->MarshalProperty(),
request->GetResults(),
&asyncInfo);
LogAssert(err != DrError_OK);
if (err != HRESULT_FROM_WIN32(ERROR_IO_PENDING))
{
//
// If failed (other than due to pending IO) log failure and update request counter
//
g_dryadNativePort->DecrementOutstandingRequests();
//
// If request assertion true, report errors as warnings, otherwise report as error and fail
// todo: this still seems backwards - need to figure out rational
// request handles retries itself
//
if (request->IsAssert())
{
DrLogW(
"Status request send failed synchronously during assert: not asserting again. done %u send tries, error %s",
request->GetSendCount(), DRERRORSTRING(err));
}
else
{
DrLogA(
"Status request send failed synchronously. done %u send tries, error %s",
request->GetSendCount(), DRERRORSTRING(err));
}
delete overlapped;
}
}