void native_win_aio_provider::worker()
{
DWORD dwTransLen;
DWORD dwErrorCode;
ULONG_PTR dwKey;
LPOVERLAPPED overLap;
do
{
bool ret = (0 != GetQueuedCompletionStatus(_iocp, &dwTransLen, &dwKey, &overLap, INFINITE));
if (dwKey) break;
if (ret)
{
windows_disk_aio_context* ctx = CONTAINING_RECORD(overLap, windows_disk_aio_context, olp);
if (!ctx->evt)
{
aio_task* aio(ctx->tsk);
complete_io(aio, ERR_OK, dwTransLen);
}
else
{
ctx->err = ERR_OK;
ctx->bytes = dwTransLen;
ctx->evt->notify();
}
}
else if (overLap)
{
dwErrorCode = ::GetLastError();
dinfo("file operation failed, err = %u", dwErrorCode);
windows_disk_aio_context* ctx = CONTAINING_RECORD(overLap, windows_disk_aio_context, olp);
error_code err = dwErrorCode == ERROR_SUCCESS ? ERR_OK :
(dwErrorCode == ERROR_HANDLE_EOF ? ERR_HANDLE_EOF : ERR_FILE_OPERATION_FAILED);
if (!ctx->evt)
{
aio_task* aio(ctx->tsk);
complete_io(aio, err, dwTransLen);
}
else
{
ctx->err = err;
ctx->bytes = dwTransLen;
ctx->evt->notify();
}
}
else
{
::Sleep(1);
}
} while (true);
}
static void dec_count(struct io *io, unsigned int region, int error)
{
if (error)
set_bit(region, &io->error_bits);
if (atomic_dec_and_test(&io->count))
complete_io(io);
}
error_code native_win_aio_provider::aio_internal(aio_task_ptr& aio_tsk, bool async, __out_param uint32_t* pbytes /*= nullptr*/)
{
auto aio = (windows_disk_aio_context*)aio_tsk->aio().get();
BOOL r = FALSE;
aio->olp.Offset = (uint32_t)aio->file_offset;
aio->olp.OffsetHigh = (uint32_t)(aio->file_offset >> 32);
if (!async)
{
aio->evt = new utils::notify_event();
aio->err = ERR_OK;
aio->bytes = 0;
}
switch (aio->type)
{
case AIO_Read:
r = ::ReadFile(aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp);
break;
case AIO_Write:
r = ::WriteFile(aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp);
break;
default:
dassert (false, "unknown aio type %u", static_cast<int>(aio->type));
break;
}
if (!r)
{
int err = ::GetLastError();
if (err != ERROR_IO_PENDING)
{
derror("file operation failed, err = %u", err);
if (async)
{
complete_io(aio_tsk, ERR_FILE_OPERATION_FAILED, 0);
}
else
{
delete aio->evt;
aio->evt = nullptr;
}
return ERR_FILE_OPERATION_FAILED;
}
}
if (async)
{
return ERR_IO_PENDING;
}
else
{
aio->evt->wait();
delete aio->evt;
aio->evt = nullptr;
*pbytes = aio->bytes;
return aio->err;
}
}
error_code hpc_aio_provider::aio_internal(aio_task* aio_tsk, bool async, /*out*/ uint32_t* pbytes /*= nullptr*/)
{
auto aio = (windows_disk_aio_context*)aio_tsk->aio();
BOOL r = FALSE;
aio->olp.Offset = (uint32_t)aio->file_offset;
aio->olp.OffsetHigh = (uint32_t)(aio->file_offset >> 32);
if (!async)
{
aio->evt = new utils::notify_event();
aio->err = ERR_OK;
aio->bytes = 0;
}
switch (aio->type)
{
case AIO_Read:
r = ::ReadFile((HANDLE)aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp);
break;
case AIO_Write:
r = ::WriteFile((HANDLE)aio->file, aio->buffer, aio->buffer_size, NULL, &aio->olp);
break;
default:
dassert (false, "unknown aio type %u", static_cast<int>(aio->type));
break;
}
if (!r)
{
int native_error = ::GetLastError();
if (native_error != ERROR_IO_PENDING)
{
derror("file operation failed, err = %u", native_error);
error_code err = native_error == ERROR_SUCCESS ? ERR_OK :
(native_error == ERROR_HANDLE_EOF ? ERR_HANDLE_EOF : ERR_FILE_OPERATION_FAILED);
if (async)
{
complete_io(aio_tsk, err, 0);
}
else
{
delete aio->evt;
aio->evt = nullptr;
}
return err;
}
}
if (async)
{
return ERR_IO_PENDING;
}
else
{
aio->evt->wait();
delete aio->evt;
aio->evt = nullptr;
if (pbytes != nullptr)
{
*pbytes = aio->bytes;
}
return aio->err;
}
}
error_code hpc_aio_provider::aio_internal(aio_task* aio_tsk, bool async, /*out*/ uint32_t* pbytes /*= nullptr*/)
{
auto aio = (posix_disk_aio_context *)aio_tsk->aio();
int r;
aio->this_ = this;
aio->cb.aio_fildes = static_cast<int>((ssize_t)aio->file);
aio->cb.aio_buf = aio->buffer;
aio->cb.aio_nbytes = aio->buffer_size;
aio->cb.aio_offset = aio->file_offset;
// set up callback
aio->cb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
aio->cb.aio_sigevent.sigev_notify_kqueue = (int)(uintptr_t)_looper->native_handle();
aio->cb.aio_sigevent.sigev_notify_kevent_flags = EV_CLEAR;
aio->cb.aio_sigevent.sigev_value.sival_ptr = &_callback;
if (!async)
{
aio->evt = new utils::notify_event();
aio->err = ERR_OK;
aio->bytes = 0;
}
switch (aio->type)
{
case AIO_Read:
r = aio_read(&aio->cb);
break;
case AIO_Write:
r = aio_write(&aio->cb);
break;
default:
dassert(false, "unknown aio type %u", static_cast<int>(aio->type));
break;
}
if (r != 0)
{
derror("file op failed, err = %d (%s). On FreeBSD, you may need to load"
" aio kernel module by running 'sudo kldload aio'.", errno, strerror(errno));
if (async)
{
complete_io(aio_tsk, ERR_FILE_OPERATION_FAILED, 0);
}
else
{
delete aio->evt;
aio->evt = nullptr;
}
return ERR_FILE_OPERATION_FAILED;
}
else
{
if (async)
{
return ERR_IO_PENDING;
}
else
{
aio->evt->wait();
delete aio->evt;
aio->evt = nullptr;
if (pbytes != nullptr)
{
*pbytes = aio->bytes;
}
return aio->err;
}
}
}
///
/// complete_io needs to inform the caller of what to do next:
///
/// - No errors (ignore status), we just don't need to send more IO right now
/// - No errors (ignore status), we are done sending IO
/// - Error - we are done sending IO
///
ctsSocket::IOStatus ctsSocket::complete_io(ctsIOTask _task, unsigned _bytes_transferred, unsigned _status_code) throw()
{
//
// ignore completions for tasks on None
//
if (ctsIOTask::IOAction::None == _task.ioAction) {
return IOStatus::SuccessMoreIO;
}
//
// if FatalAbort, no IO was completed, but last_error might need to be set (only if not yet set to an error)
//
if (ctsIOTask::IOAction::FatalAbort == _task.ioAction) {
this->set_last_error(_status_code);
return IOStatus::Failure;
}
//
// pass all other completions to the protocol layer
// *NOT* taking a ctsSocket lock before calling through io_pattern
// - as IOPattern can also initiate calls through ctsSocket, which can then deadlock
//
ctsIOPatternStatus next_pattern_status;
auto ref_io_pattern(this->io_pattern);
if (!ref_io_pattern) {
this->set_last_error(WSAENOTSOCK);
next_pattern_status = ctsIOPatternStatus::ErrorIOFailed;
} else {
// not holding a lock when calling back through the ctsIOPattern
next_pattern_status = ref_io_pattern->complete_io(_task, _bytes_transferred, _status_code);
}
//
// now that we know the pattern status, update last_error as needed
//
if (NO_ERROR == _status_code) {
// If IO failed, capture the IO error
if (ctsIOPatternProtocolError(next_pattern_status)) {
this->set_last_error(next_pattern_status);
}
} else {
// _status_code is an error and the IOPattern didn't choose to ignore it
if (ctsIOPatternError(next_pattern_status)) {
this->set_last_error(_status_code);
}
}
//
// if we now need a FIN, invoke shutdown() first to ensure a FIN is sent to the target
//
if (ctsIOPatternStatus::RequestFIN == next_pattern_status) {
SOCKET s = this->lock_socket();
{ // scoping the lifetime of the lock
if (s != INVALID_SOCKET) {
if (SOCKET_ERROR == ::shutdown(this->socket, SD_SEND)) {
auto gle = ::WSAGetLastError();
this->set_last_error(gle);
ctsConfig::PrintErrorInfo(
L"[%.3f] ctsSocket - failed to initiate shutdown(SD_SEND) [%d] for a graceful disconnect",
ctsConfig::GetStatusTimeStamp(),
gle);
// can't continue - can't reliably request a FIN
next_pattern_status = ctsIOPatternStatus::ErrorIOFailed;
}
} else {
// otherwise indicate could not set
this->set_last_error(WSAENOTSOCK);
// can't continue - can't reliably request a FIN
next_pattern_status = ctsIOPatternStatus::ErrorIOFailed;
}
}
this->unlock_socket();
} else if (ctsIOPatternStatus::CompletedTransfer == next_pattern_status) {
// If the protocol has successfully completed, update last_error to no longer be ctsIOPatternStatusIORunning
this->set_last_error(NO_ERROR);
}
//
// return to the user how to interpret this IO
//
if (ctsIOPatternError(next_pattern_status)) {
// always close the socket if the protocol sees this as a failure
this->close_socket();
return IOStatus::Failure;
} else if (ctsIOPatternContinueIO(next_pattern_status)) {
return IOStatus::SuccessMoreIO;
} else {
ctl::ctFatalCondition(
(ctsIOPatternStatus::CompletedTransfer != next_pattern_status),
L"ctsSocket: Invalid ctsIOPatternStatus (%u)\n", next_pattern_status);
// If UDP, close the socket as pended IO could now be blocked forever
if (ctsConfig::Settings->Protocol != ctsConfig::ProtocolType::TCP) {
this->close_socket();
}
return IOStatus::SuccessDone;
}
}
error_code native_linux_aio_provider::aio_internal(aio_task* aio_tsk, bool async, /*out*/ uint32_t* pbytes /*= nullptr*/)
{
struct iocb *cbs[1];
linux_disk_aio_context * aio;
int ret;
aio = (linux_disk_aio_context *)aio_tsk->aio();
memset(&aio->cb, 0, sizeof(aio->cb));
aio->this_ = this;
switch (aio->type)
{
case AIO_Read:
io_prep_pread(&aio->cb, static_cast<int>((ssize_t)aio->file), aio->buffer, aio->buffer_size, aio->file_offset);
break;
case AIO_Write:
io_prep_pwrite(&aio->cb, static_cast<int>((ssize_t)aio->file), aio->buffer, aio->buffer_size, aio->file_offset);
break;
default:
derror("unknown aio type %u", static_cast<int>(aio->type));
}
if (!async)
{
aio->evt = new utils::notify_event();
aio->err = ERR_OK;
aio->bytes = 0;
}
cbs[0] = &aio->cb;
ret = io_submit(_ctx, 1, cbs);
if (ret != 1)
{
if (ret < 0)
derror("io_submit error, ret = %d", ret);
else
derror("could not sumbit IOs, ret = %d", ret);
if (async)
{
complete_io(aio_tsk, ERR_FILE_OPERATION_FAILED, 0);
}
else
{
delete aio->evt;
aio->evt = nullptr;
}
return ERR_FILE_OPERATION_FAILED;
}
else
{
if (async)
{
return ERR_IO_PENDING;
}
else
{
aio->evt->wait();
delete aio->evt;
aio->evt = nullptr;
if (pbytes != nullptr)
{
*pbytes = aio->bytes;
}
return aio->err;
}
}
}