static int
DsRequestCb(void *notifyData, // IN: callback data
VMCIDatagram *msg) // IN: datagram
{
/* FIXME: On-stack 300byte buffer is no-no. Besides that it is ignored anyway. */
char replyBuffer[VMCI_DS_MAX_MSG_SIZE + sizeof(VMCIDatagram)];
VMCIDatagram *replyMsg = (VMCIDatagram *)replyBuffer;
int written, retval;
VMCIPrivilegeFlags srcPrivFlags;
VMCI_DEBUG_LOG((LGPFX"Got request from context: %d\n", msg->src.context));
if (VMCIDatagram_GetPrivFlags(msg->src, &srcPrivFlags) != VMCI_SUCCESS) {
retval = VMCI_ERROR_INVALID_ARGS;
goto done;
}
replyMsg->dst = msg->src;
replyMsg->src = dsAPI.handle;
DsHandleMessage(VMCI_DG_PAYLOAD(msg), VMCI_DG_PAYLOAD(replyMsg),
VMCI_DS_MAX_MSG_SIZE, &written, msg->src.context,
srcPrivFlags);
replyMsg->payloadSize = written;
/* Send reply back to source handle. */
retval = VMCIDatagramSendInt(replyMsg);
done:
if (retval >= VMCI_SUCCESS) {
VMCI_DEBUG_LOG((LGPFX"Successfully replied with %d bytes\n", written));
} else {
VMCILOG((LGPFX"Failed to reply to request: %d.\n", retval));
}
return retval;
}
int
vmci_datagram_create_handle_priv(
VMCIId resourceID, // IN: Optional, generated if VMCI_INVALID_ID
uint32 flags, // IN:
VMCIPrivilegeFlags privFlags, // IN:
VMCIDatagramRecvCB recvCB, // IN:
void *clientData, // IN:
VMCIHandle *outHandle) // OUT: newly created handle
{
if (outHandle == NULL) {
return VMCI_ERROR_INVALID_ARGS;
}
if (recvCB == NULL) {
VMCI_DEBUG_LOG(4,
(LGPFX"Client callback needed when creating datagram.\n"));
return VMCI_ERROR_INVALID_ARGS;
}
if (privFlags & ~VMCI_PRIVILEGE_ALL_FLAGS) {
return VMCI_ERROR_INVALID_ARGS;
}
return DatagramCreateHnd(resourceID, flags, privFlags, recvCB, clientData,
outHandle);
}
int
VMCI_HostInit(void)
{
int result;
/*
* In theory, it is unsafe to pass an eventHnd of -1 to platforms which use
* it (VMKernel/Windows/Mac OS at the time of this writing). In practice we
* are fine though, because the event is never used in the case of the host
* context.
*/
result = VMCIContext_InitContext(VMCI_HOST_CONTEXT_ID,
VMCI_DEFAULT_PROC_PRIVILEGE_FLAGS,
-1, VMCI_VERSION, NULL, &hostContext);
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIContext (result=%d).\n",
result));
goto errorExit;
}
result = VMCIQPBroker_Init();
if (result < VMCI_SUCCESS) {
goto hostContextExit;
}
VMCI_DEBUG_LOG(0, (LGPFX"host components initialized.\n"));
return VMCI_SUCCESS;
hostContextExit:
VMCIContext_ReleaseContext(hostContext);
errorExit:
return result;
}
static int
VPageChannelSendConnectionMessage(VPageChannel *channel) // IN
{
VPageChannelGuestConnectMessage message;
ASSERT(channel);
channel->state = VPCState_Connecting;
memset(&message, 0, sizeof message);
message.dgHandle = channel->dgHandle;
message.qpHandle = channel->qpHandle;
message.produceQSize = channel->produceQSize;
message.consumeQSize = channel->consumeQSize;
message.doorbellHandle = channel->doorbellHandle;
VMCI_DEBUG_LOG(10,
(LGPFX"Sending guest connect (channel=%p) "
"(qp handle=0x%x:0x%x).\n",
channel,
channel->qpHandle.context,
channel->qpHandle.resource));
return VPageChannelSendControl(channel, VPCPacket_GuestConnect,
(char *)&message, sizeof message, 0, NULL);
}
int
vmci_datagram_destroy_handle(VMCIHandle handle) // IN
{
DatagramEntry *entry;
VMCIResource *resource = VMCIResource_Get(handle,
VMCI_RESOURCE_TYPE_DATAGRAM);
if (resource == NULL) {
VMCI_DEBUG_LOG(4, (LGPFX"Failed to destroy datagram (handle=0x%x:0x%x).\n",
handle.context, handle.resource));
return VMCI_ERROR_NOT_FOUND;
}
entry = RESOURCE_CONTAINER(resource, DatagramEntry, resource);
VMCIResource_Remove(handle, VMCI_RESOURCE_TYPE_DATAGRAM);
/*
* We now wait on the destroyEvent and release the reference we got
* above.
*/
VMCI_WaitOnEvent(&entry->destroyEvent, DatagramReleaseCB, entry);
/*
* We know that we are now the only reference to the above entry so
* can safely free it.
*/
VMCI_DestroyEvent(&entry->destroyEvent);
VMCI_FreeKernelMem(entry, sizeof *entry);
return VMCI_SUCCESS;
}
int
VMCIDatagram_InvokeGuestHandler(VMCIDatagram *dg) // IN
{
#if defined(VMKERNEL)
VMCI_WARNING((LGPFX"Cannot dispatch within guest in VMKERNEL.\n"));
return VMCI_ERROR_DST_UNREACHABLE;
#else // VMKERNEL
int retval;
VMCIResource *resource;
DatagramEntry *dstEntry;
ASSERT(dg);
resource = VMCIResource_Get(dg->dst, VMCI_RESOURCE_TYPE_DATAGRAM);
if (NULL == resource) {
VMCI_DEBUG_LOG(4, (LGPFX"destination (handle=0x%x:0x%x) doesn't exist.\n",
dg->dst.context, dg->dst.resource));
return VMCI_ERROR_NO_HANDLE;
}
dstEntry = RESOURCE_CONTAINER(resource, DatagramEntry, resource);
if (dstEntry->runDelayed) {
VMCIDelayedDatagramInfo *dgInfo;
dgInfo = VMCI_AllocKernelMem(sizeof *dgInfo + (size_t)dg->payloadSize,
(VMCI_MEMORY_ATOMIC | VMCI_MEMORY_NONPAGED));
if (NULL == dgInfo) {
VMCIResource_Release(resource);
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
dgInfo->inDGHostQueue = FALSE;
dgInfo->entry = dstEntry;
memcpy(&dgInfo->msg, dg, VMCI_DG_SIZE(dg));
retval = VMCI_ScheduleDelayedWork(VMCIDatagramDelayedDispatchCB, dgInfo);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to schedule delayed work for datagram "
"(result=%d).\n", retval));
VMCI_FreeKernelMem(dgInfo, sizeof *dgInfo + (size_t)dg->payloadSize);
VMCIResource_Release(resource);
dgInfo = NULL;
goto exit;
}
} else {
dstEntry->recvCB(dstEntry->clientData, dg);
VMCIResource_Release(resource);
retval = VMCI_SUCCESS;
}
exit:
return retval;
#endif // VMKERNEL
}
static void
VMCIUtilCidUpdate(VMCIId subID, // IN:
VMCI_EventData *eventData, // IN:
void *clientData) // IN:
{
VMCIEventPayload_Context *evPayload = VMCIEventDataPayload(eventData);
if (subID != ctxUpdateSubID) {
VMCI_DEBUG_LOG(4, (LGPFX"Invalid subscriber (ID=0x%x).\n", subID));
return;
}
if (eventData == NULL || evPayload->contextID == VMCI_INVALID_ID) {
VMCI_DEBUG_LOG(4, (LGPFX"Invalid event data.\n"));
return;
}
VMCI_LOG((LGPFX"Updating context from (ID=0x%x) to (ID=0x%x) on event "
"(type=%d).\n", Atomic_Read(&vmContextID), evPayload->contextID,
eventData->event));
Atomic_Write(&vmContextID, evPayload->contextID);
}
int
VMCI_SharedInit(void)
{
int result;
result = VMCIResource_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIResource (result=%d).\n",
result));
goto errorExit;
}
result = VMCIContext_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIContext (result=%d).\n",
result));
goto resourceExit;
}
result = VMCIDatagram_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIDatagram (result=%d).\n",
result));
goto contextExit;
}
result = VMCIEvent_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIEvent (result=%d).\n",
result));
goto datagramExit;
}
result = VMCIDoorbell_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIDoorbell (result=%d).\n",
result));
goto eventExit;
}
VMCI_DEBUG_LOG(0, (LGPFX"shared components initialized.\n"));
return VMCI_SUCCESS;
eventExit:
VMCIEvent_Exit();
datagramExit:
VMCIDatagram_Exit();
contextExit:
VMCIContext_Exit();
resourceExit:
VMCIResource_Exit();
errorExit:
return result;
}
int
VMCIDatagram_Dispatch(VMCIId contextID,
VMCIDatagram *dg,
Bool fromGuest)
{
int retval;
VMCIRoute route;
ASSERT(dg);
ASSERT_ON_COMPILE(sizeof(VMCIDatagram) == 24);
if (VMCI_DG_SIZE(dg) > VMCI_MAX_DG_SIZE) {
VMCI_DEBUG_LOG(4, (LGPFX"Payload (size=%"FMT64"u bytes) too big to "
"send.\n", dg->payloadSize));
return VMCI_ERROR_INVALID_ARGS;
}
retval = VMCI_Route(&dg->src, &dg->dst, fromGuest, &route);
if (retval < VMCI_SUCCESS) {
VMCI_DEBUG_LOG(4, (LGPFX"Failed to route datagram (src=0x%x, dst=0x%x, "
"err=%d)\n.", dg->src.context, dg->dst.context,
retval));
return retval;
}
if (VMCI_ROUTE_AS_HOST == route) {
if (VMCI_INVALID_ID == contextID) {
contextID = VMCI_HOST_CONTEXT_ID;
}
return VMCIDatagramDispatchAsHost(contextID, dg);
}
if (VMCI_ROUTE_AS_GUEST == route) {
return VMCIDatagramDispatchAsGuest(dg);
}
VMCI_WARNING((LGPFX"Unknown route (%d) for datagram.\n", route));
return VMCI_ERROR_DST_UNREACHABLE;
}
Bool
VMCI_CheckHostCapabilities(void)
{
Bool result = VMCIEvent_CheckHostCapabilities();
result &= VMCIDatagram_CheckHostCapabilities();
result &= VMCIUtilCheckHostCapabilities();
if (!result) {
/* If it failed, then make sure this goes to the system event log. */
VMCI_WARNING((LGPFX"Host capability checked failed.\n"));
} else {
VMCI_DEBUG_LOG(0, (LGPFX"Host capability check passed.\n"));
}
return result;
}
void
VPageChannel_Destroy(VPageChannel *channel) // IN/OUT
{
ASSERT(channel);
VPageChannelDestroyQueuePair(channel);
if (!VMCI_HANDLE_INVALID(channel->doorbellHandle)) {
vmci_doorbell_destroy(channel->doorbellHandle);
}
if (!VMCI_HANDLE_INVALID(channel->dgHandle)) {
vmci_datagram_destroy_handle(channel->dgHandle);
}
channel->state = VPCState_Free;
VMCI_FreeKernelMem(channel, sizeof *channel);
VMCI_DEBUG_LOG(10,
(LGPFX"Destroyed (channel=%p).\n",
channel));
}
static void
VPageChannelPeerDetachCB(VMCIId subId, // IN
VMCI_EventData *eData, // IN
void *clientData) // IN
{
VPageChannel *channel;
VMCIEventPayload_QP *ePayload;
ASSERT(eData);
ASSERT(clientData);
channel = (VPageChannel *)clientData;
ePayload = VMCIEventDataPayload(eData);
if (VMCI_HANDLE_EQUAL(channel->qpHandle, ePayload->handle)) {
VMCI_DEBUG_LOG(10,
(LGPFX"Peer detached (channel=%p) "
"(qp handle=0x%x:0x%x).\n",
channel,
ePayload->handle.context,
ePayload->handle.resource));
channel->state = VPCState_Disconnected;
}
}
int
VPageChannel_CreateInVM(VPageChannel **channel, // IN/OUT
VMCIId resourceId, // IN
VMCIId peerResourceId, // IN
uint64 produceQSize, // IN
uint64 consumeQSize, // IN
uint32 channelFlags, // IN
VPageChannelRecvCB recvCB, // IN
void *clientRecvData, // IN
VPageChannelAllocElemFn elemAllocFn, // IN
void *allocClientData, // IN
VPageChannelFreeElemFn elemFreeFn, // IN
void *freeClientData, // IN
int defaultRecvBuffers, // IN
int maxRecvBuffers) // IN
{
int retval;
int flags;
VPageChannel *pageChannel;
ASSERT(channel);
ASSERT(VMCI_INVALID_ID != resourceId);
ASSERT(VMCI_INVALID_ID != peerResourceId);
ASSERT(recvCB);
if (channelFlags & ~(VPAGECHANNEL_FLAGS_ALL)) {
VMCI_WARNING((LGPFX"Invalid argument (flags=0x%x).\n",
channelFlags));
return VMCI_ERROR_INVALID_ARGS;
}
pageChannel =
VMCI_AllocKernelMem(sizeof *pageChannel, VMCI_MEMORY_NONPAGED);
if (!pageChannel) {
VMCI_WARNING((LGPFX"Failed to allocate channel memory.\n"));
return VMCI_ERROR_NO_MEM;
}
/*
* XXX, we should support a default internal allocation function.
*/
memset(pageChannel, 0, sizeof *pageChannel);
pageChannel->state = VPCState_Unconnected;
pageChannel->dgHandle = VMCI_INVALID_HANDLE;
pageChannel->attachSubId = VMCI_INVALID_ID;
pageChannel->detachSubId = VMCI_INVALID_ID;
pageChannel->qpHandle = VMCI_INVALID_HANDLE;
pageChannel->qpair = NULL;
pageChannel->doorbellHandle = VMCI_INVALID_HANDLE;
pageChannel->peerDoorbellHandle = VMCI_INVALID_HANDLE;
pageChannel->flags = channelFlags;
pageChannel->recvCB = recvCB;
pageChannel->clientRecvData = clientRecvData;
pageChannel->elemAllocFn = elemAllocFn;
pageChannel->allocClientData = allocClientData;
pageChannel->elemFreeFn = elemFreeFn;
pageChannel->freeClientData = freeClientData;
pageChannel->resourceId = resourceId;
pageChannel->peerDgHandle = VMCI_MAKE_HANDLE(VMCI_HOST_CONTEXT_ID,
peerResourceId);
Atomic_Write32(&pageChannel->curRecvBufs, 0);
pageChannel->recvBufsTarget = defaultRecvBuffers;
pageChannel->defaultRecvBufs = defaultRecvBuffers;
pageChannel->maxRecvBufs = maxRecvBuffers + VMCI_PACKET_RECV_THRESHOLD;
pageChannel->produceQSize = produceQSize;
pageChannel->consumeQSize = consumeQSize;
/*
* Create a datagram handle over which we will connection handshake packets
* (once the queuepair is created we can send packets over that instead).
* This handle has a delayed callback regardless of the channel flags,
* because we may have to create a queuepair inside the callback.
*/
flags = VMCI_FLAG_DG_DELAYED_CB;
retval = vmci_datagram_create_handle(resourceId, flags,
VPageChannelDgRecvFunc, pageChannel,
&pageChannel->dgHandle);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to create datagram handle "
"(channel=%p) (err=%d).\n",
channel,
retval));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created datagram (channel=%p) "
"(handle=0x%x:0x%x).\n",
channel,
pageChannel->dgHandle.context,
pageChannel->dgHandle.resource));
/*
* Create a doorbell handle. This is used by the peer to signal the
* arrival of packets in the queuepair. This handle has a delayed
* callback depending on the channel flags.
*/
flags = channelFlags & VPAGECHANNEL_FLAGS_RECV_DELAYED ?
VMCI_FLAG_DELAYED_CB : 0;
retval = vmci_doorbell_create(&pageChannel->doorbellHandle,
flags, VMCI_PRIVILEGE_FLAG_RESTRICTED,
VPageChannelDoorbellCallback, pageChannel);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to create doorbell "
"(channel=%p) (err=%d).\n",
channel,
retval));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created doorbell (channel=%p) "
"(handle=0x%x:0x%x).\n",
channel,
pageChannel->doorbellHandle.context,
pageChannel->doorbellHandle.resource));
/*
* Now create the queuepair, over which we can pass data packets.
*/
retval = VPageChannelCreateQueuePair(pageChannel);
if (retval < VMCI_SUCCESS) {
goto error;
}
/*
* Set the receiving buffers before sending the connection message to
* avoid a race when the connection is made, but there is no receiving
* buffer yet.
*/
if (defaultRecvBuffers) {
int numElems = defaultRecvBuffers + VMCI_PACKET_RECV_THRESHOLD;
if (0 == VPageChannelAddRecvBuffers(pageChannel, numElems, TRUE)) {
/*
* AddRecvBuffers() returns the number of buffers actually added. If
* we failed to add any at all, then fail.
*/
retval = VMCI_ERROR_NO_MEM;
goto error;
}
}
retval = VPageChannelSendConnectionMessage(pageChannel);
if (retval < VMCI_SUCCESS) {
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created (channel=%p) (handle=0x%x:0x%x).\n",
pageChannel,
pageChannel->dgHandle.context,
pageChannel->dgHandle.resource));
*channel = pageChannel;
return retval;
error:
VPageChannel_Destroy(pageChannel);
return retval;
}
static int
VPageChannelSendPacket(VPageChannel *channel, // IN
VPageChannelPacket *packet, // IN
Bool needsLock, // IN
Bool signalPending) // IN
{
int retval;
ssize_t totalSize, sentSize;
ssize_t freeSpace;
unsigned long flags;
ASSERT(channel);
if (VPCState_Connected != channel->state) {
VMCI_WARNING((LGPFX"Not connected (channel=%p).\n",
channel));
return VMCI_ERROR_DST_UNREACHABLE;
}
ASSERT(packet);
totalSize = sizeof(VPageChannelPacket) + packet->msgLen +
packet->numElems * sizeof(VPageChannelElem);
if (needsLock) {
VPageChannelAcquireSendLock(channel, &flags);
} else {
flags = 0; /* Silence compiler. */
}
freeSpace = vmci_qpair_produce_free_space(channel->qpair);
if (freeSpace < totalSize) {
VMCI_WARNING((LGPFX"No free space in queuepair (channel=%p) "
"(required=%"FMTSZ"d) (actual=%"FMTSZ"d).\n",
channel,
totalSize,
freeSpace));
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
sentSize = vmci_qpair_enqueue(channel->qpair, packet, totalSize, 0);
if (!signalPending) {
if (sentSize == vmci_qpair_produce_buf_ready(channel->qpair)) {
retval = VPageChannelSignal(channel);
if (retval < VMCI_SUCCESS) {
goto exit;
}
}
}
if (sentSize < totalSize) {
/*
* XXX, deal with partial sending.
*/
VMCI_WARNING((LGPFX"No free space in queuepair (channel=%p) "
"(required=%"FMTSZ"d) (actual=%"FMTSZ"d).\n",
channel,
totalSize,
sentSize));
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Sent packet (channel=%p) (size=%"FMTSZ"d).\n",
channel,
sentSize));
retval = VMCI_SUCCESS;
exit:
if (needsLock) {
VPageChannelReleaseSendLock(channel, flags);
}
return retval;
}
static int
VMCIDatagramDispatchAsHost(VMCIId contextID, // IN:
VMCIDatagram *dg) // IN:
{
int retval;
size_t dgSize;
VMCIPrivilegeFlags srcPrivFlags;
ASSERT(dg);
ASSERT(VMCI_HostPersonalityActive());
dgSize = VMCI_DG_SIZE(dg);
if (contextID == VMCI_HOST_CONTEXT_ID &&
dg->dst.context == VMCI_HYPERVISOR_CONTEXT_ID) {
VMCI_DEBUG_LOG(4, (LGPFX"Host cannot talk to hypervisor\n"));
return VMCI_ERROR_DST_UNREACHABLE;
}
ASSERT(dg->dst.context != VMCI_HYPERVISOR_CONTEXT_ID);
/* Chatty. */
// VMCI_DEBUG_LOG(10, (LGPFX"Sending from (handle=0x%x:0x%x) to "
// "(handle=0x%x:0x%x) (size=%u bytes).\n",
// dg->src.context, dg->src.resource,
// dg->dst.context, dg->dst.resource, (uint32)dgSize));
/*
* Check that source handle matches sending context.
*/
if (dg->src.context != contextID) {
VMCI_DEBUG_LOG(4, (LGPFX"Sender context (ID=0x%x) is not owner of src "
"datagram entry (handle=0x%x:0x%x).\n",
contextID, dg->src.context, dg->src.resource));
return VMCI_ERROR_NO_ACCESS;
}
/*
* Get hold of privileges of sending endpoint.
*/
retval = VMCIDatagramGetPrivFlagsInt(contextID, dg->src, &srcPrivFlags);
if (retval != VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Couldn't get privileges (handle=0x%x:0x%x).\n",
dg->src.context, dg->src.resource));
return retval;
}
/* Determine if we should route to host or guest destination. */
if (dg->dst.context == VMCI_HOST_CONTEXT_ID) {
/* Route to host datagram entry. */
DatagramEntry *dstEntry;
VMCIResource *resource;
if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID &&
dg->dst.resource == VMCI_EVENT_HANDLER) {
return VMCIEvent_Dispatch(dg);
}
resource = VMCIResource_Get(dg->dst, VMCI_RESOURCE_TYPE_DATAGRAM);
if (resource == NULL) {
VMCI_DEBUG_LOG(4, (LGPFX"Sending to invalid destination "
"(handle=0x%x:0x%x).\n",
dg->dst.context, dg->dst.resource));
return VMCI_ERROR_INVALID_RESOURCE;
}
dstEntry = RESOURCE_CONTAINER(resource, DatagramEntry, resource);
if (VMCIDenyInteraction(srcPrivFlags, dstEntry->privFlags)) {
VMCIResource_Release(resource);
return VMCI_ERROR_NO_ACCESS;
}
ASSERT(dstEntry->recvCB);
/*
* If a VMCI datagram destined for the host is also sent by the
* host, we always run it delayed. This ensures that no locks
* are held when the datagram callback runs.
*/
if (dstEntry->runDelayed ||
(dg->src.context == VMCI_HOST_CONTEXT_ID &&
VMCI_CanScheduleDelayedWork())) {
VMCIDelayedDatagramInfo *dgInfo;
if (Atomic_FetchAndAdd(&delayedDGHostQueueSize, 1) ==
VMCI_MAX_DELAYED_DG_HOST_QUEUE_SIZE) {
Atomic_Dec(&delayedDGHostQueueSize);
VMCIResource_Release(resource);
return VMCI_ERROR_NO_MEM;
}
dgInfo = VMCI_AllocKernelMem(sizeof *dgInfo + (size_t)dg->payloadSize,
(VMCI_MEMORY_ATOMIC |
VMCI_MEMORY_NONPAGED));
if (NULL == dgInfo) {
Atomic_Dec(&delayedDGHostQueueSize);
VMCIResource_Release(resource);
return VMCI_ERROR_NO_MEM;
}
dgInfo->inDGHostQueue = TRUE;
dgInfo->entry = dstEntry;
memcpy(&dgInfo->msg, dg, dgSize);
retval = VMCI_ScheduleDelayedWork(VMCIDatagramDelayedDispatchCB, dgInfo);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to schedule delayed work for datagram "
"(result=%d).\n", retval));
VMCI_FreeKernelMem(dgInfo, sizeof *dgInfo + (size_t)dg->payloadSize);
VMCIResource_Release(resource);
Atomic_Dec(&delayedDGHostQueueSize);
return retval;
}
} else {
retval = dstEntry->recvCB(dstEntry->clientData, dg);
VMCIResource_Release(resource);
if (retval < VMCI_SUCCESS) {
return retval;
}
}
} else {
/*
* Route to destination VM context.
*/
VMCIDatagram *newDG;
if (contextID != dg->dst.context) {
if (VMCIDenyInteraction(srcPrivFlags,
vmci_context_get_priv_flags(dg->dst.context))) {
VMCI_DEBUG_LOG(4, (LGPFX"Interaction denied (%X/%X - %X/%X)\n",
contextID, srcPrivFlags,
dg->dst.context,
vmci_context_get_priv_flags(dg->dst.context)));
return VMCI_ERROR_NO_ACCESS;
} else if (VMCI_CONTEXT_IS_VM(contextID)) {
/*
* If the sending context is a VM, it cannot reach another VM.
*/
if (!vmkernel) {
VMCI_DEBUG_LOG(4, (LGPFX"Datagram communication between VMs not "
"supported (src=0x%x, dst=0x%x).\n",
contextID, dg->dst.context));
return VMCI_ERROR_DST_UNREACHABLE;
}
}
}
/* We make a copy to enqueue. */
newDG = VMCI_AllocKernelMem(dgSize, VMCI_MEMORY_NORMAL);
if (newDG == NULL) {
VMCI_DEBUG_LOG(4, (LGPFX"No memory for datagram\n"));
return VMCI_ERROR_NO_MEM;
}
memcpy(newDG, dg, dgSize);
retval = VMCIContext_EnqueueDatagram(dg->dst.context, newDG);
if (retval < VMCI_SUCCESS) {
VMCI_FreeKernelMem(newDG, dgSize);
VMCI_DEBUG_LOG(4, (LGPFX"Enqueue failed\n"));
return retval;
}
}
/* The datagram is freed when the context reads it. */
/* Chatty. */
// VMCI_DEBUG_LOG(10, (LGPFX"Sent datagram (size=%u bytes).\n",
// (uint32)dgSize));
/*
* We currently truncate the size to signed 32 bits. This doesn't
* matter for this handler as it only support 4Kb messages.
*/
return (int)dgSize;
}
static int
VPageChannelRecvPacket(VPageChannel *channel, // IN
VPageChannelPacket *packet) // IN
{
int curRecvBufs;
int recvBufsTarget;
ASSERT(channel);
ASSERT(packet);
if (packet->type != VPCPacket_Data &&
packet->type != VPCPacket_Completion_Notify &&
packet->type != VPCPacket_RequestBuffer &&
packet->type != VPCPacket_HyperConnect &&
packet->type != VPCPacket_HyperDisconnect) {
VMCI_WARNING((LGPFX"Received invalid packet (channel=%p) "
"(type=%d).\n",
channel,
packet->type));
return VMCI_ERROR_INVALID_ARGS;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Received packet (channel=%p) (type=%d) "
"(elems=%d).\n",
channel,
packet->type,
packet->numElems));
if (packet->type == VPCPacket_HyperConnect) {
VPageChannelHyperConnectMessage *message;
if (packet->msgLen < sizeof *message) {
VMCI_WARNING((LGPFX"Received invalid hypervisor connection message "
"(channel=%p) (size=%u).\n",
channel,
packet->msgLen));
return VMCI_ERROR_INVALID_ARGS;
}
message = (VPageChannelHyperConnectMessage *)
VPAGECHANNEL_PACKET_MESSAGE(packet);
channel->peerDoorbellHandle = message->doorbellHandle;
VMCI_DEBUG_LOG(10,
(LGPFX"Connected to peer (channel=%p) "
"(db handle=0x%x:0x%x).\n",
channel,
channel->peerDoorbellHandle.context,
channel->peerDoorbellHandle.resource));
return VMCI_SUCCESS;
}
recvBufsTarget = channel->recvBufsTarget;
switch (packet->type) {
case VPCPacket_RequestBuffer:
/*
* Increase the number of receive buffers by channel->defaultRecvBufs
* if the hypervisor requests it.
*/
VMCI_DEBUG_LOG(10,
(LGPFX"Requested more buffers (channel=%p) "
"(cur=%d) (target=%d) (max=%d).\n",
channel,
Atomic_Read32(&channel->curRecvBufs),
channel->recvBufsTarget,
channel->maxRecvBufs));
if (channel->recvBufsTarget < channel->maxRecvBufs) {
recvBufsTarget = channel->recvBufsTarget + channel->defaultRecvBufs;
}
break;
case VPCPacket_Data:
channel->recvCB(channel->clientRecvData, packet);
Atomic_Sub32(&channel->curRecvBufs, packet->numElems);
ASSERT(Atomic_Read32(&channel->curRecvBufs) > 0);
break;
case VPCPacket_Completion_Notify:
channel->recvCB(channel->clientRecvData, packet);
break;
case VPCPacket_HyperDisconnect:
VMCI_DEBUG_LOG(10,
(LGPFX"Hypervisor requested disconnection "
"(channel=%p) (numElems=%d).\n",
channel,
packet->numElems));
if (packet->numElems > 0) {
channel->elemFreeFn(channel->freeClientData,
VPAGECHANNEL_PACKET_ELEMS(packet),
packet->numElems);
}
(void)VPageChannelSendControl(channel, VPCPacket_GuestDisconnect,
NULL, 0, 0, NULL);
if (channel->state < VPCState_Disconnecting) {
channel->state = VPCState_Disconnecting;
}
return VMCI_SUCCESS;
default:
ASSERT_NOT_IMPLEMENTED(FALSE);
break;
}
/*
* Set more receive buffers if it is below the threshold. We bump it up
* here even when not requested to do so. This is to account for buffers
* being in-flight, i.e., in packets that have not yet been processed by
* the other side. When we increase here, we also tack on extra threshold,
* in the hope that we won't hit this again.
*/
curRecvBufs = Atomic_Read32(&channel->curRecvBufs);
if (curRecvBufs < (recvBufsTarget - VMCI_PACKET_RECV_THRESHOLD)) {
int numElems = recvBufsTarget + VMCI_PACKET_RECV_THRESHOLD - curRecvBufs;
(void)VPageChannelAddRecvBuffers(channel, numElems, FALSE);
channel->recvBufsTarget = recvBufsTarget;
}
return VMCI_SUCCESS;
}
static int
VPageChannelCreateQueuePair(VPageChannel *channel) // IN/OUT
{
int err;
uint32 flags;
ASSERT(channel);
ASSERT(VMCI_HANDLE_INVALID(channel->qpHandle));
ASSERT(NULL == channel->qpair);
ASSERT(VMCI_INVALID_ID == channel->detachSubId);
ASSERT(VMCI_INVALID_ID == channel->attachSubId);
if (channel->flags & VPAGECHANNEL_FLAGS_SEND_WHILE_ATOMIC ||
!(channel->flags & VPAGECHANNEL_FLAGS_RECV_DELAYED)) {
channel->useSpinLock = TRUE;
spin_lock_init(&channel->qpSendLock);
spin_lock_init(&channel->qpRecvLock);
} else {
sema_init(&channel->qpSendMutex, 1);
sema_init(&channel->qpRecvMutex, 1);
}
err = vmci_event_subscribe(VMCI_EVENT_QP_PEER_ATTACH,
#if !defined(linux) || defined(VMKERNEL)
VMCI_FLAG_EVENT_NONE,
#endif // !linux || VMKERNEL
VPageChannelPeerAttachCB,
channel, &channel->attachSubId);
if (err < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to attach event "
"(channel=%p) (err=%d).\n",
channel,
err));
goto error;
}
err = vmci_event_subscribe(VMCI_EVENT_QP_PEER_DETACH,
#if !defined(linux) || defined(VMKERNEL)
VMCI_FLAG_EVENT_NONE,
#endif // !linux || VMKERNEL
VPageChannelPeerDetachCB,
channel, &channel->detachSubId);
if (err < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to detach event "
"(channel=%p) (err=%d).\n",
channel,
err));
goto error;
}
if (channel->useSpinLock) {
flags = VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED;
} else {
flags = 0;
}
err = vmci_qpair_alloc(&channel->qpair, &channel->qpHandle,
channel->produceQSize, channel->consumeQSize,
VMCI_HOST_CONTEXT_ID, flags,
VMCI_NO_PRIVILEGE_FLAGS);
if (err < 0) {
VMCI_WARNING((LGPFX"Could not create queue pair (err=%d).\n",
err));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Allocated queuepair (channel=%p) "
"(qp handle=0x%x:0x%x) "
"(produce=%"FMT64"u) (consume=%"FMT64"u).\n",
channel,
channel->qpHandle.context,
channel->qpHandle.resource,
channel->produceQSize,
channel->consumeQSize));
return VMCI_SUCCESS;
error:
VPageChannelDestroyQueuePair(channel);
return err;
}
void
VMCI_ReadDatagramsFromPort(VMCIIoHandle ioHandle, // IN
VMCIIoPort dgInPort, // IN
uint8 *dgInBuffer, // IN
size_t dgInBufferSize) // IN
{
VMCIDatagram *dg;
size_t currentDgInBufferSize = PAGE_SIZE;
size_t remainingBytes;
ASSERT(dgInBufferSize >= PAGE_SIZE);
VMCI_ReadPortBytes(ioHandle, dgInPort, dgInBuffer, currentDgInBufferSize);
dg = (VMCIDatagram *)dgInBuffer;
remainingBytes = currentDgInBufferSize;
while (dg->dst.resource != VMCI_INVALID_ID || remainingBytes > PAGE_SIZE) {
unsigned dgInSize;
/*
* When the input buffer spans multiple pages, a datagram can
* start on any page boundary in the buffer.
*/
if (dg->dst.resource == VMCI_INVALID_ID) {
ASSERT(remainingBytes > PAGE_SIZE);
dg = (VMCIDatagram *)ROUNDUP((uintptr_t)dg + 1, PAGE_SIZE);
ASSERT((uint8 *)dg < dgInBuffer + currentDgInBufferSize);
remainingBytes = (size_t)(dgInBuffer + currentDgInBufferSize - (uint8 *)dg);
continue;
}
dgInSize = VMCI_DG_SIZE_ALIGNED(dg);
if (dgInSize <= dgInBufferSize) {
int result;
/*
* If the remaining bytes in the datagram buffer doesn't
* contain the complete datagram, we first make sure we have
* enough room for it and then we read the reminder of the
* datagram and possibly any following datagrams.
*/
if (dgInSize > remainingBytes) {
if (remainingBytes != currentDgInBufferSize) {
/*
* We move the partial datagram to the front and read
* the reminder of the datagram and possibly following
* calls into the following bytes.
*/
memmove(dgInBuffer, dgInBuffer + currentDgInBufferSize - remainingBytes,
remainingBytes);
dg = (VMCIDatagram *)dgInBuffer;
}
if (currentDgInBufferSize != dgInBufferSize) {
currentDgInBufferSize = dgInBufferSize;
}
VMCI_ReadPortBytes(ioHandle, dgInPort, dgInBuffer + remainingBytes,
currentDgInBufferSize - remainingBytes);
}
/* We special case event datagrams from the hypervisor. */
if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID &&
dg->dst.resource == VMCI_EVENT_HANDLER) {
result = VMCIEvent_Dispatch(dg);
} else {
result = VMCIDatagram_InvokeGuestHandler(dg);
}
if (result < VMCI_SUCCESS) {
VMCI_DEBUG_LOG(4, (LGPFX"Datagram with resource (ID=0x%x) failed "
"(err=%d).\n", dg->dst.resource, result));
}
/* On to the next datagram. */
dg = (VMCIDatagram *)((uint8 *)dg + dgInSize);
} else {
size_t bytesToSkip;
/*
* Datagram doesn't fit in datagram buffer of maximal size. We drop it.
*/
VMCI_DEBUG_LOG(4, (LGPFX"Failed to receive datagram (size=%u bytes).\n",
dgInSize));
bytesToSkip = dgInSize - remainingBytes;
if (currentDgInBufferSize != dgInBufferSize) {
currentDgInBufferSize = dgInBufferSize;
}
for (;;) {
VMCI_ReadPortBytes(ioHandle, dgInPort, dgInBuffer, currentDgInBufferSize);
if (bytesToSkip <= currentDgInBufferSize) {
break;
}
bytesToSkip -= currentDgInBufferSize;
}
dg = (VMCIDatagram *)(dgInBuffer + bytesToSkip);
}
remainingBytes = (size_t) (dgInBuffer + currentDgInBufferSize - (uint8 *)dg);
if (remainingBytes < VMCI_DG_HEADERSIZE) {
/* Get the next batch of datagrams. */
VMCI_ReadPortBytes(ioHandle, dgInPort, dgInBuffer, currentDgInBufferSize);
dg = (VMCIDatagram *)dgInBuffer;
remainingBytes = currentDgInBufferSize;
}
}
}
