static int
VPageChannelDgRecvFunc(void *clientData, // IN
VMCIDatagram *dg) // IN
{
VPageChannel *channel = (VPageChannel *)clientData;
ASSERT(channel);
ASSERT(dg);
if (dg->src.context != VMCI_HOST_CONTEXT_ID ||
dg->src.resource != channel->peerDgHandle.resource) {
VMCI_WARNING((LGPFX"Received a packet from an unknown source "
"(channel=%p) (handle=0x%x:0x%x).\n",
channel,
dg->src.context,
dg->src.resource));
return VMCI_ERROR_NO_ACCESS;
}
if (dg->payloadSize < sizeof (VPageChannelPacket)) {
VMCI_WARNING((LGPFX"Received invalid packet (channel=%p) "
"(size=%"FMT64"u).\n",
channel,
dg->payloadSize));
return VMCI_ERROR_INVALID_ARGS;
}
return VPageChannelRecvPacket(channel, VMCI_DG_PAYLOAD(dg));
}
int
VMCIDatagram_InvokeGuestHandler(VMCIDatagram *dg) // IN
{
#if defined(VMKERNEL)
VMCI_WARNING((LGPFX"Cannot dispatch within guest in VMKERNEL.\n"));
return VMCI_ERROR_DST_UNREACHABLE;
#else // VMKERNEL
int retval;
VMCIResource *resource;
DatagramEntry *dstEntry;
ASSERT(dg);
resource = VMCIResource_Get(dg->dst, VMCI_RESOURCE_TYPE_DATAGRAM);
if (NULL == resource) {
VMCI_DEBUG_LOG(4, (LGPFX"destination (handle=0x%x:0x%x) doesn't exist.\n",
dg->dst.context, dg->dst.resource));
return VMCI_ERROR_NO_HANDLE;
}
dstEntry = RESOURCE_CONTAINER(resource, DatagramEntry, resource);
if (dstEntry->runDelayed) {
VMCIDelayedDatagramInfo *dgInfo;
dgInfo = VMCI_AllocKernelMem(sizeof *dgInfo + (size_t)dg->payloadSize,
(VMCI_MEMORY_ATOMIC | VMCI_MEMORY_NONPAGED));
if (NULL == dgInfo) {
VMCIResource_Release(resource);
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
dgInfo->inDGHostQueue = FALSE;
dgInfo->entry = dstEntry;
memcpy(&dgInfo->msg, dg, VMCI_DG_SIZE(dg));
retval = VMCI_ScheduleDelayedWork(VMCIDatagramDelayedDispatchCB, dgInfo);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to schedule delayed work for datagram "
"(result=%d).\n", retval));
VMCI_FreeKernelMem(dgInfo, sizeof *dgInfo + (size_t)dg->payloadSize);
VMCIResource_Release(resource);
dgInfo = NULL;
goto exit;
}
} else {
dstEntry->recvCB(dstEntry->clientData, dg);
VMCIResource_Release(resource);
retval = VMCI_SUCCESS;
}
exit:
return retval;
#endif // VMKERNEL
}
int
VMCI_SharedInit(void)
{
int result;
result = VMCIResource_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIResource (result=%d).\n",
result));
goto errorExit;
}
result = VMCIContext_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIContext (result=%d).\n",
result));
goto resourceExit;
}
result = VMCIDatagram_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIDatagram (result=%d).\n",
result));
goto contextExit;
}
result = VMCIEvent_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIEvent (result=%d).\n",
result));
goto datagramExit;
}
result = VMCIDoorbell_Init();
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIDoorbell (result=%d).\n",
result));
goto eventExit;
}
VMCI_DEBUG_LOG(0, (LGPFX"shared components initialized.\n"));
return VMCI_SUCCESS;
eventExit:
VMCIEvent_Exit();
datagramExit:
VMCIDatagram_Exit();
contextExit:
VMCIContext_Exit();
resourceExit:
VMCIResource_Exit();
errorExit:
return result;
}
static Bool
VMCIUtilCheckHostCapabilities(void)
{
int result;
VMCIResourcesQueryMsg *msg;
uint32 msgSize = sizeof(VMCIResourcesQueryHdr) +
VMCI_UTIL_NUM_RESOURCES * sizeof(VMCI_Resource);
VMCIDatagram *checkMsg = VMCI_AllocKernelMem(msgSize, VMCI_MEMORY_NONPAGED);
if (checkMsg == NULL) {
VMCI_WARNING((LGPFX"Check host: Insufficient memory.\n"));
return FALSE;
}
checkMsg->dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
VMCI_RESOURCES_QUERY);
checkMsg->src = VMCI_ANON_SRC_HANDLE;
checkMsg->payloadSize = msgSize - VMCI_DG_HEADERSIZE;
msg = (VMCIResourcesQueryMsg *)VMCI_DG_PAYLOAD(checkMsg);
msg->numResources = VMCI_UTIL_NUM_RESOURCES;
msg->resources[0] = VMCI_GET_CONTEXT_ID;
result = VMCI_SendDatagram(checkMsg);
VMCI_FreeKernelMem(checkMsg, msgSize);
/* We need the vector. There are no fallbacks. */
return (result == 0x1);
}
int
VMCI_HostInit(void)
{
int result;
/*
* In theory, it is unsafe to pass an eventHnd of -1 to platforms which use
* it (VMKernel/Windows/Mac OS at the time of this writing). In practice we
* are fine though, because the event is never used in the case of the host
* context.
*/
result = VMCIContext_InitContext(VMCI_HOST_CONTEXT_ID,
VMCI_DEFAULT_PROC_PRIVILEGE_FLAGS,
-1, VMCI_VERSION, NULL, &hostContext);
if (result < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to initialize VMCIContext (result=%d).\n",
result));
goto errorExit;
}
result = VMCIQPBroker_Init();
if (result < VMCI_SUCCESS) {
goto hostContextExit;
}
VMCI_DEBUG_LOG(0, (LGPFX"host components initialized.\n"));
return VMCI_SUCCESS;
hostContextExit:
VMCIContext_ReleaseContext(hostContext);
errorExit:
return result;
}
static int
VPageChannelAllocDatagram(VPageChannel *channel, // IN
size_t messageLen, // IN
int numElems, // IN
VMCIDatagram **outDg) // OUT
{
size_t size;
VMCIDatagram *dg;
ASSERT(channel);
ASSERT(outDg);
*outDg = NULL;
size = VMCI_DG_HEADERSIZE + sizeof(VPageChannelPacket) + messageLen +
numElems * sizeof (VPageChannelElem);
if (size > VMCI_MAX_DG_SIZE) {
VMCI_WARNING((LGPFX"Requested datagram size too large (channel=%p) "
"(size=%"FMTSZ"u).",
channel,
size));
return VMCI_ERROR_PAYLOAD_TOO_LARGE;
}
dg = VMCI_AllocKernelMem(size, VMCI_MEMORY_ATOMIC);
if (!dg) {
VMCI_WARNING((LGPFX"Failed to allocate datagram (channel=%p).",
channel));
return VMCI_ERROR_NO_MEM;
}
memset(dg, 0, size);
dg->dst = channel->peerDgHandle;
dg->src = channel->dgHandle;
dg->payloadSize = size - VMCI_DG_HEADERSIZE;
/* Chatty. */
// VMCI_DEBUG_LOG(10,
// (LGPFX"Allocated datagram (payload=%"FMT64"u).\n",
// dg->payloadSize));
*outDg = dg;
return VMCI_SUCCESS;
}
void
VMCIUtil_Exit(void)
{
if (VMCIEvent_Unsubscribe(ctxUpdateSubID) < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to unsubscribe to event (type=%d) with "
"subscriber (ID=0x%x).\n", VMCI_EVENT_CTX_ID_UPDATE,
ctxUpdateSubID));
}
}
void
VMCIUtil_Init(void)
{
/*
* We subscribe to the VMCI_EVENT_CTX_ID_UPDATE here so we can update the
* internal context id when needed.
*/
if (VMCIEvent_Subscribe(VMCI_EVENT_CTX_ID_UPDATE, VMCI_FLAG_EVENT_NONE,
VMCIUtilCidUpdate, NULL,
&ctxUpdateSubID) < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to event (type=%d).\n",
VMCI_EVENT_CTX_ID_UPDATE));
}
}
Bool
VMCI_CheckHostCapabilities(void)
{
Bool result = VMCIEvent_CheckHostCapabilities();
result &= VMCIDatagram_CheckHostCapabilities();
result &= VMCIUtilCheckHostCapabilities();
if (!result) {
/* If it failed, then make sure this goes to the system event log. */
VMCI_WARNING((LGPFX"Host capability checked failed.\n"));
} else {
VMCI_DEBUG_LOG(0, (LGPFX"Host capability check passed.\n"));
}
return result;
}
void
VMCIUtil_Init(void)
{
/*
* We subscribe to the VMCI_EVENT_CTX_ID_UPDATE here so we can update the
* internal context id when needed.
*/
if (vmci_event_subscribe(VMCI_EVENT_CTX_ID_UPDATE,
#if !defined(linux) || defined(VMKERNEL)
VMCI_FLAG_EVENT_NONE,
#endif // !linux || VMKERNEL
VMCIUtilCidUpdate, NULL,
&ctxUpdateSubID) < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to event (type=%d).\n",
VMCI_EVENT_CTX_ID_UPDATE));
}
}
static int
VMCIDatagramDispatchAsGuest(VMCIDatagram *dg)
{
#if defined(VMKERNEL)
VMCI_WARNING((LGPFX"Cannot send down to host from VMKERNEL.\n"));
return VMCI_ERROR_DST_UNREACHABLE;
#else // VMKERNEL
int retval;
VMCIResource *resource;
resource = VMCIResource_Get(dg->src, VMCI_RESOURCE_TYPE_DATAGRAM);
if (NULL == resource) {
return VMCI_ERROR_NO_HANDLE;
}
retval = VMCI_SendDatagram(dg);
VMCIResource_Release(resource);
return retval;
#endif // VMKERNEL
}
static int
VPageChannelSignal(VPageChannel *channel) // IN
{
int retval;
ASSERT(channel);
retval = vmci_doorbell_notify(channel->peerDoorbellHandle,
/* XXX, TRUSTED for VMKernel. */
VMCI_PRIVILEGE_FLAG_RESTRICTED);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to notify doorbell (channel=%p) "
"(handle=0x%x:0x%x) (err=%d).\n",
channel,
channel->peerDoorbellHandle.context,
channel->peerDoorbellHandle.resource,
retval));
}
return retval;
}
int
VMCIDatagram_Dispatch(VMCIId contextID,
VMCIDatagram *dg,
Bool fromGuest)
{
int retval;
VMCIRoute route;
ASSERT(dg);
ASSERT_ON_COMPILE(sizeof(VMCIDatagram) == 24);
if (VMCI_DG_SIZE(dg) > VMCI_MAX_DG_SIZE) {
VMCI_DEBUG_LOG(4, (LGPFX"Payload (size=%"FMT64"u bytes) too big to "
"send.\n", dg->payloadSize));
return VMCI_ERROR_INVALID_ARGS;
}
retval = VMCI_Route(&dg->src, &dg->dst, fromGuest, &route);
if (retval < VMCI_SUCCESS) {
VMCI_DEBUG_LOG(4, (LGPFX"Failed to route datagram (src=0x%x, dst=0x%x, "
"err=%d)\n.", dg->src.context, dg->dst.context,
retval));
return retval;
}
if (VMCI_ROUTE_AS_HOST == route) {
if (VMCI_INVALID_ID == contextID) {
contextID = VMCI_HOST_CONTEXT_ID;
}
return VMCIDatagramDispatchAsHost(contextID, dg);
}
if (VMCI_ROUTE_AS_GUEST == route) {
return VMCIDatagramDispatchAsGuest(dg);
}
VMCI_WARNING((LGPFX"Unknown route (%d) for datagram.\n", route));
return VMCI_ERROR_DST_UNREACHABLE;
}
static int
VPageChannelCreateQueuePair(VPageChannel *channel) // IN/OUT
{
int err;
uint32 flags;
ASSERT(channel);
ASSERT(VMCI_HANDLE_INVALID(channel->qpHandle));
ASSERT(NULL == channel->qpair);
ASSERT(VMCI_INVALID_ID == channel->detachSubId);
ASSERT(VMCI_INVALID_ID == channel->attachSubId);
if (channel->flags & VPAGECHANNEL_FLAGS_SEND_WHILE_ATOMIC ||
!(channel->flags & VPAGECHANNEL_FLAGS_RECV_DELAYED)) {
channel->useSpinLock = TRUE;
spin_lock_init(&channel->qpSendLock);
spin_lock_init(&channel->qpRecvLock);
} else {
sema_init(&channel->qpSendMutex, 1);
sema_init(&channel->qpRecvMutex, 1);
}
err = vmci_event_subscribe(VMCI_EVENT_QP_PEER_ATTACH,
#if !defined(linux) || defined(VMKERNEL)
VMCI_FLAG_EVENT_NONE,
#endif // !linux || VMKERNEL
VPageChannelPeerAttachCB,
channel, &channel->attachSubId);
if (err < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to attach event "
"(channel=%p) (err=%d).\n",
channel,
err));
goto error;
}
err = vmci_event_subscribe(VMCI_EVENT_QP_PEER_DETACH,
#if !defined(linux) || defined(VMKERNEL)
VMCI_FLAG_EVENT_NONE,
#endif // !linux || VMKERNEL
VPageChannelPeerDetachCB,
channel, &channel->detachSubId);
if (err < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to subscribe to detach event "
"(channel=%p) (err=%d).\n",
channel,
err));
goto error;
}
if (channel->useSpinLock) {
flags = VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED;
} else {
flags = 0;
}
err = vmci_qpair_alloc(&channel->qpair, &channel->qpHandle,
channel->produceQSize, channel->consumeQSize,
VMCI_HOST_CONTEXT_ID, flags,
VMCI_NO_PRIVILEGE_FLAGS);
if (err < 0) {
VMCI_WARNING((LGPFX"Could not create queue pair (err=%d).\n",
err));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Allocated queuepair (channel=%p) "
"(qp handle=0x%x:0x%x) "
"(produce=%"FMT64"u) (consume=%"FMT64"u).\n",
channel,
channel->qpHandle.context,
channel->qpHandle.resource,
channel->produceQSize,
channel->consumeQSize));
return VMCI_SUCCESS;
error:
VPageChannelDestroyQueuePair(channel);
return err;
}
static int
VMCIDatagramDispatchAsHost(VMCIId contextID, // IN:
VMCIDatagram *dg) // IN:
{
int retval;
size_t dgSize;
VMCIPrivilegeFlags srcPrivFlags;
ASSERT(dg);
ASSERT(VMCI_HostPersonalityActive());
dgSize = VMCI_DG_SIZE(dg);
if (contextID == VMCI_HOST_CONTEXT_ID &&
dg->dst.context == VMCI_HYPERVISOR_CONTEXT_ID) {
VMCI_DEBUG_LOG(4, (LGPFX"Host cannot talk to hypervisor\n"));
return VMCI_ERROR_DST_UNREACHABLE;
}
ASSERT(dg->dst.context != VMCI_HYPERVISOR_CONTEXT_ID);
/* Chatty. */
// VMCI_DEBUG_LOG(10, (LGPFX"Sending from (handle=0x%x:0x%x) to "
// "(handle=0x%x:0x%x) (size=%u bytes).\n",
// dg->src.context, dg->src.resource,
// dg->dst.context, dg->dst.resource, (uint32)dgSize));
/*
* Check that source handle matches sending context.
*/
if (dg->src.context != contextID) {
VMCI_DEBUG_LOG(4, (LGPFX"Sender context (ID=0x%x) is not owner of src "
"datagram entry (handle=0x%x:0x%x).\n",
contextID, dg->src.context, dg->src.resource));
return VMCI_ERROR_NO_ACCESS;
}
/*
* Get hold of privileges of sending endpoint.
*/
retval = VMCIDatagramGetPrivFlagsInt(contextID, dg->src, &srcPrivFlags);
if (retval != VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Couldn't get privileges (handle=0x%x:0x%x).\n",
dg->src.context, dg->src.resource));
return retval;
}
/* Determine if we should route to host or guest destination. */
if (dg->dst.context == VMCI_HOST_CONTEXT_ID) {
/* Route to host datagram entry. */
DatagramEntry *dstEntry;
VMCIResource *resource;
if (dg->src.context == VMCI_HYPERVISOR_CONTEXT_ID &&
dg->dst.resource == VMCI_EVENT_HANDLER) {
return VMCIEvent_Dispatch(dg);
}
resource = VMCIResource_Get(dg->dst, VMCI_RESOURCE_TYPE_DATAGRAM);
if (resource == NULL) {
VMCI_DEBUG_LOG(4, (LGPFX"Sending to invalid destination "
"(handle=0x%x:0x%x).\n",
dg->dst.context, dg->dst.resource));
return VMCI_ERROR_INVALID_RESOURCE;
}
dstEntry = RESOURCE_CONTAINER(resource, DatagramEntry, resource);
if (VMCIDenyInteraction(srcPrivFlags, dstEntry->privFlags)) {
VMCIResource_Release(resource);
return VMCI_ERROR_NO_ACCESS;
}
ASSERT(dstEntry->recvCB);
/*
* If a VMCI datagram destined for the host is also sent by the
* host, we always run it delayed. This ensures that no locks
* are held when the datagram callback runs.
*/
if (dstEntry->runDelayed ||
(dg->src.context == VMCI_HOST_CONTEXT_ID &&
VMCI_CanScheduleDelayedWork())) {
VMCIDelayedDatagramInfo *dgInfo;
if (Atomic_FetchAndAdd(&delayedDGHostQueueSize, 1) ==
VMCI_MAX_DELAYED_DG_HOST_QUEUE_SIZE) {
Atomic_Dec(&delayedDGHostQueueSize);
VMCIResource_Release(resource);
return VMCI_ERROR_NO_MEM;
}
dgInfo = VMCI_AllocKernelMem(sizeof *dgInfo + (size_t)dg->payloadSize,
(VMCI_MEMORY_ATOMIC |
VMCI_MEMORY_NONPAGED));
if (NULL == dgInfo) {
Atomic_Dec(&delayedDGHostQueueSize);
VMCIResource_Release(resource);
return VMCI_ERROR_NO_MEM;
}
dgInfo->inDGHostQueue = TRUE;
dgInfo->entry = dstEntry;
memcpy(&dgInfo->msg, dg, dgSize);
retval = VMCI_ScheduleDelayedWork(VMCIDatagramDelayedDispatchCB, dgInfo);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to schedule delayed work for datagram "
"(result=%d).\n", retval));
VMCI_FreeKernelMem(dgInfo, sizeof *dgInfo + (size_t)dg->payloadSize);
VMCIResource_Release(resource);
Atomic_Dec(&delayedDGHostQueueSize);
return retval;
}
} else {
retval = dstEntry->recvCB(dstEntry->clientData, dg);
VMCIResource_Release(resource);
if (retval < VMCI_SUCCESS) {
return retval;
}
}
} else {
/*
* Route to destination VM context.
*/
VMCIDatagram *newDG;
if (contextID != dg->dst.context) {
if (VMCIDenyInteraction(srcPrivFlags,
vmci_context_get_priv_flags(dg->dst.context))) {
VMCI_DEBUG_LOG(4, (LGPFX"Interaction denied (%X/%X - %X/%X)\n",
contextID, srcPrivFlags,
dg->dst.context,
vmci_context_get_priv_flags(dg->dst.context)));
return VMCI_ERROR_NO_ACCESS;
} else if (VMCI_CONTEXT_IS_VM(contextID)) {
/*
* If the sending context is a VM, it cannot reach another VM.
*/
if (!vmkernel) {
VMCI_DEBUG_LOG(4, (LGPFX"Datagram communication between VMs not "
"supported (src=0x%x, dst=0x%x).\n",
contextID, dg->dst.context));
return VMCI_ERROR_DST_UNREACHABLE;
}
}
}
/* We make a copy to enqueue. */
newDG = VMCI_AllocKernelMem(dgSize, VMCI_MEMORY_NORMAL);
if (newDG == NULL) {
VMCI_DEBUG_LOG(4, (LGPFX"No memory for datagram\n"));
return VMCI_ERROR_NO_MEM;
}
memcpy(newDG, dg, dgSize);
retval = VMCIContext_EnqueueDatagram(dg->dst.context, newDG);
if (retval < VMCI_SUCCESS) {
VMCI_FreeKernelMem(newDG, dgSize);
VMCI_DEBUG_LOG(4, (LGPFX"Enqueue failed\n"));
return retval;
}
}
/* The datagram is freed when the context reads it. */
/* Chatty. */
// VMCI_DEBUG_LOG(10, (LGPFX"Sent datagram (size=%u bytes).\n",
// (uint32)dgSize));
/*
* We currently truncate the size to signed 32 bits. This doesn't
* matter for this handler as it only support 4Kb messages.
*/
return (int)dgSize;
}
static int
DatagramCreateHnd(VMCIId resourceID, // IN:
uint32 flags, // IN:
VMCIPrivilegeFlags privFlags, // IN:
VMCIDatagramRecvCB recvCB, // IN:
void *clientData, // IN:
VMCIHandle *outHandle) // OUT:
{
int result;
VMCIId contextID;
VMCIHandle handle;
DatagramEntry *entry;
ASSERT(recvCB != NULL);
ASSERT(outHandle != NULL);
ASSERT(!(privFlags & ~VMCI_PRIVILEGE_ALL_FLAGS));
if ((flags & VMCI_FLAG_WELLKNOWN_DG_HND) != 0) {
return VMCI_ERROR_INVALID_ARGS;
} else {
if ((flags & VMCI_FLAG_ANYCID_DG_HND) != 0) {
contextID = VMCI_INVALID_ID;
} else {
contextID = vmci_get_context_id();
if (contextID == VMCI_INVALID_ID) {
return VMCI_ERROR_NO_RESOURCES;
}
}
if (resourceID == VMCI_INVALID_ID) {
resourceID = VMCIResource_GetID(contextID);
if (resourceID == VMCI_INVALID_ID) {
return VMCI_ERROR_NO_HANDLE;
}
}
handle = VMCI_MAKE_HANDLE(contextID, resourceID);
}
entry = VMCI_AllocKernelMem(sizeof *entry, VMCI_MEMORY_NONPAGED);
if (entry == NULL) {
VMCI_WARNING((LGPFX"Failed allocating memory for datagram entry.\n"));
return VMCI_ERROR_NO_MEM;
}
if (!VMCI_CanScheduleDelayedWork()) {
if (flags & VMCI_FLAG_DG_DELAYED_CB) {
VMCI_FreeKernelMem(entry, sizeof *entry);
return VMCI_ERROR_INVALID_ARGS;
}
entry->runDelayed = FALSE;
} else {
entry->runDelayed = (flags & VMCI_FLAG_DG_DELAYED_CB) ? TRUE : FALSE;
}
entry->flags = flags;
entry->recvCB = recvCB;
entry->clientData = clientData;
VMCI_CreateEvent(&entry->destroyEvent);
entry->privFlags = privFlags;
/* Make datagram resource live. */
result = VMCIResource_Add(&entry->resource, VMCI_RESOURCE_TYPE_DATAGRAM,
handle, DatagramFreeCB, entry);
if (result != VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to add new resource (handle=0x%x:0x%x).\n",
handle.context, handle.resource));
VMCI_DestroyEvent(&entry->destroyEvent);
VMCI_FreeKernelMem(entry, sizeof *entry);
return result;
}
*outHandle = handle;
return VMCI_SUCCESS;
}
static int
VPageChannelSendControl(VPageChannel *channel, // IN
VPageChannelPacketType type, // IN
char *message, // IN
int len, // IN
int numElems, // IN
VPageChannelElem *elems) // IN
{
int retval;
VPageChannelPacket *packet;
VMCIDatagram *dg;
ASSERT(channel);
ASSERT(type == VPCPacket_Data ||
type == VPCPacket_GuestConnect ||
type == VPCPacket_SetRecvBuffer ||
type == VPCPacket_GuestDisconnect);
dg = NULL;
retval = VPageChannelAllocDatagram(channel, len, numElems, &dg);
if (retval < VMCI_SUCCESS) {
return retval;
}
packet = (VPageChannelPacket *)VMCI_DG_PAYLOAD(dg);
packet->type = type;
packet->msgLen = len;
packet->numElems = numElems;
if (len) {
ASSERT(message);
memcpy(VPAGECHANNEL_PACKET_MESSAGE(packet), message, len);
}
if (numElems) {
ASSERT(elems);
memcpy(VPAGECHANNEL_PACKET_ELEMS(packet), elems,
numElems * sizeof (VPageChannelElem));
}
retval = vmci_datagram_send(dg);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to send packet (channel=%p) to "
"(handle=0x%x:0x%x) (err=%d).\n",
channel,
dg->dst.context,
dg->dst.resource,
retval));
} else {
/*
* We don't care about how many bytes were sent, and callers may not
* expect > 0 to mean success, so just convert to exactly success.
*/
retval = VMCI_SUCCESS;
}
VMCI_FreeKernelMem(dg, VMCI_DG_SIZE(dg));
return retval;
}
static int
VPageChannelSendPacket(VPageChannel *channel, // IN
VPageChannelPacket *packet, // IN
Bool needsLock, // IN
Bool signalPending) // IN
{
int retval;
ssize_t totalSize, sentSize;
ssize_t freeSpace;
unsigned long flags;
ASSERT(channel);
if (VPCState_Connected != channel->state) {
VMCI_WARNING((LGPFX"Not connected (channel=%p).\n",
channel));
return VMCI_ERROR_DST_UNREACHABLE;
}
ASSERT(packet);
totalSize = sizeof(VPageChannelPacket) + packet->msgLen +
packet->numElems * sizeof(VPageChannelElem);
if (needsLock) {
VPageChannelAcquireSendLock(channel, &flags);
} else {
flags = 0; /* Silence compiler. */
}
freeSpace = vmci_qpair_produce_free_space(channel->qpair);
if (freeSpace < totalSize) {
VMCI_WARNING((LGPFX"No free space in queuepair (channel=%p) "
"(required=%"FMTSZ"d) (actual=%"FMTSZ"d).\n",
channel,
totalSize,
freeSpace));
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
sentSize = vmci_qpair_enqueue(channel->qpair, packet, totalSize, 0);
if (!signalPending) {
if (sentSize == vmci_qpair_produce_buf_ready(channel->qpair)) {
retval = VPageChannelSignal(channel);
if (retval < VMCI_SUCCESS) {
goto exit;
}
}
}
if (sentSize < totalSize) {
/*
* XXX, deal with partial sending.
*/
VMCI_WARNING((LGPFX"No free space in queuepair (channel=%p) "
"(required=%"FMTSZ"d) (actual=%"FMTSZ"d).\n",
channel,
totalSize,
sentSize));
retval = VMCI_ERROR_NO_MEM;
goto exit;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Sent packet (channel=%p) (size=%"FMTSZ"d).\n",
channel,
sentSize));
retval = VMCI_SUCCESS;
exit:
if (needsLock) {
VPageChannelReleaseSendLock(channel, flags);
}
return retval;
}
int
VPageChannel_Send(VPageChannel *channel, // IN/OUT
VPageChannelPacketType type, // IN
char *message, // IN
int len, // IN
VPageChannelBuffer *buffer) // IN
{
int retval;
int numElems;
ssize_t totalSize;
VPageChannelPacket *packet;
ASSERT(channel);
if (VPCState_Connected != channel->state) {
VMCI_WARNING((LGPFX"Not connected (channel=%p).\n",
channel));
return VMCI_ERROR_DST_UNREACHABLE;
}
if (buffer) {
numElems = buffer->numElems;
} else {
numElems = 0;
}
totalSize = sizeof(VPageChannelPacket) + len +
numElems * sizeof(VPageChannelElem);
packet = (VPageChannelPacket *)
VMCI_AllocKernelMem(totalSize,
channel->flags & VPAGECHANNEL_FLAGS_SEND_WHILE_ATOMIC ?
VMCI_MEMORY_ATOMIC : VMCI_MEMORY_NORMAL);
if (!packet) {
VMCI_WARNING((LGPFX"Failed to allocate packet (channel=%p) "
"(size=%"FMTSZ"d).",
channel,
totalSize));
return VMCI_ERROR_NO_MEM;
}
packet->type = type;
packet->msgLen = len;
packet->numElems = numElems;
if (len) {
ASSERT(message);
memcpy(VPAGECHANNEL_PACKET_MESSAGE(packet), message, len);
}
if (numElems) {
ASSERT(buffer);
ASSERT(buffer->elems);
memcpy(VPAGECHANNEL_PACKET_ELEMS(packet), buffer->elems,
numElems * sizeof (VPageChannelElem));
}
retval = VPageChannel_SendPacket(channel, packet);
VMCI_FreeKernelMem(packet, totalSize);
return retval;
}
static int
VPageChannelAddRecvBuffers(VPageChannel *channel, // IN
int numElems, // IN
Bool onInit) // IN
{
int n;
int sent;
int maxElems;
Bool isAtomic;
size_t size;
unsigned long flags;
VPageChannelElem *elems;
VPageChannelPacket *packet;
ASSERT(channel);
sent = 0;
size = 0;
elems = NULL;
packet = NULL;
if (onInit || (channel->flags & VPAGECHANNEL_FLAGS_RECV_DELAYED)) {
/*
* If we are initializing the channel, or we are running in a delayed
* context (recv() in this case), then we can using blocking allocation
* and we can allocate large packets. Also, no need to take the
* send lock here, we can just take it for each packet.
*/
isAtomic = FALSE;
maxElems = VMCI_PACKET_DGRAM_MAX_ELEMS;
flags = 0; /* Silence compiler. */
} else {
/*
* We're in an atomic context. We must allocate page-sized packets
* atomically and send them over the queuepair. Since this can
* cause a lot of signalling, we optimize by taking the send lock
* once for all packets, and only signalling when we are done.
*/
isAtomic = TRUE;
maxElems = VMCI_PACKET_PAGE_MAX_ELEMS;
VPageChannelAcquireSendLock(channel, &flags);
}
n = min_t(int, maxElems, numElems);
while (n > 0) {
int retval;
int allocNum;
/*
* First packet is always big enough to cover any remaining elements,
* so just allocate it once.
*/
if (NULL == packet) {
size = sizeof(VPageChannelPacket) + (n * sizeof(VPageChannelElem));
packet = (VPageChannelPacket *)
VMCI_AllocKernelMem(size,
isAtomic ? VMCI_MEMORY_ATOMIC : VMCI_MEMORY_NORMAL);
if (packet == NULL) {
VMCI_WARNING((LGPFX"Failed to allocate packet (channel=%p) "
"(size=%"FMTSZ"u).\n",
channel,
size));
goto exit;
}
packet->type = VPCPacket_SetRecvBuffer;
packet->msgLen = 0;
elems = VPAGECHANNEL_PACKET_ELEMS(packet);
}
allocNum = channel->elemAllocFn(channel->allocClientData, elems, n);
if (0 == allocNum) {
/*
* If the client failed to allocate any elements at all then just
* bail out and return whatever number we managed to send so far
* (if any).
*/
VMCI_WARNING((LGPFX"Failed to allocate receive buffer (channel=%p) "
"(expected=%d).\n",
channel,
n));
goto exit;
}
/*
* We wanted "n" elements, but we might only have "allocNum" because
* that's all the client could allocate. Pass down whatever we got.
*/
packet->numElems = allocNum;
if (onInit) {
retval = VPageChannelSendControl(channel, VPCPacket_SetRecvBuffer,
NULL, 0, allocNum, elems);
} else {
/*
* Do not ask for the lock here if we are atomic, we take care of
* that ourselves. Similarly, if we are atomic then we will do our
* own signalling, so inform the send that there is a signal already
* pending.
*/
retval = VPageChannelSendPacket(channel, packet,
isAtomic ? FALSE : TRUE, // needsLock
isAtomic ? TRUE : FALSE); // signalPending
/*
* XXX, what if this is a non-blocking queuepair and we fail to
* send because it's full and we can't wait? Is it even worth it
* to loop?
*/
}
if (retval < VMCI_SUCCESS) {
/*
* Failure to send is fatal. Release the client's elements and
* bail out.
*/
VMCI_WARNING((LGPFX"Failed to set receive buffers (channel=%p) "
"(err=%d).\n",
channel,
retval));
channel->elemFreeFn(channel->freeClientData, elems, allocNum);
goto exit;
}
Atomic_Add32(&channel->curRecvBufs, allocNum);
sent += allocNum;
numElems -= allocNum;
n = min_t(int, maxElems, numElems);
}
exit:
if (isAtomic) {
/*
* We're done sending packets, so now we can signal. Even if we only
* sent some of the requested buffers, we must signal anyway, otherwise
* the peer won't know about the ones we did send.
*/
(void)VPageChannelSignal(channel);
VPageChannelReleaseSendLock(channel, flags);
}
if (NULL != packet) {
VMCI_FreeKernelMem(packet, size);
}
return sent;
}
static int
VPageChannelRecvPacket(VPageChannel *channel, // IN
VPageChannelPacket *packet) // IN
{
int curRecvBufs;
int recvBufsTarget;
ASSERT(channel);
ASSERT(packet);
if (packet->type != VPCPacket_Data &&
packet->type != VPCPacket_Completion_Notify &&
packet->type != VPCPacket_RequestBuffer &&
packet->type != VPCPacket_HyperConnect &&
packet->type != VPCPacket_HyperDisconnect) {
VMCI_WARNING((LGPFX"Received invalid packet (channel=%p) "
"(type=%d).\n",
channel,
packet->type));
return VMCI_ERROR_INVALID_ARGS;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Received packet (channel=%p) (type=%d) "
"(elems=%d).\n",
channel,
packet->type,
packet->numElems));
if (packet->type == VPCPacket_HyperConnect) {
VPageChannelHyperConnectMessage *message;
if (packet->msgLen < sizeof *message) {
VMCI_WARNING((LGPFX"Received invalid hypervisor connection message "
"(channel=%p) (size=%u).\n",
channel,
packet->msgLen));
return VMCI_ERROR_INVALID_ARGS;
}
message = (VPageChannelHyperConnectMessage *)
VPAGECHANNEL_PACKET_MESSAGE(packet);
channel->peerDoorbellHandle = message->doorbellHandle;
VMCI_DEBUG_LOG(10,
(LGPFX"Connected to peer (channel=%p) "
"(db handle=0x%x:0x%x).\n",
channel,
channel->peerDoorbellHandle.context,
channel->peerDoorbellHandle.resource));
return VMCI_SUCCESS;
}
recvBufsTarget = channel->recvBufsTarget;
switch (packet->type) {
case VPCPacket_RequestBuffer:
/*
* Increase the number of receive buffers by channel->defaultRecvBufs
* if the hypervisor requests it.
*/
VMCI_DEBUG_LOG(10,
(LGPFX"Requested more buffers (channel=%p) "
"(cur=%d) (target=%d) (max=%d).\n",
channel,
Atomic_Read32(&channel->curRecvBufs),
channel->recvBufsTarget,
channel->maxRecvBufs));
if (channel->recvBufsTarget < channel->maxRecvBufs) {
recvBufsTarget = channel->recvBufsTarget + channel->defaultRecvBufs;
}
break;
case VPCPacket_Data:
channel->recvCB(channel->clientRecvData, packet);
Atomic_Sub32(&channel->curRecvBufs, packet->numElems);
ASSERT(Atomic_Read32(&channel->curRecvBufs) > 0);
break;
case VPCPacket_Completion_Notify:
channel->recvCB(channel->clientRecvData, packet);
break;
case VPCPacket_HyperDisconnect:
VMCI_DEBUG_LOG(10,
(LGPFX"Hypervisor requested disconnection "
"(channel=%p) (numElems=%d).\n",
channel,
packet->numElems));
if (packet->numElems > 0) {
channel->elemFreeFn(channel->freeClientData,
VPAGECHANNEL_PACKET_ELEMS(packet),
packet->numElems);
}
(void)VPageChannelSendControl(channel, VPCPacket_GuestDisconnect,
NULL, 0, 0, NULL);
if (channel->state < VPCState_Disconnecting) {
channel->state = VPCState_Disconnecting;
}
return VMCI_SUCCESS;
default:
ASSERT_NOT_IMPLEMENTED(FALSE);
break;
}
/*
* Set more receive buffers if it is below the threshold. We bump it up
* here even when not requested to do so. This is to account for buffers
* being in-flight, i.e., in packets that have not yet been processed by
* the other side. When we increase here, we also tack on extra threshold,
* in the hope that we won't hit this again.
*/
curRecvBufs = Atomic_Read32(&channel->curRecvBufs);
if (curRecvBufs < (recvBufsTarget - VMCI_PACKET_RECV_THRESHOLD)) {
int numElems = recvBufsTarget + VMCI_PACKET_RECV_THRESHOLD - curRecvBufs;
(void)VPageChannelAddRecvBuffers(channel, numElems, FALSE);
channel->recvBufsTarget = recvBufsTarget;
}
return VMCI_SUCCESS;
}
int
VPageChannel_CreateInVM(VPageChannel **channel, // IN/OUT
VMCIId resourceId, // IN
VMCIId peerResourceId, // IN
uint64 produceQSize, // IN
uint64 consumeQSize, // IN
uint32 channelFlags, // IN
VPageChannelRecvCB recvCB, // IN
void *clientRecvData, // IN
VPageChannelAllocElemFn elemAllocFn, // IN
void *allocClientData, // IN
VPageChannelFreeElemFn elemFreeFn, // IN
void *freeClientData, // IN
int defaultRecvBuffers, // IN
int maxRecvBuffers) // IN
{
int retval;
int flags;
VPageChannel *pageChannel;
ASSERT(channel);
ASSERT(VMCI_INVALID_ID != resourceId);
ASSERT(VMCI_INVALID_ID != peerResourceId);
ASSERT(recvCB);
if (channelFlags & ~(VPAGECHANNEL_FLAGS_ALL)) {
VMCI_WARNING((LGPFX"Invalid argument (flags=0x%x).\n",
channelFlags));
return VMCI_ERROR_INVALID_ARGS;
}
pageChannel =
VMCI_AllocKernelMem(sizeof *pageChannel, VMCI_MEMORY_NONPAGED);
if (!pageChannel) {
VMCI_WARNING((LGPFX"Failed to allocate channel memory.\n"));
return VMCI_ERROR_NO_MEM;
}
/*
* XXX, we should support a default internal allocation function.
*/
memset(pageChannel, 0, sizeof *pageChannel);
pageChannel->state = VPCState_Unconnected;
pageChannel->dgHandle = VMCI_INVALID_HANDLE;
pageChannel->attachSubId = VMCI_INVALID_ID;
pageChannel->detachSubId = VMCI_INVALID_ID;
pageChannel->qpHandle = VMCI_INVALID_HANDLE;
pageChannel->qpair = NULL;
pageChannel->doorbellHandle = VMCI_INVALID_HANDLE;
pageChannel->peerDoorbellHandle = VMCI_INVALID_HANDLE;
pageChannel->flags = channelFlags;
pageChannel->recvCB = recvCB;
pageChannel->clientRecvData = clientRecvData;
pageChannel->elemAllocFn = elemAllocFn;
pageChannel->allocClientData = allocClientData;
pageChannel->elemFreeFn = elemFreeFn;
pageChannel->freeClientData = freeClientData;
pageChannel->resourceId = resourceId;
pageChannel->peerDgHandle = VMCI_MAKE_HANDLE(VMCI_HOST_CONTEXT_ID,
peerResourceId);
Atomic_Write32(&pageChannel->curRecvBufs, 0);
pageChannel->recvBufsTarget = defaultRecvBuffers;
pageChannel->defaultRecvBufs = defaultRecvBuffers;
pageChannel->maxRecvBufs = maxRecvBuffers + VMCI_PACKET_RECV_THRESHOLD;
pageChannel->produceQSize = produceQSize;
pageChannel->consumeQSize = consumeQSize;
/*
* Create a datagram handle over which we will connection handshake packets
* (once the queuepair is created we can send packets over that instead).
* This handle has a delayed callback regardless of the channel flags,
* because we may have to create a queuepair inside the callback.
*/
flags = VMCI_FLAG_DG_DELAYED_CB;
retval = vmci_datagram_create_handle(resourceId, flags,
VPageChannelDgRecvFunc, pageChannel,
&pageChannel->dgHandle);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to create datagram handle "
"(channel=%p) (err=%d).\n",
channel,
retval));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created datagram (channel=%p) "
"(handle=0x%x:0x%x).\n",
channel,
pageChannel->dgHandle.context,
pageChannel->dgHandle.resource));
/*
* Create a doorbell handle. This is used by the peer to signal the
* arrival of packets in the queuepair. This handle has a delayed
* callback depending on the channel flags.
*/
flags = channelFlags & VPAGECHANNEL_FLAGS_RECV_DELAYED ?
VMCI_FLAG_DELAYED_CB : 0;
retval = vmci_doorbell_create(&pageChannel->doorbellHandle,
flags, VMCI_PRIVILEGE_FLAG_RESTRICTED,
VPageChannelDoorbellCallback, pageChannel);
if (retval < VMCI_SUCCESS) {
VMCI_WARNING((LGPFX"Failed to create doorbell "
"(channel=%p) (err=%d).\n",
channel,
retval));
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created doorbell (channel=%p) "
"(handle=0x%x:0x%x).\n",
channel,
pageChannel->doorbellHandle.context,
pageChannel->doorbellHandle.resource));
/*
* Now create the queuepair, over which we can pass data packets.
*/
retval = VPageChannelCreateQueuePair(pageChannel);
if (retval < VMCI_SUCCESS) {
goto error;
}
/*
* Set the receiving buffers before sending the connection message to
* avoid a race when the connection is made, but there is no receiving
* buffer yet.
*/
if (defaultRecvBuffers) {
int numElems = defaultRecvBuffers + VMCI_PACKET_RECV_THRESHOLD;
if (0 == VPageChannelAddRecvBuffers(pageChannel, numElems, TRUE)) {
/*
* AddRecvBuffers() returns the number of buffers actually added. If
* we failed to add any at all, then fail.
*/
retval = VMCI_ERROR_NO_MEM;
goto error;
}
}
retval = VPageChannelSendConnectionMessage(pageChannel);
if (retval < VMCI_SUCCESS) {
goto error;
}
VMCI_DEBUG_LOG(10,
(LGPFX"Created (channel=%p) (handle=0x%x:0x%x).\n",
pageChannel,
pageChannel->dgHandle.context,
pageChannel->dgHandle.resource));
*channel = pageChannel;
return retval;
error:
VPageChannel_Destroy(pageChannel);
return retval;
}
static void
VPageChannelDoDoorbellCallback(VPageChannel *channel) // IN/OUT
{
Bool inUse;
unsigned long flags;
VPageChannelPacket packetHeader;
ASSERT(channel);
if (VPCState_Connected != channel->state) {
VMCI_WARNING((LGPFX"Not connected (channel=%p).\n",
channel));
return;
}
VPageChannelAcquireRecvLock(channel, &flags);
inUse = channel->inPoll;
channel->inPoll = TRUE;
VPageChannelReleaseRecvLock(channel, flags);
if (inUse) {
return;
}
retry:
while (vmci_qpair_consume_buf_ready(channel->qpair) >= sizeof packetHeader) {
ssize_t retSize, totalSize;
VPageChannelPacket *packet;
retSize = vmci_qpair_peek(channel->qpair, &packetHeader,
sizeof packetHeader,
/* XXX, UTIL_VMKERNEL_BUFFER for VMKernel. */
0);
if (retSize < sizeof packetHeader) {
/*
* XXX, deal with partial read.
*/
VMCI_WARNING((LGPFX"Failed to peek (channel=%p) "
"(required=%"FMTSZ"d) (err=%"FMTSZ"d).\n",
channel,
sizeof packetHeader,
retSize));
break;
}
totalSize = sizeof packetHeader + packetHeader.msgLen +
packetHeader.numElems * sizeof(VPageChannelElem);
retSize = vmci_qpair_consume_buf_ready(channel->qpair);
if (retSize < totalSize) {
/*
* XXX, deal with partial read.
*/
VMCI_WARNING((LGPFX"Received partial packet (channel=%p) "
"(type=%d) (len=%d) (num elems=%d) (avail=%"FMTSZ"d) "
"(requested=%"FMTSZ"d).\n",
channel,
packetHeader.type,
packetHeader.msgLen,
packetHeader.numElems,
retSize,
totalSize));
break;
}
packet = (VPageChannelPacket *)
VMCI_AllocKernelMem(totalSize, VMCI_MEMORY_ATOMIC);
if (!packet) {
VMCI_WARNING((LGPFX"Failed to allocate packet (channel=%p) "
"(size=%"FMTSZ"d).\n",
channel,
totalSize));
break;
}
retSize = vmci_qpair_dequeue(channel->qpair, packet,
totalSize,
/* XXX, UTIL_VMKERNEL_BUFFER for VMKernel. */
0);
if (retSize < totalSize) {
/*
* XXX, deal with partial read.
*/
VMCI_WARNING((LGPFX"Failed to dequeue (channel=%p) "
"(required=%"FMTSZ"d) (err=%"FMTSZ"d).\n",
channel,
totalSize,
retSize));
VMCI_FreeKernelMem(packet, totalSize);
break;
}
VPageChannelRecvPacket(channel, packet);
VMCI_FreeKernelMem(packet, totalSize);
}
VPageChannelAcquireRecvLock(channel, &flags);
/*
* The doorbell may have been notified between when we we finished reading
* data and when we grabbed the lock. If that happens, then there may be
* data, but we bailed out of that second notification because inPoll was
* already set. So that we don't miss anything, do a final check here under
* the lock for any data that might have arrived.
*/
if (vmci_qpair_consume_buf_ready(channel->qpair) >= sizeof packetHeader) {
VPageChannelReleaseRecvLock(channel, flags);
goto retry;
}
channel->inPoll = FALSE;
VPageChannelReleaseRecvLock(channel, flags);
}
