CoreServer::Result CoreServer::test()
{
#ifdef INTEL
if (m_info.coreId != 0)
{
FileSystemMessage msg;
msg.type = ChannelMessage::Request;
msg.action = StatFile;
msg.path = (char *)0x12345678;
msg.size = m_info.coreId;
m_toMaster->write(&msg);
}
else
{
FileSystemMessage msg;
Size numCores = m_cores->getCores().count();
for (Size i = 1; i < numCores; i++)
{
MemoryChannel *ch = (MemoryChannel *) m_fromSlave->get(i);
if (!ch)
return IOError;
// TODO: replace with ChannelClient::syncReceiveFrom
while (ch->read(&msg) != Channel::Success);
if (msg.action == StatFile)
{
NOTICE("core" << i << " send a Ping");
}
}
}
#endif /* INTEL */
return Success;
}
CoreServer::Result CoreServer::setupChannels()
{
#ifdef INTEL
SystemInformation info;
if (info.coreId == 0)
{
Size numCores = m_cores->getCores().count();
m_toSlave = new Index<MemoryChannel>(numCores);
m_fromSlave = new Index<MemoryChannel>(numCores);
for (Size i = 1; i < numCores; i++)
{
MemoryChannel *ch = new MemoryChannel();
CoreInfo *coreInfo = (CoreInfo *) m_coreInfo->get(i);
ch->setMode(Channel::Producer);
ch->setMessageSize(sizeof(FileSystemMessage));
ch->setPhysical(coreInfo->coreChannelAddress + (PAGESIZE * 2),
coreInfo->coreChannelAddress + (PAGESIZE * 3));
m_toSlave->insert(i, *ch);
ch = new MemoryChannel();
ch->setMode(Channel::Consumer);
ch->setMessageSize(sizeof(FileSystemMessage));
ch->setPhysical(coreInfo->coreChannelAddress,
coreInfo->coreChannelAddress + PAGESIZE);
m_fromSlave->insert(i, *ch);
}
}
else
{
m_toMaster = new MemoryChannel();
m_toMaster->setMode(Channel::Producer);
m_toMaster->setMessageSize(sizeof(FileSystemMessage));
m_toMaster->setPhysical(info.coreChannelAddress,
info.coreChannelAddress + PAGESIZE);
m_fromMaster = new MemoryChannel();
m_fromMaster->setMode(Channel::Consumer);
m_fromMaster->setMessageSize(sizeof(FileSystemMessage));
m_fromMaster->setPhysical(info.coreChannelAddress + (PAGESIZE * 2),
info.coreChannelAddress + (PAGESIZE * 3));
}
#endif /* INTEL */
return Success;
}
ChannelClient::Result ChannelClient::connect(ProcessID pid)
{
Address prodAddr, consAddr;
SystemInformation info;
// Allocate consumer
MemoryChannel *cons = new MemoryChannel;
if (!cons)
{
ERROR("failed to allocate consumer MemoryChannel object");
return OutOfMemory;
}
cons->setMessageSize(sizeof(FileSystemMessage));
cons->setMode(Channel::Consumer);
// Allocate producer
MemoryChannel *prod = new MemoryChannel;
if (!prod)
{
ERROR("failed to allocate producer MemoryChannel object");
delete cons;
return OutOfMemory;
}
prod->setMessageSize(sizeof(FileSystemMessage));
prod->setMode(Channel::Producer);
// Call VMShare to create shared memory mapping for MemoryChannel.
ProcessShares::MemoryShare share;
share.pid = pid;
share.coreId = info.coreId;
share.tagId = 0;
share.range.size = PAGESIZE * 4;
share.range.virt = 0;
share.range.phys = 0;
share.range.access = Memory::User | Memory::Readable | Memory::Writable | Memory::Uncached;
// Create shared memory mapping
Error r = VMShare(pid, API::Create, &share);
switch (r)
{
case API::Success:
{
DEBUG("mapped new shared at phys=" << (void *) share.range.phys <<
" virt=" << (void *) share.range.virt);
prodAddr = share.range.virt;
consAddr = share.range.virt + (PAGESIZE * 2);
break;
}
case API::AlreadyExists:
{
DEBUG("using already shared at phys=" << (void *) share.range.phys <<
" virt=" << (void *) share.range.virt);
prodAddr = share.range.virt + (PAGESIZE * 2);
consAddr = share.range.virt;
break;
}
default:
{
ERROR("failed to create shared memory mapping for new channel");
return IOError;
}
}
// Setup producer memory address
if (prod->setVirtual(prodAddr, prodAddr + PAGESIZE) != MemoryChannel::Success)
{
ERROR("failed to set producer virtual memory address");
delete prod;
delete cons;
return IOError;
}
// Setup consumer memory address
if (cons->setVirtual(consAddr, consAddr + PAGESIZE) != MemoryChannel::Success)
{
ERROR("failed to set consumer virtual memory address");
delete prod;
delete cons;
return IOError;
}
// Register channels
m_registry->registerConsumer(pid, cons);
m_registry->registerProducer(pid, prod);
return Success;
}
void CoreServer::createProcess(FileSystemMessage *msg)
{
char cmd[128];
Memory::Range range;
if (m_info.coreId == 0)
{
MemoryChannel *ch = (MemoryChannel *) m_toSlave->get(msg->size);
if (!ch)
{
ERROR("invalid coreId=" << msg->size);
msg->result = EBADF;
return;
}
// TODO:move in libmpi?
range.virt = (Address) msg->buffer;
VMCtl(msg->from, LookupVirtual, &range);
msg->buffer = (char *) range.phys;
range.virt = (Address) msg->path;
VMCtl(msg->from, LookupVirtual, &range);
msg->path = (char *) range.phys;
if (ch->write(msg) != Channel::Success)
{
ERROR("failed to write channel on core"<<msg->size);
msg->result = EBADF;
return;
}
DEBUG("creating program at phys " << (void *) msg->buffer << " on core" << msg->size);
ch = (MemoryChannel *) m_fromSlave->get(msg->size);
if (!ch)
{
ERROR("cannot find read channel for core" << msg->size);
msg->result = EBADF;
return;
}
// TODO: replace with ChannelClient::syncReceiveFrom
while (ch->read(msg) != Channel::Success);
DEBUG("program created with result " << (int)msg->result << " at core" << msg->size);
msg->result = ESUCCESS;
//IPCMessage(msg->from, API::Send, msg, sizeof(*msg));
ChannelClient::instance->syncSendTo(msg, msg->from);
}
else
{
VMCopy(SELF, API::ReadPhys, (Address) cmd, (Address) msg->path, sizeof(cmd));
range.phys = (Address) msg->buffer;
range.virt = 0;
range.access = Memory::Readable | Memory::User;
range.size = msg->offset;
VMCtl(SELF, Map, &range);
pid_t pid = spawn(range.virt, msg->offset, cmd);
int status;
// reply to master
msg->result = ESUCCESS;
while (m_toMaster->write(msg) != Channel::Success);
// TODO: temporary make coreserver waitpid() to save polling time
waitpid(pid, &status, 0);
}
}
int MPI_Init(int *argc, char ***argv)
{
SystemInformation info;
FileSystemMessage msg;
struct stat st;
char *programName = (*argv)[0];
char programPath[64];
u8 *programBuffer;
int fd;
Memory::Range memChannelBase;
// If we are master (node 0):
if (info.coreId == 0)
{
msg.type = ChannelMessage::Request;
msg.action = ReadFile;
msg.from = SELF;
ChannelClient::instance->syncSendReceive(&msg, CORESRV_PID);
// provide -n COUNT, --help and other stuff in here too.
// to influence the launching of more MPI programs
coreCount = msg.size;
// Read our own ELF program to a buffer and pass it to CoreServer
// for creating new programs on the remote core.
if (strncmp(programName, "/bin/", 5) != 0)
snprintf(programPath, sizeof(programPath), "/bin/%s", programName);
else
strlcpy(programPath, programName, sizeof(programPath));
if (stat(programPath, &st) != 0)
{
printf("%s: failed to stat '%s': %s\n",
programName, programPath, strerror(errno));
return MPI_ERR_BAD_FILE;
}
programBuffer = new u8[st.st_size];
MemoryBlock::set(programBuffer, 0, st.st_size);
// Read ELF program
if ((fd = open(programPath, O_RDONLY)) == -1)
{
printf("%s: failed to open '%s': %s\n",
programName, programPath, strerror(errno));
return MPI_ERR_BAD_FILE;
}
if (read(fd, programBuffer, st.st_size) != st.st_size)
{
printf("%s: failed to read '%s': %s\n",
programName, programPath, strerror(errno));
return MPI_ERR_BAD_FILE;
}
if (close(fd) != 0)
{
printf("%s: failed to close '%s': %s\n",
programName, programPath, strerror(errno));
return MPI_ERR_BAD_FILE;
}
// Allocate memory space on the local processor for the whole
// UniChannel array, NxN communication with MPI.
// Then pass the channel offset physical address as an argument -addr 0x.... to spawn()
memChannelBase.size = (PAGESIZE * 2) * (msg.size * msg.size);
memChannelBase.phys = 0;
memChannelBase.virt = 0;
memChannelBase.access = Memory::Readable | Memory::Writable | Memory::User;
if (VMCtl(SELF, Map, &memChannelBase) != API::Success)
{
printf("%s: failed to allocate MemoryChannel\n",
programName);
return MPI_ERR_NO_MEM;
}
printf("%s: MemoryChannel at physical address %x\n",
programName, memChannelBase.phys);
// Clear channel pages
MemoryBlock::set((void *) memChannelBase.virt, 0, memChannelBase.size);
// now create the slaves using coreservers.
for (Size i = 1; i < coreCount; i++)
{
// TODO: check for cmd buffer size...
char *cmd = new char[512];
snprintf(cmd, 512, "%s -a %x -c %d",
programPath, memChannelBase.phys, coreCount);
for (int j = 1; j < *argc; j++)
{
strcat(cmd, " ");
strcat(cmd, (*argv)[j]);
}
msg.type = ChannelMessage::Request;
msg.action = CreateFile;
msg.size = i;
msg.buffer = (char *) programBuffer;
msg.offset = st.st_size;
msg.path = cmd;
ChannelClient::instance->syncSendReceive(&msg, CORESRV_PID);
if (msg.result != ESUCCESS)
{
printf("%s: failed to create process on core%d\n",
programName, i);
return MPI_ERR_SPAWN;
}
}
}
else
{
// If we are slave (node N):
// read the -addr argument, and map the UniChannels into our address space.
for (int i = 1; i < (*argc); i++)
{
if (!strcmp((*argv)[i], "--addr") ||
!strcmp((*argv)[i], "-a"))
{
if ((*argc) < i+1)
return MPI_ERR_ARG;
String s = (*argv)[i+1];
memChannelBase.phys = s.toLong(Number::Hex);
i++;
}
else if (!strcmp((*argv)[i], "--cores") ||
!strcmp((*argv)[i], "-c"))
{
if ((*argc) < i+1)
return MPI_ERR_ARG;
coreCount = atoi((*argv)[i+1]);
i++;
}
// Unknown MPI argument. Pass the rest to the user program.
else
{
(*argc) -= (i-1);
(*argv)[i-1] = (*argv)[0];
(*argv) += (i-1);
break;
}
}
}
// Create MemoryChannels
readChannel = new Index<MemoryChannel>(coreCount);
writeChannel = new Index<MemoryChannel>(coreCount);
// Fill read channels
for (Size i = 0; i < coreCount; i++)
{
MemoryChannel *ch = new MemoryChannel();
ch->setMode(MemoryChannel::Consumer);
ch->setMessageSize(sizeof(MPIMessage));
ch->setPhysical(MEMBASE(info.coreId) + (PAGESIZE * 2 * i),
MEMBASE(info.coreId) + (PAGESIZE * 2 * i) + PAGESIZE);
readChannel->insert(i, *ch);
if (info.coreId == 0)
printf("%s: read: core%d: data=%x feedback=%x base%d=%x\n", (*argv)[0], i,
MEMBASE(info.coreId) + (PAGESIZE * 2 * i),
MEMBASE(info.coreId) + (PAGESIZE * 2 * i) + PAGESIZE,
i, MEMBASE(i));
}
// Fill write channels
for (Size i = 0; i < coreCount; i++)
{
MemoryChannel *ch = new MemoryChannel();
ch->setMode(MemoryChannel::Producer);
ch->setMessageSize(sizeof(MPIMessage));
ch->setPhysical(MEMBASE(i) + (PAGESIZE * 2 * info.coreId),
MEMBASE(i) + (PAGESIZE * 2 * info.coreId) + PAGESIZE);
writeChannel->insert(i, *ch);
if (info.coreId == 0)
printf("%s: write: core%d: data=%x feedback=%x base%d=%x\n", (*argv)[0], i,
MEMBASE(i) + (PAGESIZE * 2 * info.coreId),
MEMBASE(i) + (PAGESIZE * 2 * info.coreId) + PAGESIZE,
i, MEMBASE(i));
}
return MPI_SUCCESS;
}