/* This routine is needed when the application is built on top of the bigemulator
* layer of Charm. In this case, the real CCS handler must be called within a
* worker thread. The function of this function is to receive the CCS message in
* the bottom converse layer and forward it to the emulated layer. */
static void bg_req_fw_handler(char *msg) {
/* Get out of the message who is the destination pe */
int offset = CmiReservedHeaderSize + sizeof(CcsImplHeader);
CcsImplHeader *hdr = (CcsImplHeader *)(msg+CmiReservedHeaderSize);
int destPE = (int)ChMessageInt(hdr->pe);
if (CpvAccess(_bgCcsAck) < BgNodeSize()) {
CcsBufferMessage(msg);
return;
}
//CmiPrintf("CCS scheduling message\n");
if (destPE == -1) destPE = 0;
if (destPE < -1) {
ChMessageInt_t *pes_nbo = (ChMessageInt_t *)(msg+CmiReservedHeaderSize+sizeof(CcsImplHeader));
destPE = ChMessageInt(pes_nbo[0]);
}
//CmiAssert(destPE >= 0); // FixME: should cover also broadcast and multicast -> create generic function to extract destpe
(((CmiBlueGeneMsgHeader*)msg)->tID) = 0;
(((CmiBlueGeneMsgHeader*)msg)->n) = 0;
(((CmiBlueGeneMsgHeader*)msg)->flag) = 0;
(((CmiBlueGeneMsgHeader*)msg)->t) = 0;
(((CmiBlueGeneMsgHeader*)msg)->hID) = CpvAccess(_bgCcsHandlerIdx);
/* Get the right thread to deliver to (for now assume it is using CyclicMapInfo) */
addBgNodeInbuffer(msg, destPE/CmiNumPes());
//CmiPrintf("message CCS added %d to %d\n",((CmiBlueGeneMsgHeader*)msg)->hID, ((CmiBlueGeneMsgHeader*)msg)->tID);
}
/**
* This is the entrance point of a CCS request into the server.
* It is executed only on proc 0, and it forwards the request to the appropriate PE.
*/
void CcsImpl_netRequest(CcsImplHeader *hdr,const void *reqData)
{
char *msg;
int len,repPE=ChMessageInt(hdr->pe);
if (repPE<=-CmiNumPes() || repPE>=CmiNumPes()) {
#if ! CMK_BIGSIM_CHARM
/*Treat out of bound values as errors. Helps detecting bugs*/
if (repPE==-CmiNumPes()) CmiPrintf("Invalid processor index in CCS request: are you trying to do a broadcast instead?");
else CmiPrintf("Invalid processor index in CCS request.");
CpvAccess(ccsReq)=hdr;
CcsSendReply(0,NULL); /*Send an empty reply to the possibly waiting client*/
return;
#endif
}
msg=CcsImpl_ccs2converse(hdr,reqData,&len);
if (repPE >= 0) {
/* The following %CmiNumPes() follows the assumption that in BigSim the mapping is round-robin */
//CmiPrintf("CCS message received for %d\n",repPE);
CmiSyncSendAndFree(repPE%CmiNumPes(),len,msg);
} else if (repPE == -1) {
/* Broadcast to all processors */
//CmiPrintf("CCS broadcast received\n");
CmiSyncSendAndFree(0,len,msg);
} else {
/* Multicast to -repPE processors, specified right at the beginning of reqData (as a list of pes) */
int firstPE = ChMessageInt(*(ChMessageInt_t*)reqData);
/* The following %CmiNumPes() follows the assumption that in BigSim the mapping is round-robin */
//CmiPrintf("CCS multicast received\n");
CmiSyncSendAndFree(firstPE%CmiNumPes(),len,msg);
}
}
/********************************************************
Authenticate incoming request for a client salt value.
Exchange looks like:
1.) Client sends request code 0x80 (SHA-1), 0x00 (version 0),
0x01 (create salt), 0xNN (security level); followed by
client challenge (4 bytes, s1)
2.) Server replies with server challenge (4 bytes, s2)
3.) Client replies with hashed key & server challenge (20 bytes, s2hash)
4.) Server replies with hashed key & client challenge (20 bytes, s1hash),
as well as client identifier and initial client salt. (8 bytes total).
*/
static const char *CcsServer_createSalt(SOCKET fd,CCS_AUTH_clients *cl,
CcsSecMan *security,CcsSecAttr *attr)
{
ChMessageInt_t s1;
ChMessageInt_t s2=ChMessageInt_new(CCS_RAND_next(&cl->rand));
SHA1_hash_t s2hash;
int clientId;
struct {
SHA1_hash_t s1hash;
ChMessageInt_t clientId;
ChMessageInt_t clientSalt;
} reply;
if (-1==skt_recvN(fd,&s1,sizeof(s1))) return "ERROR> CreateSalt challenge recv";
if (-1==skt_sendN(fd,&s2,sizeof(s2))) return "ERROR> CreateSalt challenge send";
if (-1==skt_recvN(fd,&s2hash,sizeof(s2hash))) return "ERROR> CreateSalt reply recv";
if (CCS_AUTH_differ(security->getKey(security,attr),ChMessageInt(s2),
NULL,&s2hash))
return "ERROR> CreateSalt client hash mismatch! (bad password?)";
CCS_AUTH_hash(security->getKey(security,attr),ChMessageInt(s1),
NULL,&reply.s1hash);
clientId=CCS_AUTH_addClient(cl);
reply.clientId=ChMessageInt_new(clientId);
reply.clientSalt=ChMessageInt_new(CCS_AUTH_clientSalt(cl,clientId));
if (-1==skt_sendN(fd,&reply,sizeof(reply))) return "ERROR> CreateSalt reply send";
/*HACK: this isn't an error return, and returning an error code
here is wrong; but all we want is to close the socket (not process
a CCS request), and printing out this text isn't a bad idea, so...
*/
return "Created new client";
}
static int CcsServer_recvRequestData(SOCKET fd,
CcsImplHeader *hdr,void **reqData)
{
CcsMessageHeader req;/*CCS header, from requestor*/
int reqBytes, numPes, destPE;
const char *err;
if (NULL!=(err=CcsServer_readHeader(fd,&ccs_clientlist,security,
&hdr->attr,&req)))
{ /*Not a regular message-- write error message and return error.*/
fprintf(stdout,"CCS %s\n",err);
return 0;
}
/*Fill out the internal CCS header*/
strncpy(hdr->handler,req.handler,CCS_MAXHANDLER);
hdr->pe=req.pe;
hdr->len=req.len;
hdr->replyFd=ChMessageInt_new(fd);
/*Is it a multicast?*/
numPes = 0;
destPE = ChMessageInt(hdr->pe);
if (destPE < -1) numPes = -destPE;
/*Grab the user data portion of the message*/
reqBytes=ChMessageInt(req.len) + numPes*sizeof(ChMessageInt_t);
*reqData=(char *)malloc(reqBytes);
if (-1==skt_recvN(fd,*reqData,reqBytes)) {
fprintf(stdout,"CCS ERROR> Retrieving %d message bytes\n",reqBytes);
free(*reqData);
return 0;
}
return 1;
}
extern "C" void req_fw_handler(char *msg)
{
int offset = CmiReservedHeaderSize + sizeof(CcsImplHeader);
CcsImplHeader *hdr = (CcsImplHeader *)(msg+CmiReservedHeaderSize);
int destPE = (int)ChMessageInt(hdr->pe);
if (CmiMyPe() == 0 && destPE == -1) {
/* Broadcast message to all other processors */
int len=CmiReservedHeaderSize+sizeof(CcsImplHeader)+ChMessageInt(hdr->len);
CmiSyncBroadcast(len, msg);
}
else if (destPE < -1) {
/* Multicast the message to your children */
int len=CmiReservedHeaderSize+sizeof(CcsImplHeader)+ChMessageInt(hdr->len)-destPE*sizeof(ChMessageInt_t);
int index, child, i;
int *pes = (int*)(msg+CmiReservedHeaderSize+sizeof(CcsImplHeader));
ChMessageInt_t *pes_nbo = (ChMessageInt_t *)pes;
offset -= destPE * sizeof(ChMessageInt_t);
if (ChMessageInt(pes_nbo[0]) == CmiMyPe()) {
for (index=0; index<-destPE; ++index) pes[index] = ChMessageInt(pes_nbo[index]);
}
for (index=0; index<-destPE; ++index) {
if (pes[index] == CmiMyPe()) break;
}
child = (index << 2) + 1;
for (i=0; i<4; ++i) {
if (child+i < -destPE) {
CmiSyncSend(pes[child+i], len, msg);
}
}
}
CcsHandleRequest(hdr, msg+offset);
CmiFree(msg);
}
/* initnode node table reply format:
+-------------------------------------------------------
| 4 bytes | Number of nodes n ^
| | (big-endian binary integer) 4+12*n bytes
+------------------------------------------------- |
^ | (one entry for each node) ^ |
| | 4 bytes | Number of PEs for this node | |
n | 4 bytes | IP address of this node 12*n bytes |
| | 4 bytes | Data (UDP) port of this node | |
v | | (big-endian binary integers) v v
---+----------------------------------------------------
*/
static void node_addresses_store(ChMessage *msg)
{
ChMessageInt_t *n32=(ChMessageInt_t *)msg->data;
ChNodeinfo *d=(ChNodeinfo *)(n32+1);
int nodestart;
int i,j,n;
MACHSTATE(1,"node_addresses_store {");
_Cmi_numnodes=ChMessageInt(n32[0]);
if ((sizeof(ChMessageInt_t)+sizeof(ChNodeinfo)*_Cmi_numnodes)
!=(unsigned int)msg->len)
{printf("Node table has inconsistent length!");machine_exit(1);}
nodes = (OtherNode)malloc(_Cmi_numnodes * sizeof(struct OtherNodeStruct));
nodestart=0;
for (i=0; i<_Cmi_numnodes; i++) {
nodes[i].nodestart = nodestart;
nodes[i].nodesize = ChMessageInt(d[i].nPE);
MACHSTATE2(3,"node %d nodesize %d",i,nodes[i].nodesize);
nodes[i].mach_id = ChMessageInt(d[i].mach_id);
nodes[i].IP=d[i].IP;
if (i==_Cmi_mynode) {
Cmi_nodestart=nodes[i].nodestart;
_Cmi_mynodesize=nodes[i].nodesize;
Cmi_self_IP=nodes[i].IP;
}
nodes[i].dataport = ChMessageInt(d[i].dataport);
nodes[i].addr = skt_build_addr(nodes[i].IP,nodes[i].dataport);
#if CMK_USE_TCP
nodes[i].sock = INVALID_SOCKET;
#endif
nodestart+=nodes[i].nodesize;
}
_Cmi_numpes=nodestart;
n = _Cmi_numpes;
#ifdef CMK_CPV_IS_SMP
n += _Cmi_numnodes;
#endif
nodes_by_pe = (OtherNode*)malloc(n * sizeof(OtherNode));
_MEMCHECK(nodes_by_pe);
for (i=0; i<_Cmi_numnodes; i++) {
OtherNode node = nodes + i;
OtherNode_init(node);
for (j=0; j<node->nodesize; j++) {
nodes_by_pe[j + node->nodestart] = node;
}
}
#ifdef CMK_CPV_IS_SMP
/* index for communication threads */
for (i=_Cmi_numpes; i<_Cmi_numpes+_Cmi_numnodes; i++) {
OtherNode node = nodes + i-_Cmi_numpes;
nodes_by_pe[i] = node;
}
#endif
MACHSTATE(1,"} node_addresses_store");
}
CcsDelayedReply CcsDelayReply(void)
{
CcsDelayedReply ret;
int len = sizeof(CcsImplHeader);
if (ChMessageInt(CpvAccess(ccsReq)->pe) < -1)
len += ChMessageInt(CpvAccess(ccsReq)->pe) * sizeof(int);
ret.hdr = (CcsImplHeader*)malloc(len);
memcpy(ret.hdr, CpvAccess(ccsReq), len);
CpvAccess(ccsReq)=NULL;
return ret;
}
/**
* Decide if the reply is ready to be forwarded to the waiting client,
* or if combination is required (for broadcast/multicast CCS requests.
*/
extern "C" int CcsReply(CcsImplHeader *rep,int repLen,const void *repData) {
int repPE = (int)ChMessageInt(rep->pe);
if (repPE <= -1) {
/* Reduce the message to get the final reply */
CcsHandlerRec *fn;
int len=CmiReservedHeaderSize+sizeof(CcsImplHeader)+repLen;
char *msg=(char*)CmiAlloc(len);
char *r=msg+CmiReservedHeaderSize;
char *handlerStr;
rep->len = ChMessageInt_new(repLen);
*(CcsImplHeader *)r=*rep; r+=sizeof(CcsImplHeader);
memcpy(r,repData,repLen);
CmiSetHandler(msg,rep_fw_handler_idx);
handlerStr=rep->handler;
fn=(CcsHandlerRec *)CcsGetHandler(handlerStr);
if (fn->mergeFn == NULL) CmiAbort("Called CCS broadcast with NULL merge function!\n");
if (repPE == -1) {
/* CCS Broadcast */
CkReduce(msg, len, fn->mergeFn);
} else {
/* CCS Multicast */
CmiListReduce(-repPE, (int*)(rep+1), msg, len, fn->mergeFn, fn->redID);
}
} else {
if (_conditionalDelivery == 0) CcsImpl_reply(rep, repLen, repData);
else {
/* We are the child of a conditional delivery, write to the parent the reply */
write(conditionalPipe[1], &repLen, 4);
write(conditionalPipe[1], repData, repLen);
}
}
return 0;
}
/***********************************************
Send the given data as a CCS reply on the given socket.
Goes to some effort to minimize the number of "sends"
(and hence outgoing TCP packets) by assembling the
header and data in-place.
*/
static void CcsServer_writeReply(SOCKET fd,
CcsSecMan *security,
CcsSecAttr *attr,
int replyLen,char *reply)
{
const void *bufs[3]; int lens[3]; int nBuffers=0;
struct { /*Authentication header*/
SHA1_hash_t hash;
} aheader;
struct { /*Reply header*/
ChMessageInt_t len;
} header;
if (attr->auth==1)
{ /*Compose a reply SHA-1 hash header*/
CCS_AUTH_hash(security->getKey(security,attr),
ChMessageInt(attr->replySalt),NULL,&aheader.hash);
bufs[nBuffers]=&aheader; lens[nBuffers]=sizeof(aheader); nBuffers++;
}
/*Compose a simple reply header*/
header.len=ChMessageInt_new(replyLen);
bufs[nBuffers]=&header; lens[nBuffers]=sizeof(header); nBuffers++;
bufs[nBuffers]=reply; lens[nBuffers]=replyLen; nBuffers++;
if (-1==skt_sendV(fd,nBuffers,bufs,lens)) return;
skt_close(fd);
#undef n
}
/*Send a Ccs reply down the given socket.
Closes the socket afterwards.
A CcsImplHeader len field equal to 0 means do not send any reply.
*/
void CcsServer_sendReply(CcsImplHeader *hdr,int repBytes,const void *repData)
{
int fd=ChMessageInt(hdr->replyFd);
skt_abortFn old;
if (ChMessageInt(hdr->len)==0) {
CCSDBG(("CCS Closing reply socket without a reply.\n"));
skt_close(fd);
return;
}
old=skt_set_abort(reply_abortFn);
CCSDBG(("CCS Sending %d bytes of reply data\n",repBytes));
CcsServer_writeReply(fd,security,&hdr->attr,repBytes,(char *)repData);
skt_close(fd);
CCSDBG(("CCS Reply socket closed.\n"));
skt_set_abort(old);
}
/*******************
Grab an ordinary authenticated message off this socket.
The format is:
-4 byte client ID number (returned by createSalt earlier)
-4 byte client challenge (used to by client to authenticate reply)
-20 byte authentication hash code
-Regular CcsMessageHeader
*/
static const char *CcsServer_SHA1_message(SOCKET fd,CCS_AUTH_clients *cl,
CcsSecMan *security,CcsSecAttr *attr,
CcsMessageHeader *hdr)
{
ChMessageInt_t clientNo_net;
int clientNo;
unsigned int salt;
SHA1_hash_t hash;
/* An ordinary authenticated message */
if (-1==skt_recvN(fd,&clientNo_net,sizeof(clientNo_net)))
return "ERROR> During recv. client number";
if (-1==skt_recvN(fd,&attr->replySalt,sizeof(attr->replySalt)))
return "ERROR> During recv. reply salt";
if (-1==skt_recvN(fd,&hash,sizeof(hash)))
return "ERROR> During recv. authentication hash";
if (-1==skt_recvN(fd,hdr,sizeof(CcsMessageHeader)))
return "ERROR> During recv. message header";
clientNo=ChMessageInt(clientNo_net);
if (clientNo<0 || clientNo>=CCS_AUTH_numClients(cl))
return "ERROR> Bad client number in SHA-1 request!";
salt=CCS_AUTH_clientSalt(cl,clientNo);
/*Check the client's hash*/
if (CCS_AUTH_differ(security->getKey(security,attr),salt,
hdr,&hash))
return "ERROR> Authentication hash code MISMATCH-- bad or faked key";
CCS_AUTH_advanceSalt(cl,clientNo);
return NULL; /*It's a good message*/
}
//Read a list contents request header:
// first item to send, 4-byte network integer
// last item+1 to send, 4-byte network integer
// extra data length, 4-byte network integer
// extra data, list-defined bytes
// list path length, 4-byte network integer (character count)
// list path name, null-terminated ASCII
static CpdListAccessor *CpdListHeader_ccs_list_items(char *msg,
CpdListItemsRequest &h)
{
int msgLen=CmiSize((void *)msg)-CmiReservedHeaderSize;
CpdListAccessor *ret=NULL;
const ChMessageInt_t *req=(const ChMessageInt_t *)(msg+CmiReservedHeaderSize);
h.lo=ChMessageInt(req[0]); // first item to send
h.hi=ChMessageInt(req[1]); // last item to send+1
h.extraLen=ChMessageInt(req[2]); // extra data length
if (h.extraLen>=0
&& ((int)(3*sizeof(ChMessageInt_t)+h.extraLen))<msgLen) {
h.extra=(void *)(req+3); // extra data
ret=CpdListLookup((ChMessageInt_t *)(h.extraLen+(char *)h.extra));
if (ret!=NULL) CpdListBoundsCheck(ret,h.lo,h.hi);
}
return ret;
}
int ChMessageHeader_recv(SOCKET fd,ChMessage *dst)
{
/*Get the binary header*/
if (0!=skt_recvN(fd,(char *)&dst->header,sizeof(dst->header))) return -1;
/*Allocate a recieve buffer*/
dst->len=ChMessageInt(dst->header.len);
dst->data=0;
return 0;
}
/*Receives reply messages passed up from
converse to node 0.*/
static void rep_fw_handler(char *msg)
{
int len;
char *r=msg+CmiReservedHeaderSize;
CcsImplHeader *hdr=(CcsImplHeader *)r;
r+=sizeof(CcsImplHeader);
len=ChMessageInt(hdr->len);
CcsImpl_reply(hdr,len,r);
CmiFree(msg);
}
static void ccs_killport(char *msg)
{
killPortStruct *oldList=killList;
int port=ChMessageInt(*(ChMessageInt_t *)(msg+CmiReservedHeaderSize));
skt_ip_t ip;
unsigned int connPort;
CcsCallerId(&ip,&connPort);
killList=(killPortStruct *)malloc(sizeof(killPortStruct));
killList->ip=ip;
killList->port=port;
killList->next=oldList;
CmiFree(msg);
}
/**
Return a CpdListAccessor, given a network string containing the
list path. A network string is a big-endian 32-bit "length"
field, followed by a null-terminated ASCII string of that length.
*/
static CpdListAccessor *CpdListLookup(const ChMessageInt_t *lenAndPath)
{
static const int CpdListMaxLen=80;
int len=ChMessageInt(lenAndPath[0]);
const char *path=(const char *)(lenAndPath+1);
char pathBuf[CpdListMaxLen+1]; //Temporary null-termination buffer
if ((len<0) || (len>CpdListMaxLen)) {
CmiError("CpdListAccessor> Invalid list path length %d!\n",len);
return NULL; //Character count is invalid
}
strncpy(pathBuf,path,len);
pathBuf[len]=0; //Ensure string is null-terminated
return CpdListLookup(pathBuf);
}
void * CcsMerge_concat(int *size,void *local,void **remote,int n) {
CcsImplHeader *hdr;
int total = *size;
void *reply;
char *ptr;
int i;
for (i=0; i<n; ++i) {
hdr = (CcsImplHeader*)(((char*)remote[i])+CmiReservedHeaderSize);
total += ChMessageInt(hdr->len);
}
reply = CmiAlloc(total);
memcpy(reply, local, *size);
((CcsImplHeader*)(((char*)reply)+CmiReservedHeaderSize))->len = ChMessageInt_new(total-CmiReservedHeaderSize-sizeof(CcsImplHeader));
CmiFree(local);
ptr = ((char*)reply)+*size;
for (i=0; i<n; ++i) {
int len = ChMessageInt(((CcsImplHeader*)(((char*)remote[i])+CmiReservedHeaderSize))->len);
memcpy(ptr, ((char*)remote[i])+CmiReservedHeaderSize+sizeof(CcsImplHeader), len);
ptr += len;
}
*size = total;
return reply;
}
/*Convert CCS header & message data into a converse message
addressed to handler*/
char *CcsImpl_ccs2converse(const CcsImplHeader *hdr,const void *data,int *ret_len)
{
int reqLen=ChMessageInt(hdr->len);
int destPE = ChMessageInt(hdr->pe);
int len;
char *msg;
if (destPE < -1) reqLen -= destPE*sizeof(int);
len=CmiReservedHeaderSize+sizeof(CcsImplHeader)+reqLen;
msg=(char *)CmiAlloc(len);
memcpy(msg+CmiReservedHeaderSize,hdr,sizeof(CcsImplHeader));
memcpy(msg+CmiReservedHeaderSize+sizeof(CcsImplHeader),data,reqLen);
if (ret_len!=NULL) *ret_len=len;
if (_ccsHandlerIdx != 0) {
CmiSetHandler(msg, _ccsHandlerIdx);
return msg;
} else {
#if NODE_0_IS_CONVHOST
CmiAbort("Why do we need to buffer messages when node 0 is Convhost?");
#else
CcsBufferMessage(msg);
return NULL;
#endif
}
}
/*CCS Bottleneck:
Deliver the given message data to the given
CCS handler.
*/
void CcsHandleRequest(CcsImplHeader *hdr,const char *reqData)
{
char *cmsg;
int reqLen=ChMessageInt(hdr->len);
/*Look up handler's converse ID*/
char *handlerStr=hdr->handler;
CcsHandlerRec *fn=(CcsHandlerRec *)CkHashtableGet(CpvAccess(ccsTab),(void *)&handlerStr);
if (fn==NULL) {
CmiPrintf("CCS: Unknown CCS handler name '%s' requested. Ignoring...\n",
hdr->handler);
CpvAccess(ccsReq)=hdr;
CcsSendReply(0,NULL); /*Send an empty reply to the possibly waiting client*/
return;
/* CmiAbort("CCS: Unknown CCS handler name.\n");*/
}
/* Call the handler */
CpvAccess(ccsReq)=hdr;
#if CMK_CHARMDEBUG
if (conditionalPipe[1]!=0 && _conditionalDelivery==0) {
/* We are conditionally delivering, the message has been sent to the child, wait for its response */
int bytes;
if (4==read(conditionalPipe[0], &bytes, 4)) {
char *buf = malloc(bytes);
read(conditionalPipe[0], buf, bytes);
CcsSendReply(bytes,buf);
free(buf);
} else {
/* the pipe has been closed */
CpdEndConditionalDeliver_master();
}
}
else
#endif
{
callHandlerRec(fn,reqLen,reqData);
/*Check if a reply was sent*/
if (CpvAccess(ccsReq)!=NULL)
CcsSendReply(0,NULL);/*Send an empty reply if not*/
}
}
void CcsCallerId(skt_ip_t *pip, unsigned int *pport)
{
*pip = CpvAccess(ccsReq)->attr.ip;
*pport = ChMessageInt(CpvAccess(ccsReq)->attr.port);
}
