void acptboot_getconn(struct work_struct *work)
{
mic_ctx_t *node_ctx;
struct scif_portID data;
scif_epd_t conn_epd;
struct timespec tod;
int proto;
int version;
int err;
if ((err = scif_accept(acptboot_data->listen_epd, &data, &conn_epd,
SCIF_ACCEPT_SYNC))) {
pr_debug("ACPTBOOT: scif_accept_failed %d\n", err);
return;
//goto requeue_accept;
}
if (!data.node) {
printk(KERN_ERR "ACPTBOOT: connect received from invalid dev %d\n",
-EINVAL);
goto close_epd;
}
if ((err = scif_recv(conn_epd, &version, sizeof(version), SCIF_RECV_BLOCK)) != sizeof(version)) {
printk(KERN_ERR "ACPTBOOT: failed to recieve version number err %d\n", err);
goto close_epd;
}
if ((err = scif_recv(conn_epd, &proto, sizeof(proto), SCIF_RECV_BLOCK)) != sizeof(proto)) {
printk(KERN_ERR "ACPTBOOT: failed to recieve proto id %d\n", err);
goto close_epd;
}
switch (proto) {
case ACPT_BOOTED:
node_ctx = get_per_dev_ctx(data.node - 1);
mic_setstate(node_ctx, MIC_ONLINE);
node_ctx->boot_count++;
proto = ACPT_BOOT_ACK;
scif_send(conn_epd, &proto, sizeof(proto), SCIF_SEND_BLOCK);
break;
case ACPT_REQUEST_TIME:
getnstimeofday(&tod);
proto = ACPT_TIME_DATA;
scif_send(conn_epd, &proto, sizeof(proto), SCIF_SEND_BLOCK);
scif_send(conn_epd, &tod, sizeof(tod), SCIF_SEND_BLOCK);
break;
}
close_epd:
if ((err = scif_close(conn_epd)))
printk(KERN_ERR "ACPTBOOT: scif_close failed %d\n", err);
//requeue_accept:
queue_work(acptboot_data->acptbootwq, &acptboot_data->acptbootwork);
}
int main( )
{
uint8_t len = 1;
uint64_t loops = 50000;
scif_epd_t endpoint;
struct scif_portID portid;
int ret;
endpoint = scif_open( );
if( endpoint == SCIF_OPEN_FAILED )
{
printf("scif open failed\n");
return 1;
}
ret = scif_bind(endpoint, 23955);
if(ret==-1)
{
printf("scif_bind failed");
return 1;
}
portid.node = 0;
portid.port = 23968;
ret = scif_connect(endpoint, &portid);
for( int attempt = 0; ret == -1 && attempt < 10; ++attempt )
{
sleep(1);
ret = scif_connect(endpoint, &portid);
}
if (ret==-1)
{
printf("scif_connect failed\n");
return 1;
}
uint8_t* buffer = malloc(len * sizeof(uint8_t));
struct timespec start_latency, stop_latency;
clock_gettime(CLOCK_REALTIME, &start_latency);
for (uint64_t i = 0; i<loops; i++)
{
scif_recv(endpoint, (void*)buffer, len, SCIF_RECV_BLOCK);
scif_send(endpoint, &buffer[0], len, SCIF_SEND_BLOCK);
}
clock_gettime(CLOCK_REALTIME, & stop_latency);
double time_latency = (stop_latency.tv_sec - start_latency.tv_sec) + (stop_latency.tv_nsec - start_latency.tv_nsec) / NANOSECONDS;
double result = (time_latency/(2*loops));
printf("%1f\n", result);
free(buffer);
scif_close(endpoint);
}
static inline int mca_btl_scif_ep_connect_start_active (mca_btl_base_endpoint_t *ep) {
int rc = OPAL_SUCCESS;
BTL_VERBOSE(("initiaiting connection to remote peer %d with port: %u on local scif node: %u",
ep->peer_proc->proc_name.vpid, ep->port_id.port, ep->port_id.node));
opal_mutex_lock (&ep->lock);
do {
if (MCA_BTL_SCIF_EP_STATE_INIT != ep->state) {
/* the accept thread has already finished this connection */
rc = OPAL_SUCCESS;
break;
}
ep->state = MCA_BTL_SCIF_EP_STATE_CONNECTING;
ep->scif_epd = scif_open ();
if (OPAL_UNLIKELY(SCIF_OPEN_FAILED == ep->scif_epd)) {
BTL_VERBOSE(("error creating new scif endpoint"));
rc = OPAL_ERROR;
break;
}
rc = scif_connect (ep->scif_epd, &ep->port_id);
if (OPAL_UNLIKELY(-1 == rc)) {
/* the connection attempt failed. this could mean the peer is currently
* processing connections. we will to try again later. */
BTL_VERBOSE(("error connecting to scif peer. %d", errno));
rc = OPAL_ERR_RESOURCE_BUSY;
break;
}
rc = scif_send (ep->scif_epd, &OPAL_PROC_MY_NAME, sizeof (OPAL_PROC_MY_NAME), SCIF_SEND_BLOCK);
if (OPAL_UNLIKELY(-1 == rc)) {
BTL_VERBOSE(("error in scif_send"));
rc = OPAL_ERROR;
break;
}
/* build connection data */
rc = mca_btl_scif_ep_connect_finish (ep, false);
} while (0);
if (OPAL_SUCCESS != rc) {
scif_close (ep->scif_epd);
ep->scif_epd = -1;
ep->state = MCA_BTL_SCIF_EP_STATE_INIT;
}
opal_mutex_unlock (&ep->lock);
return rc;
}
int
pm_send_to_host(PM_MESSAGE opcode, void *msg, size_t len)
{
// FUNCTION_ENTRY;
int err = 0;
size_t psize = sizeof(pm_msg_header) + len;
char *payload;
unsigned long flags;
if (pm_scif->con_state != PM_CONNECTED) {
err = -EINVAL;
goto error;
}
if (!(payload = kmalloc(psize, GFP_ATOMIC))) {
err = -ENOMEM;
goto error;
}
read_lock_irqsave(&pmscif_send,flags);
if (atomic_xchg(&epinuse,1) != 0) {
read_unlock_irqrestore(&pmscif_send,flags);
kfree(payload);
return -1;
}
((pm_msg_header*)payload)->opcode = opcode;
((pm_msg_header*)payload)->len = len;
if (len)
memcpy((char*)payload + sizeof(pm_msg_header), msg, len);
//0 for non blocking
if ((err = scif_send(pm_scif->ep, payload, psize, 0)) < 0) {
PM_DB("scif_recv failed\n");
}
atomic_set(&epinuse,0);
//for (i = 0; i < psize; i++)
// printk(KERN_ALERT" buff: %X\n", payload[i]);
read_unlock_irqrestore(&pmscif_send,flags);
kfree(payload);
// FUNCTION_EXIT;
error:
return err;
}
int main(int argc, char *argv[])
{
scif_epd_t epd;
int bytes_sent, bytes_received, count;
/* message state related variables */
int *id, *type, *threads;
size_t message_size, request, request_size;
void *message, *params;
/* do the standard open, bind, connect in SCIF */
epd = scif_obc();
/* Create request */
request_size = sizeof(size_t);
request = 1;
printf("= About to send %zd bytes\n", request_size);
/* Send message */
bytes_sent = scif_send(epd, &request, request_size, 1);
printf("= Sent %d bytes\n= Waiting for reply ...\n", bytes_sent);
/* Receive size of the reply */
bytes_received = scif_recv(epd, &message_size, sizeof(size_t), 1);
printf("= Received %d bytes. Expecting a message of size %zu bytes\n",
bytes_received, message_size);
/* Receive the actual reply */
message = malloc(message_size);
count = scif_recv(epd, message, message_size, 1);
bytes_received += count;
printf("= Received %d bytes. Total bytes received: %d bytes\n",
count, bytes_received);
/*
* Extract the message received:
* --------------------------------------------------
* | type (int) | threads (int) | id (int) | params |
* --------------------------------------------------
*/
type = (int *) message;
threads = type + 1;
id = threads + 1;
params = (void *) ((char *) id + sizeof(int));
printf("= Content size: %zu bytes - Type: %d - Threads: %d - ID: %d\n",
message_size, *type, *threads, *id);
/* output results */
int matrix_width, matrix_size;
float *result;
switch(*type) {
case 1:
printf("= Sleep duration left: %u\n", *((unsigned int *) params));
break;
case 2:
matrix_width = *((int *) params);
result = (float *) (((char *) params) + sizeof(int));
print_mtx(result, matrix_width);
break;
case 3: case 4: case 6:
matrix_width = *((int *) params);
matrix_size = matrix_width * matrix_width;
result = (float *) (((char *) params) + sizeof(int)
+ 2 * matrix_size * sizeof(float));
print_mtx(result, matrix_width);
break;
case 5:
matrix_width = *((int *) params + 1);
matrix_size = matrix_width * matrix_width;
result = (float *) (((char *) params) + 2 * sizeof(int)
+ 2 * matrix_size * sizeof(float));
print_mtx(result, matrix_width);
break;
default:
printf("= Dat shit cray!\n");
break;
}
free(message);
if (scif_close(epd) != 0) {
fprintf(stderr, "scif_close failed with error %d\n", errno);
exit(EXIT_FAILURE);
}
printf("= scif_close success\n");
return EXIT_SUCCESS;
}
void *
mic_credentials(void *arg)
{
struct mic_info *mic;
struct mpssd_info *mpssdi;
struct jobs *job;
struct jobs *jlist;
struct scif_portID portID;
struct passwd *pass;
char *username = NULL;
char cookie[MPSS_COOKIE_SIZE];
int len;
unsigned int proto;
scif_epd_t lep;
scif_epd_t dep;
uid_t uid;
int err;
if ((lep = scif_open()) < 0) {
mpsslog(PINFO, "Cannot open mpssd credentials SCIF listen port: %s\n",
strerror(errno));
pthread_exit((void *)1);
}
if (scif_bind(lep, MPSSD_CRED) < 0) {
mpsslog(PINFO, "Cannot bind to mpssd credentials SCIF PORT: %s\n", strerror(errno));
pthread_exit((void *)1);
}
if (scif_listen(lep, 16) < 0) {
mpsslog(PINFO, "Set Listen on mpssd credentials SCIF PORT fail: %s\n", strerror(errno));
pthread_exit((void *)1);
}
while (1) {
if (scif_accept(lep, &portID, &dep, SCIF_ACCEPT_SYNC)) {
if (errno != EINTR) {
mpsslog(PINFO, "Wait for credentials request fail: %s\n", strerror(errno));
scif_close(dep);
}
continue;
}
if ((err = scif_recv(dep, &uid, sizeof(uid), SCIF_RECV_BLOCK)) != sizeof(uid)) {
mpsslog(PINFO, "Credential connect recieve error %s\n", strerror(errno));
scif_close(dep);
continue;
}
username = NULL;
while ((pass = getpwent()) != NULL) {
if (uid == pass->pw_uid) {
username = pass->pw_name;
break;
}
}
endpwent();
if (username == NULL) {
mpsslog(PERROR, "User request unknown UID %d\n", uid);
proto = CRED_FAIL_UNKNOWNUID;
scif_send(dep, &proto, sizeof(proto), 0);
scif_close(dep);
continue;
};
if (get_cookie(pass, cookie) < 0) {
proto = CRED_FAIL_READCOOKIE;
scif_send(dep, &proto, sizeof(proto), 0);
scif_close(dep);
continue;
}
if ((job = malloc(sizeof(struct jobs))) == NULL) {
proto = CRED_FAIL_MALLOC;
scif_send(dep, &proto, sizeof(proto), 0);
scif_close(dep);
continue;
}
job->jobid = nextjobid++;
job->dep = dep;
job->cnt = 0;
len = strlen(username);
while (pthread_mutex_lock(&jobs_lock) != 0);
for (mic = miclist; mic != NULL; mic = mic->next) {
mpssdi = (struct mpssd_info *)mic->data;
if (mpssdi->send_ep != -1) {
job->cnt++;
proto = REQ_CREDENTIAL;
if ((scif_send(mpssdi->send_ep, &proto, sizeof(proto), 0)) < 0) {
if (errno == ECONNRESET) {
job->cnt--;
continue;
}
}
scif_send(mpssdi->send_ep, &job->jobid, sizeof(job->jobid), 0);
scif_send(mpssdi->send_ep, &len, sizeof(len), 0);
scif_send(mpssdi->send_ep, username, len, 0);
len = sizeof(cookie);
scif_send(mpssdi->send_ep, &len, sizeof(len), 0);
scif_send(mpssdi->send_ep, cookie, len, SCIF_SEND_BLOCK);
}
}
if (job->cnt == 0) {
proto = CRED_SUCCESS;
scif_send(job->dep, &proto, sizeof(proto), 0);
scif_close(job->dep);
} else {
jlist = &gjobs;
while (jlist->next)
jlist = jlist->next;
jlist->next = job;
job->next = NULL;
}
while (pthread_mutex_unlock(&jobs_lock) != 0);
}
}
void *
mic_monitor(void *arg)
{
struct mic_info *mic;
struct mpssd_info *mpssdi;
pthread_attr_t attr;
struct scif_portID sendID = {0, MPSSD_MONSEND};
struct scif_portID recvID;
scif_epd_t lep;
scif_epd_t recv_ep;
scif_epd_t send_ep;
unsigned int proto;
uint16_t send_port;
uint16_t remote_port = 0;
int err;
if ((lep = scif_open()) < 0) {
mpsslog(PINFO, "Cannot open mpssd monitor SCIF listen port: %s\n", strerror(errno));
pthread_exit((void *)1);
}
if (scif_bind(lep, MPSSD_MONRECV) < 0) {
mpsslog(PINFO, "Cannot bind to mpssd monitor SCIF PORT: %s\n", strerror(errno));
pthread_exit((void *)1);
}
if (scif_listen(lep, 16) < 0) {
mpsslog(PINFO, "Set Listen on mpssd monitor SCIF PORT fail: %s\n", strerror(errno));
pthread_exit((void *)1);
}
while (1) {
if (scif_accept(lep, &recvID, &recv_ep, SCIF_ACCEPT_SYNC)) {
if (errno != EINTR)
mpsslog(PINFO, "Wait for card connect failed: %s\n", strerror(errno));
sleep(1);
continue;
}
if ((mic = mpss_find_micid_inlist(miclist, recvID.node - 1)) == NULL) {
mpsslog(PINFO, "Cannot configure - node %d does not seem to exist\n",
recvID.node - 1);
scif_close(recv_ep);
continue;
}
mpssdi = (struct mpssd_info *)mic->data;
if ((send_ep = scif_open()) < 0) {
fprintf(logfp, "Failed to open SCIF: %s\n", strerror(errno));
scif_close(recv_ep);
pthread_exit((void *)1);
}
mpssdi->send_ep = send_ep;
if ((err = scif_recv(recv_ep, &proto, sizeof(proto), SCIF_RECV_BLOCK)) != sizeof(proto)) {
mpsslog(PINFO, "%s: MIC card mpssd daemon startup connection error %s\n",
mic->name, strerror(errno));
scif_close(recv_ep);
mpssdi->recv_ep = -1;
continue;
}
switch (proto) {
case MONITOR_START:
sendID.node = mic->id + 1;
while ((send_port = scif_connect(send_ep, &sendID)) < 0) {
fprintf(logfp, "Failed to connect to monitor thread on card: %s\n",
strerror(errno));
sleep(1);
}
// Over reliable connection, mpssd tells us which port number it uses
// to talk back to us. If this port matches actual recv_ep remote port
// then we know that recv_ep and send_ep reference the same client.
// We also know that send_ep, references mpssd on mic, as port we
// connect to on that endpoint requires privliges to listen on.
if (scif_recv(send_ep, &remote_port, sizeof(remote_port), SCIF_RECV_BLOCK) < 0) {
mpsslog(PINFO, "%s: MIC card mpssd daemon handshake error %s\n",
mic->name, strerror(errno));
scif_close(send_ep);
scif_close(recv_ep);
continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards
}
if (remote_port != recvID.port || sendID.node != recvID.node) {
mpsslog(PINFO, "%s: Failed to authenticate connection with mic mpssd\n",
mic->name);
scif_close(send_ep);
scif_close(recv_ep);
continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards
}
// Similarily, provide info for the client, so that he can also verify
// that both connections send_ep & recv_ep belong to us.
if (scif_send(recv_ep, &send_port, sizeof(send_port), SCIF_SEND_BLOCK) < 0) {
mpsslog(PINFO, "%s: MIC card mpssd daemon handshake error %s\n",
mic->name, strerror(errno));
scif_close(send_ep);
scif_close(recv_ep);
continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards
}
mpssdi->recv_ep = recv_ep;
pthread_attr_init(&attr);
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
pthread_create(&mpssdi->monitor_pth, &attr, monitor, mic);
proto = MONITOR_START_ACK;
scif_send(send_ep, &proto, sizeof(proto), SCIF_RECV_BLOCK);
mpsslog(PINFO, "%s: Monitor connection established\n", mic->name);
break;
}
}
}
void *
monitor(void *arg)
{
struct mic_info *mic = (struct mic_info *)arg;
struct mpssd_info *mpssdi = (struct mpssd_info *)mic->data;
unsigned int proto;
unsigned int jobid;
struct pollfd pfds[1];
struct jobs *jlist;
struct jobs *job = NULL;
uint16_t stopID;
while (1) {
pfds[0].fd = mpssdi->recv_ep;
pfds[0].events = POLLIN | POLLERR | POLLPRI;
poll(pfds, 1, -1);
if (scif_recv(mpssdi->recv_ep, &proto, sizeof(proto), SCIF_RECV_BLOCK) < 0) {
if (errno == ECONNRESET) {
mpsslog(PERROR, "%s: MIC card mpssd daemon disconnect: %s\n", mic->name,strerror(errno));
scif_close(mpssdi->recv_ep);
scif_close(mpssdi->send_ep);
mpssdi->recv_ep = -1;
mpssdi->send_ep = -1;
pthread_exit((void *)1);
}
continue;
}
switch (proto) {
case REQ_CREDENTIAL_ACK:
case REQ_CREDENTIAL_NACK:
scif_recv(mpssdi->recv_ep, &jobid, sizeof(jobid), SCIF_RECV_BLOCK);
while (pthread_mutex_lock(&jobs_lock) != 0);
jlist = &gjobs;
while (jlist->next) {
if (jlist->next->jobid == jobid) {
job = jlist->next;
if (--job->cnt == 0) {
jlist->next = job->next;
while (pthread_mutex_unlock(&jobs_lock) != 0);
proto = CRED_SUCCESS;
scif_send(job->dep, &proto, sizeof(proto), 0);
scif_close(job->dep);
continue;
}
break;
}
jlist = jlist->next;
}
while (pthread_mutex_unlock(&jobs_lock) != 0);
break;
case MONITOR_STOPPING:
scif_recv(mpssdi->recv_ep, &stopID, sizeof(stopID), SCIF_RECV_BLOCK);
mpsslog(PERROR, "%s: card mpssd daemon exiting\n", mic->name);
scif_close(mpssdi->recv_ep);
scif_close(mpssdi->send_ep);
mpssdi->recv_ep = -1;
mpssdi->send_ep = -1;
pthread_exit((void *)0);
}
}
}
int MPID_nem_scif_vc_init(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = &vc->ch;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
int ret;
size_t s;
scifconn_t *sc;
off_t offset;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_VC_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_VC_INIT);
vc->sendNoncontig_fn = MPID_nem_scif_SendNoncontig;
vc_ch->iStartContigMsg = MPID_nem_scif_iStartContigMsg;
vc_ch->iSendContig = MPID_nem_scif_iSendContig;
vc_ch->next = NULL;
vc_ch->prev = NULL;
ASSIGN_SC_TO_VC(vc_scif, NULL);
vc_scif->send_queue.head = vc_scif->send_queue.tail = NULL;
vc_scif->sc = sc = &MPID_nem_scif_conns[vc->pg_rank];
vc_scif->terminate = 0;
sc->vc = vc;
/* do the connection */
if (vc->pg_rank < MPID_nem_scif_myrank) {
sc->fd = scif_open();
MPIU_ERR_CHKANDJUMP1(sc->fd == -1, mpi_errno, MPI_ERR_OTHER,
"**scif_open", "**scif_open %s", MPIU_Strerror(errno));
mpi_errno = get_addr(vc, &sc->addr);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
ret = scif_connect(sc->fd, &sc->addr);
MPIU_ERR_CHKANDJUMP1(ret == -1, mpi_errno, MPI_ERR_OTHER,
"**scif_connect", "**scif_connect %s", MPIU_Strerror(errno));
}
else {
ret = scif_accept(listen_fd, &sc->addr, &sc->fd, SCIF_ACCEPT_SYNC);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
"**scif_accept", "**scif_accept %s", MPIU_Strerror(errno));
}
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
ret = MPID_nem_scif_init_shmsend(&sc->csend, sc->fd, vc->pg_rank);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
"**scif_init_shmsend", "**scif_init_shmsend %s",
MPIU_Strerror(errno));
/* Exchange offsets */
s = scif_send(sc->fd, &sc->csend.offset, sizeof(off_t), SCIF_SEND_BLOCK);
MPIU_ERR_CHKANDJUMP1(s != sizeof(off_t), mpi_errno, MPI_ERR_OTHER,
"**scif_send", "**scif_send %s", MPIU_Strerror(errno));
s = scif_recv(sc->fd, &offset, sizeof(off_t), SCIF_RECV_BLOCK);
MPIU_ERR_CHKANDJUMP1(s != sizeof(off_t), mpi_errno, MPI_ERR_OTHER,
"**scif_recv", "**scif_recv %s", MPIU_Strerror(errno));
ret = MPID_nem_scif_init_shmrecv(&sc->crecv, sc->fd, offset, vc->pg_rank);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
"**scif_init_shmrecv", "**scif_init_shmrecv %s",
MPIU_Strerror(errno));
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_VC_INIT);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int main( )
{
size_t len = 536870912;
int align = 4096;
scif_epd_t endpoint;
struct scif_portID portid;
int ret;
uint8_t *in_key = malloc(16 * sizeof(uint8_t));
struct crypto_tfm *tfm
= malloc(
sizeof(struct crypto_tfm) +
sizeof(struct crypto_aes_ctx)
);
struct crypto_aes_ctx *ctx
= crypto_tfm_ctx(tfm);
ctx->key_length = AES_KEYSIZE_256;
crypto_aes_set_key(tfm, in_key, AES_KEYSIZE_256);
endpoint = scif_open( );
if( endpoint == SCIF_OPEN_FAILED )
{
printf("scif open failed\n");
return 1;
}
ret = scif_bind(endpoint, 23955);
if(ret==-1)
{
printf("scif_bind failed");
return 1;
}
portid.node = 0;
portid.port = 23968;
ret = scif_connect(endpoint, &portid);
for( int attempt = 0; ret == -1 && attempt < 10; ++attempt )
{
sleep(1);
ret = scif_connect(endpoint, &portid);
}
if (ret==-1)
{
printf("scif_connect failed\n");
return 1;
}
void *ptr;
ret = posix_memalign((void**)&ptr, align, len);
if (ret)
{
printf("Allocating memory failed\n");
return 1;
}
memset(ptr, 0, len);
if( SCIF_REGISTER_FAILED == scif_register(endpoint, ptr, len, (long)ptr, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
{
printf("scif_register of ptr failed due to: %s\n", strerror(errno));
return 1;
}
void *tempbuffer;
ret = posix_memalign((void**)&tempbuffer, align, len);
if (ret)
{
printf("Allocating tempbuffer failed\n");
return 1;
}
if( SCIF_REGISTER_FAILED == scif_register(endpoint, tempbuffer, len, (long)tempbuffer, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
{
printf("scif_register of temp failed due to: %s\n", strerror(errno));
return 1;
}
void *outbuffer;
ret = posix_memalign((void**)&outbuffer, align, len);
if (ret)
{
printf("Allocating outbuffer failed %s\n", strerror(errno));
return 1;
}
if( SCIF_REGISTER_FAILED == scif_register(endpoint, outbuffer, len, (long)outbuffer, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
{
printf("scif_register of outbuffer failed due to: %s\n", strerror(errno));
return 1;
}
void *remote_ptr;
void *return_ptr;
ret = scif_recv(endpoint, &remote_ptr, sizeof(void*), SCIF_RECV_BLOCK);
if (ret==-1)
{
printf("scif_recv failed due to: %s\n", strerror(errno));
return 1;
}
ret = scif_recv(endpoint, &return_ptr, sizeof(void*), SCIF_RECV_BLOCK);
if (ret==-1)
{
printf("scif_recv failed due to: %s\n", strerror(errno));
return 1;
}
struct timespec start_enc, stop_enc;
clock_gettime(CLOCK_REALTIME, &start_enc);
if (scif_readfrom(endpoint, (long)ptr, len, (long)remote_ptr, SCIF_RMA_SYNC))
{
printf("scif_readfrom failed due to: %s\n", strerror(errno));
return 1;
}
#pragma omp parallel for
for (int k = 0; k<len; k+=16)
{ aes_encrypt(tfm, (uint8_t*)&tempbuffer[k], (uint8_t*)&ptr[k]); }
if (scif_writeto(endpoint, (long)tempbuffer, len, (long)return_ptr, SCIF_RMA_SYNC))
{
printf("scif_writeto failed due to: %s\n", strerror(errno));
return 1;
}
clock_gettime(CLOCK_REALTIME, &stop_enc);
double time_enc = (stop_enc.tv_sec - start_enc.tv_sec) + ( stop_enc.tv_nsec - start_enc.tv_nsec) / NANOSECONDS;
double result0 = len/time_enc/1048576;
printf("%1f,", result0);
struct timespec start_for, stop_for;
clock_gettime(CLOCK_REALTIME, &start_for);
if (scif_readfrom(endpoint, (long)ptr, len, (long)remote_ptr, SCIF_RMA_SYNC))
{
printf("scif_readfrom failed due to: %s\n", strerror(errno));
return 1;
}
#pragma omp parallel for
for (int k=0; k<len; k+=16)
{ aes_decrypt(tfm, (uint8_t*)&outbuffer[k], (uint8_t*)&tempbuffer[k]); }
if (scif_writeto(endpoint, (long)outbuffer, len, (long)return_ptr, SCIF_RMA_SYNC))
{
printf("scif_writeto failed due to: %s\n", strerror(errno));
return 1;
}
clock_gettime(CLOCK_REALTIME, &stop_for);
double time_for = (stop_for.tv_sec - start_for.tv_sec) + ( stop_for.tv_nsec - start_for.tv_nsec) / NANOSECONDS;
double result = 536870912/time_for/1048576;
printf("%1f \n", result);
ret = scif_send(endpoint, &ptr, sizeof(long), SCIF_SEND_BLOCK);
if (ret==-1)
{
printf("scif_send failed due to: %s\n", strerror(errno));
return 1;
}
ret = scif_unregister(endpoint, (off_t)ptr, len );
if(ret==-1 && errno!=ENOTCONN )
{
printf("scif_unregister failed %s\n", strerror(errno));
return 1;
}
scif_close(endpoint);
}
/* must be called with the endpoint lock held */
static int mca_btl_scif_ep_connect_finish (mca_btl_base_endpoint_t *ep, bool passive) {
int rc;
rc = mca_btl_scif_ep_get_buffer (ep);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
BTL_VERBOSE(("error allocating buffer for scif peer"));
return rc;
}
if (passive) {
rc = scif_recv (ep->scif_epd, &ep->send_buffer.scif_offset,
sizeof (ep->send_buffer.scif_offset), SCIF_RECV_BLOCK);
if (OPAL_LIKELY(-1 != rc)) {
rc = scif_send (ep->scif_epd, &ep->recv_buffer.scif_offset,
sizeof (ep->recv_buffer.scif_offset), SCIF_SEND_BLOCK);
}
} else {
rc = scif_send (ep->scif_epd, &ep->recv_buffer.scif_offset,
sizeof (ep->recv_buffer.scif_offset), SCIF_SEND_BLOCK);
if (OPAL_LIKELY(-1 != rc)) {
rc = scif_recv (ep->scif_epd, &ep->send_buffer.scif_offset,
sizeof (ep->send_buffer.scif_offset), SCIF_RECV_BLOCK);
}
}
if (OPAL_UNLIKELY(-1 == rc)) {
BTL_VERBOSE(("error exchanging connection data with peer %d", ep->peer_proc->proc_name.vpid));
mca_btl_scif_ep_free_buffer (ep);
return OPAL_ERROR;
}
BTL_VERBOSE(("remote peer %d has scif offset %lu", ep->peer_proc->proc_name.vpid,
(unsigned long) ep->send_buffer.scif_offset));
ep->send_buffer.buffer = scif_mmap (0, mca_btl_scif_component.segment_size,
SCIF_PROT_READ | SCIF_PROT_WRITE,
0, ep->scif_epd, ep->send_buffer.scif_offset);
if (OPAL_UNLIKELY(NULL == ep->send_buffer.buffer)) {
BTL_VERBOSE(("error in scif_mmap"));
mca_btl_scif_ep_free_buffer (ep);
return OPAL_ERROR;
}
opal_memchecker_base_mem_defined (ep->send_buffer.buffer, mca_btl_scif_component.segment_size);
BTL_VERBOSE(("remote peer %d buffer mapped to local pointer %p", ep->peer_proc->proc_name.vpid,
ep->send_buffer.buffer));
/* setup the circular send buffers */
ep->send_buffer.start = ep->send_buffer.end = 64;
ep->send_buffer.startp = (uint32_t *) ep->send_buffer.buffer;
ep->send_buffer.endp = ep->send_buffer.startp + 1;
ep->recv_buffer.start = 64;
/* connection complete */
ep->state = MCA_BTL_SCIF_EP_STATE_CONNECTED;
BTL_VERBOSE(("btl/scif connection to remote peer %d established", ep->peer_proc->proc_name.vpid));
return OPAL_SUCCESS;
}
