int main( )
{
	uint8_t len = 1;
	uint64_t loops = 50000;

	scif_epd_t endpoint;
	struct scif_portID portid;
	int ret;

        endpoint = scif_open( );
	if( endpoint == SCIF_OPEN_FAILED ) 
	{
            printf("scif open failed\n");
            return 1;
        }

	ret = scif_bind(endpoint, 23955);
	if(ret==-1) 
	{
		printf("scif_bind failed");
		return 1;
	}

	portid.node = 0;
	portid.port = 23968;

	ret = scif_connect(endpoint, &portid);
	for( int attempt = 0; ret == -1 && attempt < 10; ++attempt ) 
	{
        	sleep(1);
        	ret = scif_connect(endpoint, &portid);
	}
	if (ret==-1)
	{ 
		printf("scif_connect failed\n");
		return 1;
	}

	uint8_t* buffer = malloc(len * sizeof(uint8_t));

	struct timespec start_latency, stop_latency;
	clock_gettime(CLOCK_REALTIME, &start_latency);
	for (uint64_t i = 0; i<loops; i++)
	{	
		scif_recv(endpoint, (void*)buffer, len, SCIF_RECV_BLOCK);
		scif_send(endpoint, &buffer[0], len, SCIF_SEND_BLOCK);
	}
	clock_gettime(CLOCK_REALTIME, & stop_latency);
	double time_latency = (stop_latency.tv_sec - start_latency.tv_sec) + (stop_latency.tv_nsec - start_latency.tv_nsec) / NANOSECONDS;
	double result = (time_latency/(2*loops));
	printf("%1f\n", result);
	free(buffer);
	scif_close(endpoint);

	

}
Beispiel #2
0
static inline int mca_btl_scif_ep_connect_start_active (mca_btl_base_endpoint_t *ep) {
    int rc = OPAL_SUCCESS;

    BTL_VERBOSE(("initiaiting connection to remote peer %d with port: %u on local scif node: %u",
                 ep->peer_proc->proc_name.vpid, ep->port_id.port, ep->port_id.node));

    opal_mutex_lock (&ep->lock);
    do {
        if (MCA_BTL_SCIF_EP_STATE_INIT != ep->state) {
            /* the accept thread has already finished this connection */
            rc = OPAL_SUCCESS;
            break;
        }

        ep->state = MCA_BTL_SCIF_EP_STATE_CONNECTING;

        ep->scif_epd = scif_open ();
        if (OPAL_UNLIKELY(SCIF_OPEN_FAILED == ep->scif_epd)) {
            BTL_VERBOSE(("error creating new scif endpoint"));
            rc = OPAL_ERROR;
            break;
        }

        rc = scif_connect (ep->scif_epd, &ep->port_id);
        if (OPAL_UNLIKELY(-1 == rc)) {
            /* the connection attempt failed. this could mean the peer is currently
             * processing connections. we will to try again later. */
            BTL_VERBOSE(("error connecting to scif peer. %d", errno));
            rc = OPAL_ERR_RESOURCE_BUSY;
            break;
        }

        rc = scif_send (ep->scif_epd, &OPAL_PROC_MY_NAME, sizeof (OPAL_PROC_MY_NAME), SCIF_SEND_BLOCK);
        if (OPAL_UNLIKELY(-1 == rc)) {
            BTL_VERBOSE(("error in scif_send"));
            rc = OPAL_ERROR;
            break;
        }

        /* build connection data */
        rc = mca_btl_scif_ep_connect_finish (ep, false);
    } while (0);

    if (OPAL_SUCCESS != rc) {
        scif_close (ep->scif_epd);
        ep->scif_epd = -1;
        ep->state = MCA_BTL_SCIF_EP_STATE_INIT;
    }

    opal_mutex_unlock (&ep->lock);

    return rc;
}
scif_epd_t scif_obc(void)
{
    scif_epd_t epd;
    struct scif_portID portID;
    int connection_port, request_port;
    int err, tries = MAX_TRIES;

    /* create "socket" and connect */
    request_port = LOCAL_PORT;
    portID.node = PEER_NODE;
    portID.port = PEER_PORT;

    if ((epd = scif_open()) == SCIF_OPEN_FAILED) {
        fprintf(stderr, "= scif_open failed with error %d\n", (int) epd);
        exit(EXIT_FAILURE);
    }

    if ((connection_port = scif_bind(epd, request_port)) < 0) {
        fprintf(stderr, "= scif_bind failed with error %d\n",
                connection_port);
        exit(EXIT_FAILURE);
    }
    printf("= scif_bind to port %d success\n", connection_port);

__retry:
    if ((err = scif_connect(epd, &portID)) < 0) {
        if ((errno == ECONNREFUSED) && (tries > 0)) {
            printf("= connection to node %d failed : trial %d\n",
                   portID.node, tries);
            tries--;
            sleep(1);
            goto __retry;
        }
        fprintf(stderr, "= scif_connect failed with error %d\n", errno);
        exit(EXIT_FAILURE);
    }
    printf("= conect to node %d success\n", portID.node);

    return epd;
}
Beispiel #4
0
static int
mca_btl_scif_module_finalize (struct mca_btl_base_module_t *btl)
{
    mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
    unsigned int i;

    OBJ_DESTRUCT(&mca_btl_scif_module.dma_frags);
    OBJ_DESTRUCT(&mca_btl_scif_module.eager_frags);

    mca_btl_scif_module.exiting = true;

    /* close all open connections and release endpoints */
    if (NULL != scif_module->endpoints) {
        for (i = 0 ; i < scif_module->endpoint_count ; ++i) {
            mca_btl_scif_ep_release (scif_module->endpoints + i);
        }

        free (scif_module->endpoints);

        scif_module->endpoint_count = 0;
        scif_module->endpoints = NULL;
    }

    /* close the listening endpoint */
    if (mca_btl_scif_module.listening && -1 != mca_btl_scif_module.scif_fd) {
        /* wake up the scif thread */
        scif_epd_t tmpfd;
        tmpfd = scif_open();
        scif_connect (tmpfd, &mca_btl_scif_module.port_id);
        pthread_join(mca_btl_scif_module.listen_thread, NULL);
        scif_close(tmpfd);
        scif_close (mca_btl_scif_module.scif_fd);
    }

    mca_btl_scif_module.scif_fd = -1;

    return OPAL_SUCCESS;
}
Beispiel #5
0
void *
mic_monitor(void *arg)
{
	struct mic_info *mic;
	struct mpssd_info *mpssdi;
	pthread_attr_t attr;
	struct scif_portID sendID = {0, MPSSD_MONSEND};
	struct scif_portID recvID;
	scif_epd_t lep;
	scif_epd_t recv_ep;
	scif_epd_t send_ep;
	unsigned int proto;
	uint16_t send_port;
	uint16_t remote_port = 0;
	int err;

	if ((lep = scif_open()) < 0) {
		mpsslog(PINFO, "Cannot open mpssd monitor SCIF listen port: %s\n", strerror(errno));
		pthread_exit((void *)1);
	}

	if (scif_bind(lep, MPSSD_MONRECV) < 0) {
		mpsslog(PINFO, "Cannot bind to mpssd monitor SCIF PORT: %s\n", strerror(errno));
		pthread_exit((void *)1);
	}

	if (scif_listen(lep, 16) < 0) {
		mpsslog(PINFO, "Set Listen on mpssd monitor SCIF PORT fail: %s\n", strerror(errno));
		pthread_exit((void *)1);
	}

	while (1) {
		if (scif_accept(lep, &recvID, &recv_ep, SCIF_ACCEPT_SYNC)) {
			if (errno != EINTR)
				mpsslog(PINFO, "Wait for card connect failed: %s\n", strerror(errno));
			sleep(1);
			continue;
		}

		if ((mic = mpss_find_micid_inlist(miclist, recvID.node - 1)) == NULL) {
			mpsslog(PINFO, "Cannot configure - node %d does not seem to exist\n",
				       recvID.node - 1);
			scif_close(recv_ep);
			continue;
		}

		mpssdi = (struct mpssd_info *)mic->data;

		if ((send_ep = scif_open()) < 0) {
			fprintf(logfp, "Failed to open SCIF: %s\n", strerror(errno));
			scif_close(recv_ep);
			pthread_exit((void *)1);
		}
		mpssdi->send_ep = send_ep;

		if ((err = scif_recv(recv_ep, &proto, sizeof(proto), SCIF_RECV_BLOCK)) != sizeof(proto)) {
			mpsslog(PINFO, "%s: MIC card mpssd daemon startup connection error %s\n",
					mic->name, strerror(errno));
			scif_close(recv_ep);
			mpssdi->recv_ep = -1;
			continue;
		}

		switch (proto) {
		case MONITOR_START:
			sendID.node = mic->id + 1;
			while ((send_port = scif_connect(send_ep, &sendID)) < 0) {
				fprintf(logfp, "Failed to connect to monitor thread on card: %s\n",
					strerror(errno));
				sleep(1);
			}

			// Over reliable connection, mpssd tells us which port number it uses
			// to talk back to us. If this port matches actual recv_ep remote port
			// then we know that recv_ep and send_ep reference the same client.
			// We also know that send_ep, references mpssd on mic, as port we
			// connect to on that endpoint requires privliges to listen on.
			if (scif_recv(send_ep, &remote_port, sizeof(remote_port), SCIF_RECV_BLOCK) < 0) {
				mpsslog(PINFO, "%s: MIC card mpssd daemon handshake error %s\n",
					mic->name, strerror(errno));
				scif_close(send_ep);
				scif_close(recv_ep);
				continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards
			}

			if (remote_port != recvID.port || sendID.node != recvID.node) {
				mpsslog(PINFO, "%s: Failed to authenticate connection with mic mpssd\n",
					mic->name);
				scif_close(send_ep);
				scif_close(recv_ep);
				continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards

			}

			// Similarily, provide info for the client, so that he can also verify
			// that both connections send_ep & recv_ep belong to us.
			if (scif_send(recv_ep, &send_port, sizeof(send_port), SCIF_SEND_BLOCK) < 0) {
				mpsslog(PINFO, "%s: MIC card mpssd daemon handshake error %s\n",
					mic->name, strerror(errno));
				scif_close(send_ep);
				scif_close(recv_ep);
				continue; // go back to next iteration of while(1), we cannot break the while loop because hosts mpssd can connect with multiple mic cards

			}

			mpssdi->recv_ep = recv_ep;
			pthread_attr_init(&attr);
			pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
			pthread_create(&mpssdi->monitor_pth, &attr, monitor, mic);
			proto = MONITOR_START_ACK;
			scif_send(send_ep, &proto, sizeof(proto), SCIF_RECV_BLOCK);
			mpsslog(PINFO, "%s: Monitor connection established\n", mic->name);
			break;
		}
	}
}
Beispiel #6
0
int MPID_nem_scif_vc_init(MPIDI_VC_t * vc)
{
    int mpi_errno = MPI_SUCCESS;
    MPIDI_CH3I_VC *vc_ch = &vc->ch;
    MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
    int ret;
    size_t s;
    scifconn_t *sc;
    off_t offset;
    MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_VC_INIT);

    MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_VC_INIT);

    vc->sendNoncontig_fn = MPID_nem_scif_SendNoncontig;
    vc_ch->iStartContigMsg = MPID_nem_scif_iStartContigMsg;
    vc_ch->iSendContig = MPID_nem_scif_iSendContig;

    vc_ch->next = NULL;
    vc_ch->prev = NULL;

    ASSIGN_SC_TO_VC(vc_scif, NULL);
    vc_scif->send_queue.head = vc_scif->send_queue.tail = NULL;
    vc_scif->sc = sc = &MPID_nem_scif_conns[vc->pg_rank];
    vc_scif->terminate = 0;
    sc->vc = vc;

    /* do the connection */
    if (vc->pg_rank < MPID_nem_scif_myrank) {
        sc->fd = scif_open();
        MPIU_ERR_CHKANDJUMP1(sc->fd == -1, mpi_errno, MPI_ERR_OTHER,
                             "**scif_open", "**scif_open %s", MPIU_Strerror(errno));
        mpi_errno = get_addr(vc, &sc->addr);
        if (mpi_errno)
            MPIU_ERR_POP(mpi_errno);
        ret = scif_connect(sc->fd, &sc->addr);
        MPIU_ERR_CHKANDJUMP1(ret == -1, mpi_errno, MPI_ERR_OTHER,
                             "**scif_connect", "**scif_connect %s", MPIU_Strerror(errno));
    }
    else {
        ret = scif_accept(listen_fd, &sc->addr, &sc->fd, SCIF_ACCEPT_SYNC);
        MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
                             "**scif_accept", "**scif_accept %s", MPIU_Strerror(errno));
    }
    MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
    ret = MPID_nem_scif_init_shmsend(&sc->csend, sc->fd, vc->pg_rank);
    MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
                         "**scif_init_shmsend", "**scif_init_shmsend %s",
                         MPIU_Strerror(errno));

    /* Exchange offsets */
    s = scif_send(sc->fd, &sc->csend.offset, sizeof(off_t), SCIF_SEND_BLOCK);
    MPIU_ERR_CHKANDJUMP1(s != sizeof(off_t), mpi_errno, MPI_ERR_OTHER,
                         "**scif_send", "**scif_send %s", MPIU_Strerror(errno));
    s = scif_recv(sc->fd, &offset, sizeof(off_t), SCIF_RECV_BLOCK);
    MPIU_ERR_CHKANDJUMP1(s != sizeof(off_t), mpi_errno, MPI_ERR_OTHER,
                         "**scif_recv", "**scif_recv %s", MPIU_Strerror(errno));

    ret = MPID_nem_scif_init_shmrecv(&sc->crecv, sc->fd, offset, vc->pg_rank);
    MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER,
                         "**scif_init_shmrecv", "**scif_init_shmrecv %s",
                         MPIU_Strerror(errno));

    MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_VC_INIT);

  fn_exit:
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}
int main( )
{
	size_t len = 536870912;
	int align = 4096;
	scif_epd_t endpoint;
	struct scif_portID portid;
	int ret;
	
	uint8_t *in_key     = malloc(16 * sizeof(uint8_t));
	struct crypto_tfm *tfm
        	= malloc(
                	 sizeof(struct crypto_tfm) +
                 	 sizeof(struct crypto_aes_ctx)
               		 );
	struct crypto_aes_ctx *ctx
        	= crypto_tfm_ctx(tfm);

	ctx->key_length = AES_KEYSIZE_256;
	crypto_aes_set_key(tfm, in_key, AES_KEYSIZE_256);
	
        endpoint = scif_open( );
	if( endpoint == SCIF_OPEN_FAILED ) 
	{
		printf("scif open failed\n");
		return 1;
        }

	ret = scif_bind(endpoint, 23955);
	if(ret==-1) 
	{
		printf("scif_bind failed");
		return 1;
	}

	portid.node = 0;
	portid.port = 23968;

	ret = scif_connect(endpoint, &portid);
	for( int attempt = 0; ret == -1 && attempt < 10; ++attempt ) 
	{
        	sleep(1);
        	ret = scif_connect(endpoint, &portid);
	}
	if (ret==-1)
	{ 
		printf("scif_connect failed\n");
		return 1;
	}

	void *ptr;
	ret = posix_memalign((void**)&ptr, align, len);	
	if (ret)
	{
		printf("Allocating memory failed\n");
		return 1;

	}
	memset(ptr, 0, len);

	if( SCIF_REGISTER_FAILED == scif_register(endpoint, ptr, len, (long)ptr, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
        {
                printf("scif_register of ptr failed due to: %s\n", strerror(errno));
                return 1;
        }

	void *tempbuffer;
	ret = posix_memalign((void**)&tempbuffer, align, len);
	if (ret)
	{
		printf("Allocating tempbuffer failed\n");
		return 1;
	}

       	if( SCIF_REGISTER_FAILED == scif_register(endpoint, tempbuffer, len, (long)tempbuffer, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
        {
               	printf("scif_register of temp failed due to: %s\n", strerror(errno));
                return 1;
        }

	void *outbuffer;
	ret = posix_memalign((void**)&outbuffer, align, len);
	if (ret)
	{
		printf("Allocating outbuffer failed %s\n", strerror(errno));
		return 1;
	}

	if( SCIF_REGISTER_FAILED == scif_register(endpoint, outbuffer, len, (long)outbuffer, SCIF_PROT_READ | SCIF_PROT_WRITE, SCIF_MAP_FIXED ) )
        {
                printf("scif_register of outbuffer failed due to: %s\n", strerror(errno));
                return 1;
        }

	void *remote_ptr;
	void *return_ptr;

	ret = scif_recv(endpoint, &remote_ptr, sizeof(void*), SCIF_RECV_BLOCK);
	if (ret==-1)
	{
		printf("scif_recv failed due to: %s\n", strerror(errno));
		return 1;
	}

        ret = scif_recv(endpoint, &return_ptr, sizeof(void*), SCIF_RECV_BLOCK);
        if (ret==-1)
        {
                printf("scif_recv failed due to: %s\n", strerror(errno));
                return 1;
        }
	
	struct timespec start_enc, stop_enc;
	clock_gettime(CLOCK_REALTIME, &start_enc);
	if (scif_readfrom(endpoint, (long)ptr, len, (long)remote_ptr, SCIF_RMA_SYNC))
	{
		printf("scif_readfrom failed due to: %s\n", strerror(errno));
		return 1;
	}

	#pragma omp parallel for 
	for (int k = 0; k<len; k+=16)
		{ aes_encrypt(tfm, (uint8_t*)&tempbuffer[k], (uint8_t*)&ptr[k]); }

	if (scif_writeto(endpoint, (long)tempbuffer, len, (long)return_ptr, SCIF_RMA_SYNC))
        {
		printf("scif_writeto failed due to: %s\n", strerror(errno));
		return 1;
	}					

	clock_gettime(CLOCK_REALTIME, &stop_enc);
	double time_enc = (stop_enc.tv_sec - start_enc.tv_sec) + ( stop_enc.tv_nsec - start_enc.tv_nsec) / NANOSECONDS;
	double result0 = len/time_enc/1048576;
       	printf("%1f,", result0);

        struct timespec start_for, stop_for;
        clock_gettime(CLOCK_REALTIME, &start_for);
        if (scif_readfrom(endpoint, (long)ptr, len, (long)remote_ptr, SCIF_RMA_SYNC))
        {
	        printf("scif_readfrom failed due to: %s\n", strerror(errno));
		return 1;
	}

	#pragma omp parallel for
        for (int k=0; k<len; k+=16)
    		{ aes_decrypt(tfm, (uint8_t*)&outbuffer[k], (uint8_t*)&tempbuffer[k]); }

        if (scif_writeto(endpoint, (long)outbuffer, len, (long)return_ptr, SCIF_RMA_SYNC))
    	{
        	printf("scif_writeto failed due to: %s\n", strerror(errno));
                return 1;
        }

	clock_gettime(CLOCK_REALTIME, &stop_for);
        double time_for = (stop_for.tv_sec - start_for.tv_sec) + ( stop_for.tv_nsec - start_for.tv_nsec) / NANOSECONDS;
        double result = 536870912/time_for/1048576;
        printf("%1f \n", result);

	ret = scif_send(endpoint, &ptr, sizeof(long), SCIF_SEND_BLOCK);
	if (ret==-1)
	{
		printf("scif_send failed due to: %s\n", strerror(errno));
		return 1;
	}

        ret = scif_unregister(endpoint, (off_t)ptr, len );
        if(ret==-1 && errno!=ENOTCONN )
        {
                printf("scif_unregister failed %s\n", strerror(errno));
                return 1;
        }

	scif_close(endpoint);	

}
Beispiel #8
0
int pm_scif_init(void)
{
	int err = 1;
	int retry = 0;

	FUNCTION_ENTRY;
	PM_DB("pm_scif insmoded \n");
#ifdef PM_SCIF_IOCTL
	if ((err = spm_dev_init())) {
		PM_DB(" spm_dev_init failed\n");
		goto done;
	}
#endif
	atomic_set(&epinuse,0);
	pm_scif = kzalloc(sizeof(mic_pm_scif), GFP_KERNEL);

	if (!pm_scif) {
		err = -ENOMEM;
		goto end_con;
	}

	pm_scif_register(&micpmscif);

	if ((pm_scif->ep = scif_open()) == NULL) {
		PM_DB(" scif_open failed\n");
		goto end_con;
	}

	if ((pm_scif->lport = scif_bind(pm_scif->ep, 0)) < 0) {
		PM_DB(" scif_bind failed\n");
		goto end_con;
	}

	PM_DB(" scif_bind successfull. Local port number = %d, ep =  \n",
							 pm_scif->lport);
	dump_ep(pm_scif->ep, __func__,__LINE__);
	pm_scif->rport_id.node = 0;
	pm_scif->rport_id.port = SCIF_PM_PORT_0;

	while ((err = scif_connect(pm_scif->ep, &pm_scif->rport_id)) != 0) {
		PM_DB(" scif_connect failed with err = %d ep %p\n",err,
			pm_scif->ep);
		msleep(1000);
		if (retry++ > PM_SCIF_RETRY_COUNT)
			goto end_con;
	}

	pm_scif->pm_recvq = create_singlethread_workqueue("pm_recvq");
	INIT_WORK(&pm_scif->pm_recv, pm_recv_from_host);
	queue_work(pm_scif->pm_recvq, &pm_scif->pm_recv);
	pm_scif->con_state = PM_CONNECTED;
	err = 0;
#ifdef PM_SCIF_IOCTL
done:
#endif
	return err;
end_con:
	pm_scif_exit();
	FUNCTION_EXIT;
	return err;
}