/* Creates a new instance of the dev_pcie */
static llio_dev_pcie_t * llio_dev_pcie_new (const char *dev_entry)
{
llio_dev_pcie_t *self = (llio_dev_pcie_t *) zmalloc (sizeof *self);
ASSERT_ALLOC (self, err_llio_dev_pcie_alloc);
self->dev = (pd_device_t *) zmalloc (sizeof *self->dev);
ASSERT_ALLOC (self->dev, err_dev_pcie_alloc);
/* FIXME: hardcoded dev number */
/* TODO: should we use llio_endpoint_get to get the endpoint name? */
int err = pd_open (0, self->dev, dev_entry);
if (err != 0) {
perror ("pd_open");
}
ASSERT_TEST(err==0, "Error opening device", err_dev_pcie_open);
/* Map all available BARs */
self->bar0 = (uint32_t *) pd_mapBAR (self->dev, BAR0NO);
ASSERT_TEST(self->bar0!=NULL, "Could not allocate bar0", err_bar0_alloc);
self->bar2 = (uint32_t *) pd_mapBAR (self->dev, BAR2NO);
ASSERT_TEST(self->bar2!=NULL, "Could not allocate bar2", err_bar2_alloc);
DBE_DEBUG (DBG_LL_IO | DBG_LVL_TRACE, "[ll_io_pcie] BAR2 addr = %p\n",
self->bar2);
self->bar4 = (uint64_t *) pd_mapBAR (self->dev, BAR4NO);
ASSERT_TEST(self->bar4!=NULL, "Could not allocate bar4", err_bar4_alloc);
DBE_DEBUG (DBG_LL_IO | DBG_LVL_TRACE, "[ll_io_pcie] BAR4 addr = %p\n",
self->bar4);
self->bar0_size = pd_getBARsize (self->dev, BAR0NO);
ASSERT_TEST(self->bar0_size > 0, "Could not get bar0 size", err_bar0_size);
self->bar2_size = pd_getBARsize (self->dev, BAR2NO);
ASSERT_TEST(self->bar2_size > 0, "Could not get bar2 size", err_bar2_size);
self->bar4_size = pd_getBARsize (self->dev, BAR4NO);
ASSERT_TEST(self->bar4_size > 0, "Could not get bar4 size", err_bar4_size);
/* Initialize PCIE timeout pattern */
memset (&pcie_timeout_patt, PCIE_TIMEOUT_PATT_INIT, sizeof (pcie_timeout_patt));
DBE_DEBUG (DBG_LL_IO | DBG_LVL_TRACE, "[ll_io_pcie] Created instance of llio_dev_pcie\n");
return self;
err_bar4_size:
err_bar2_size:
err_bar0_size:
pd_unmapBAR (self->dev, BAR4NO, self->bar4);
err_bar4_alloc:
pd_unmapBAR (self->dev, BAR2NO, self->bar2);
err_bar2_alloc:
pd_unmapBAR (self->dev, BAR0NO, self->bar0);
err_bar0_alloc:
pd_close (self->dev);
err_dev_pcie_open:
free (self->dev);
err_dev_pcie_alloc:
free (self);
err_llio_dev_pcie_alloc:
return NULL;
}
void testDevice( int i )
{
pd_device_t dev;
int ret;
printf("Trying device %d ...", i);
ret = pd_open( i, &dev );
if (ret != 0) {
printf("failed\n");
return;
}
printf("ok\n");
testDirectIO(&dev);
testDMA(&dev);
pd_close( &dev );
}
int main(int argc, char **argv)
{
unsigned int *data_samples;
int i;
int size;
pd_device_t dev;
void *memr, *mems;
float throughput;
struct timeval tv1,tv2;
int mintime, curtime;
printf("====================================\n");
printf("= DMA Throughput Test =\n");
printf("====================================\n\n");
if(argc != 2){
printf("Usage: %s [buffer size]\n",argv[0]);
return -1;
}
size = atoi(argv[1]);
if(size > 524288){
printf("Wohaa, easy there. Maximum buffer size is 262144\n");
return -1;
}
if(size % 4 != 0){
printf("Buffer size must be divisible by 4 bytes\n");
return -1;
}
printf("Buffer size: %d\n\n",size);
if(pd_open(0,&dev) != 0){
printf("Failed to open file \n");
return -1;
}
printf("Device file opened successfuly\n");
// Allocate send and receive buffers
posix_memalign( (void**)&memr, 16, size);
posix_memalign( (void**)&mems, 16, size);
//for(i=0;i<size/4;i++)
// ((unsigned int*)mems)[i] = i;
//for(i=0;i<size/4;i++)
// ((unsigned int*)memr)[i] = 9;
//--------------------------
if(initDMA(&dev) < 0)
printf("Init DMA failed\n");
/*
gettimeofday(&tv1,NULL);
if(setupSend(&dev, (void*)mems, size) < 0)
printf("Setup Send failed\n");
gettimeofday(&tv2,NULL);
printf(">> Time taken to setup send transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
//--------------------------
gettimeofday(&tv1,NULL);
if(setupRecv(&dev, memr, size) < 0)
printf("Setup Recv failed\n");
gettimeofday(&tv2,NULL);
printf(">> Time taken to setup receive transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
//--------------------------
gettimeofday(&tv1,NULL);
if(startSend(&dev, 0) < 0)
printf("DMA Send failed\n");
if(checkSend() < 0)
printf("Check DMA send failed\n");
gettimeofday(&tv2,NULL);
printf(">> Time taken to complete and check send transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
printf(">> Throughput: %f MB/s\n",(float)size/((float)(tv2.tv_usec - tv1.tv_usec)));
//--------------------------
gettimeofday(&tv1,NULL);
if(startRecv() < 0)
printf("DMA Recv failed\n");
if(checkRecv() < 0)
printf("Check DMA recv failed\n");
gettimeofday(&tv2,NULL);
printf(">> Time taken to complete and check recv transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
printf(">> Throughput: %f MB/s\n",(float)size/((float)(tv2.tv_usec - tv1.tv_usec)));
*/
//--------------------------
// Send and receive in simultaneous
mintime = 9999;
for(i=0;i<NRUNS;i++){
gettimeofday(&tv1,NULL);
if(setupSend(&dev, (void*)mems, size,1) < 0)
printf("Setup Send failed\n");
if(setupRecv(&dev, (void*)memr, size) < 0)
printf("Setup Recv failed\n");
gettimeofday(&tv2,NULL);
printf("Time taken to setup receive AND send transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
//--------------------------
gettimeofday(&tv1,NULL);
if(startRecv(&dev, 0) < 0)
printf("DMA Recv failed\n");
if(startSend(&dev, 0) < 0)
printf("DMA Send failed\n");
if(checkRecv() < 0)
printf("Check DMA recv failed\n");
if(checkSend() < 0)
printf("Check DMA send failed\n");
gettimeofday(&tv2,NULL);
curtime = tv2.tv_usec - tv1.tv_usec;
printf("Time taken to complete and check send AND receive transfer:%d\n", curtime);
printf("Throughput: %f MB/s\n",(float)2*size/((float)(curtime)));
freeSend(&dev);
freeRecv(&dev);
if(curtime < mintime)
mintime = curtime;
}
stopDMA(&dev);
//for(i=0;i<size/4;i++)
// printf("%d ",((unsigned int*)memr)[i]);
free(memr); free(mems);
printf("Maximum throughput for %d points: %f MB/s\n",size,(float)2*size/((float)(mintime)));
return 0;
}
int main(int argc, char *argv[])
{
int verbose = 0;
char *devicefile_str = NULL;
int rw_fpga = -1;
char *barno_str = NULL;
char *address_str = NULL;
char *data_str = NULL;
int opt;
pd_device_t _dev = {0};
pd_device_t *dev = &_dev;
uint32_t *bar0 = NULL;
uint32_t bar0_size;
uint32_t *bar2 = NULL;
uint32_t bar2_size;
uint64_t *bar4 = NULL;
uint32_t bar4_size;
while ((opt = getopt_long (argc, argv, shortopt, long_options, NULL)) != -1) {
/* Get the user selected options */
switch (opt) {
/* Display Help */
case 'h':
print_help (argv [0]);
exit (1);
break;
case 'b':
devicefile_str = strdup (optarg);
break;
case 'v':
verbose = 1;
break;
case 'r':
rw_fpga = 1;
break;
case 'w':
rw_fpga = 0;
break;
case 'n':
barno_str = strdup (optarg);
break;
case 'a':
address_str = strdup (optarg);
break;
case 'd':
data_str = strdup (optarg);
break;
case '?':
fprintf (stderr, "Option not recognized or missing argument\n");
print_help (argv [0]);
exit (1);
break;
default:
fprintf (stderr, "Could not parse options\n");
print_help (argv [0]);
exit (1);
}
}
/* Device file is mandatory */
if (devicefile_str == NULL) {
fprintf (stderr, "--devicefile option not set!\n");
print_help (argv [0]);
goto exit;
}
/* BAR number must be set */
int barno = 0;
if (barno_str == NULL) {
fprintf (stderr, "--barno option not set!\n");
print_help (argv [0]);
goto exit;
}
else {
barno = strtoul (barno_str, NULL, 10);
if (barno != 0 && barno != 2 && barno != 4) {
fprintf (stderr, "Invalid option for BAR number!\n");
print_help (argv [0]);
goto exit;
}
}
if (rw_fpga != 0 && rw_fpga != 1) {
fprintf (stderr, "Neither --read or --write was set!\n");
print_help (argv [0]);
goto exit;
}
/* If read access, address must be set */
if (rw_fpga == 1 && address_str == NULL) {
fprintf (stderr, "--read_fpga is set but no --address!\n");
print_help (argv [0]);
goto exit;
}
if (rw_fpga == 0 && (address_str == NULL || data_str == NULL)) {
fprintf (stderr, "--write_fpga is set but either --address or --data not set!\n");
print_help (argv [0]);
goto exit;
}
/* Parse data/address */
uint64_t address = 0;
if (address_str != NULL) {
if (sscanf (address_str, "%"PRIx64, &address) != 1) {
fprintf (stderr, "--address format is invalid!\n");
print_help (argv [0]);
goto exit;
}
if (verbose) {
fprintf (stdout, "Address = 0x%08X\n", address);
}
}
uint32_t data = 0;
if (data_str != NULL) {
if (sscanf (data_str, "%"PRIx32, &data) != 1) {
fprintf (stderr, "--data format is invalid!\n");
print_help (argv [0]);
goto exit;
}
if (verbose) {
fprintf (stdout, "Data = 0x%08X\n", data);
}
}
/* Open device */
int err = pd_open (0, dev, devicefile_str);
if (err != 0) {
fprintf (stderr, "Could not open device %s, error = %d\n", devicefile_str, err);
exit (1);
goto exit;
}
/* Map BARs */
bar0 = pd_mapBAR (dev, 0);
if (bar0 == NULL) {
fprintf (stderr, "Could not map BAR 0\n");
exit (1);
goto exit_close;
}
bar2 = pd_mapBAR (dev, 2);
if (bar2 == NULL) {
fprintf (stderr, "Could not map BAR 2\n");
goto exit_unmap_bar0;
}
bar4 = pd_mapBAR (dev, 4);
if (bar4 == NULL) {
fprintf (stderr, "Could not map BAR 4\n");
goto exit_unmap_bar2;
}
if (verbose) {
fprintf (stdout, "BAR 0 host address = %p\n", bar0);
fprintf (stdout, "BAR 2 host address = %p\n", bar2);
fprintf (stdout, "BAR 4 host address = %p\n", bar4);
}
/* Get BAR sizes */
bar0_size = pd_getBARsize (dev, 0);
if (bar0_size == -1) {
fprintf (stderr, "Could not get BAR 0 size\n");
goto exit_unmap_bar4;
}
bar2_size = pd_getBARsize (dev, 2);
if (bar2_size == -1) {
fprintf (stderr, "Could not get BAR 2 size\n");
goto exit_unmap_bar4;
}
bar4_size = pd_getBARsize (dev, 4);
if (bar4_size == -1) {
fprintf (stderr, "Could not get BAR 4 size\n");
goto exit_unmap_bar4;
}
if (verbose) {
fprintf (stdout, "BAR 0 size = %u bytes\n", bar0_size);
fprintf (stdout, "BAR 2 size = %u bytes\n", bar2_size);
fprintf (stdout, "BAR 4 size = %u bytes\n", bar4_size);
}
/* Read/Write to BAR */
int pg_num = 0;
uint64_t pg_offs = 0;
switch (barno) {
case 0:
BAR0_RW(bar0, address, &data, rw_fpga);
if (verbose) {
fprintf (stdout, "%s from BAR0, data = 0x%08X, addr = 0x%08X\n",
(rw_fpga)? "Reading":"Writing", data, address);
}
break;
case 2:
pg_num = PCIE_ADDR_SDRAM_PG (address);
pg_offs = PCIE_ADDR_SDRAM_PG_OFFS (address);
SET_SDRAM_PG (bar0, pg_num);
BAR2_RW(bar2, pg_offs, &data, rw_fpga);
if (verbose) {
fprintf (stdout, "%s from BAR2, data = 0x%08X, addr = 0x%08X, page = %d\n",
(rw_fpga)? "Reading":"Writing", data, address, pg_num);
}
break;
case 4:
pg_num = PCIE_ADDR_WB_PG (address);
pg_offs = PCIE_ADDR_WB_PG_OFFS (address);
SET_WB_PG (bar0, pg_num);
BAR4_RW(bar4, pg_offs, &data, rw_fpga);
if (verbose) {
fprintf (stdout, "%s from BAR4, data = 0x%08X, addr = 0x%08X, page = %d\n",
(rw_fpga)? "Reading":"Writing", data, address, pg_num);
}
break;
default:
fprintf (stderr, "Invalid BAR number, %d\n", barno);
goto exit_unmap_bar4;
}
/* Output Reading value only */
if (rw_fpga) {
fprintf (stdout, "0x%08X\n", data);
}
exit_unmap_bar4:
pd_unmapBAR (dev, 4, bar4);
exit_unmap_bar2:
pd_unmapBAR (dev, 2, bar2);
exit_unmap_bar0:
pd_unmapBAR (dev, 0, bar0);
exit_close:
pd_close (dev);
exit:
free (devicefile_str);
free (barno_str);
free (address_str);
free (data_str);
return 0;
}
int main(int argc, char **argv)
{
unsigned int *data_samples;
int i,val,j;
int size, new_size;
pd_device_t dev;
void *memr, *mems, *mem_new;
float throughput;
struct timeval tv1,tv2;
int mintime, curtime, vsize;
printf("====================================\n");
printf("= Pattern Tests =\n");
printf("====================================\n\n");
if(pd_open(0,&dev) != 0){
printf("Failed to open file \n");
return -1;
}
printf("Device file opened successfuly\n");
// Allocate send and receive buffers
size = 4*1024*1024; // 32x32 matrix with 4 byte entries
posix_memalign( (void**)&memr, 16, size);
posix_memalign( (void**)&mems, 16, size);
//for(i=0;i < size/4; i++){
// ((int*)mems)[i] = i;
// printf("%d ",((int*)mems)[i]);
//}
/*
vsize = 1;
for(i=1; i < 16; i++)
{
mintime = 99999;
for(j=0; j < 500; j++)
{
gettimeofday(&tv1,NULL);
compressData(mems, &mem_new, &new_size, 0, 4, 120, vsize);
gettimeofday(&tv2,NULL);
curtime = tv2.tv_usec - tv1.tv_usec;
if (curtime < mintime)
mintime = curtime;
free(mem_new);
}
printf("Total Size: %d, Time elapsed: %d\n", new_size, mintime);
vsize *= 2;
}
return;
*/
// Each 5x5 subblock will hold the same value
val = 0;
for(i=0; i < size/4; i++){
if (i % 16 == 0 && i!=0)
val ++;
val = val % 2;
((int *)mems)[i] = val;
}
// Test matrix
for(i=0; i < size/4; i++){
if (i%32 == 0 && i!=0)
printf("\n");
printf("%d ", ((int*)mems)[i]);
}
//--------------------------
if(initDMA(&dev) < 0)
printf("Init DMA failed\n");
// Send and receive in simultaneous
gettimeofday(&tv1,NULL);
#ifdef SEND
if(setupSend(&dev, (void*)mems, size,1) < 0)
printf("Setup Send failed\n");
if(applyBlocking_send(16, 32, 4) < 0)
printf("Apply Blocking send failed\n");
#endif
if(setupRecv(&dev, (void*)memr, size) < 0)
printf("Setup Recv failed\n");
//int applyLinear_recv(int offset, int hsize, int stride, int total_size);
// if(applyBlocking_recv(16, 32, 4) < 0)
if(applyLinear_recv(0,128,128,4096) < 0)
printf("Apply Blocking recv failed\n");
gettimeofday(&tv2,NULL);
printf("Time taken to setup receive AND send transfer:%d\n", tv2.tv_usec - tv1.tv_usec);
//--------------------------
gettimeofday(&tv1,NULL);
#ifdef SEND
//if(startSend(&dev, 0) < 0)
// printf("DMA Send failed\n");
#endif
// printf("Press any key to continue ;)\n");
// getchar();
if(startRecv(&dev, 0) < 0)
printf("DMA Recv failed\n");
if(startSend(&dev, 0) < 0)
printf("DMA Send failed\n");
if(checkRecv() < 0)
printf("Check DMA recv failed\n");
#ifdef SEND
if(checkSend() < 0)
printf("Check DMA send failed\n");
#endif
gettimeofday(&tv2,NULL);
curtime = tv2.tv_usec - tv1.tv_usec;
printf("Time taken to complete and check send AND receive transfer:%d\n", curtime);
printf("Throughput: %f MB/s\n",(float)2*size/((float)(curtime)));
// Test matrix
for(i=0; i < size/4; i++){
if (i%32 == 0 && i!=0)
printf("\n");
printf("%d ", ((int*)memr)[i]);
}
freeSend(&dev);
freeRecv(&dev);
stopDMA(&dev);
//for(i=0;i<size/4;i++)
// printf("%d ",((unsigned int*)memr)[i]);
free(memr); free(mems);
return 0;
}
