uint8_t * get_afu_psa_addr(char * i_afu_path) {
/*------------------------------------------------------------------------*/
/* Local Variables */
/*------------------------------------------------------------------------*/
struct cxl_afu_h * afu_master_h = NULL;
int afu_master_fd = 0;
uint8_t * afu_psa_addr = NULL;
/*------------------------------------------------------------------------*/
/* Code */
/*------------------------------------------------------------------------*/
if ((afu_master_h = cxl_afu_open_dev(i_afu_path)) < 0) {
TRACEE("Unable to open AFU Master cxl device. errno = %d (%s).\n",
errno, strerror(errno));
return NULL;
}
TRACEI("Opened %p, now attempting to get FD.\n", afu_master_h);
afu_master_fd = cxl_afu_fd(afu_master_h);
TRACEI("Got FD! = %d\n", afu_master_fd);
if (cxl_afu_attach(afu_master_h, 0)) {
TRACEE("Call to cxl_afu_attach failed. errno = %d (%s)\n",
errno, strerror(errno));
return NULL;
}
afu_psa_addr = mmap_afu_registers(afu_master_fd);
if (!afu_psa_addr) {
TRACEE("Error attempting to map AFU problem state registers. \
errno = %d (%s)\n", errno, strerror(errno));
return NULL;
}
/*
* Open AFU Master Device
*/
static int afu_m_open(struct mdev_ctx *mctx)
{
int rc = 0;
char device[64];
long api_version, cr_device, cr_vendor;
sprintf(device, "/dev/cxl/afu%d.0m", mctx->card);
VERBOSE3("[%s] Enter, Open Device: %s\n", __func__, device);
mctx->afu_h = cxl_afu_open_dev(device);
if (NULL == mctx->afu_h) {
VERBOSE0("[%s] Exit, Card Open error rc: %d\n", __func__, rc);
return -1;
}
/* Check if the compiled in API version is compatible with the
one reported by the kernel driver */
rc = cxl_get_api_version_compatible(mctx->afu_h, &api_version);
if ((rc != 0) || (api_version != CXL_KERNEL_API_VERSION)) {
VERBOSE0(" [%s] ERR: incompatible API version: %ld/%d rc=%d\n",
__func__, api_version, CXL_KERNEL_API_VERSION, rc);
rc = -2;
goto err_afu_free;
}
/* Check vendor id */
rc = cxl_get_cr_vendor(mctx->afu_h, 0, &cr_vendor);
if ((rc != 0) || (cr_vendor != CGZIP_CR_VENDOR)) {
VERBOSE0(" [%s] ERR: vendor_id: %ld/%d rc=%d\n",
__func__, (unsigned long)cr_vendor,
CGZIP_CR_VENDOR, rc);
rc = -3;
goto err_afu_free;
}
/* Check device id */
rc = cxl_get_cr_device(mctx->afu_h, 0, &cr_device);
if ((rc != 0) || (cr_device != CGZIP_CR_DEVICE)) {
VERBOSE0(" [%s] ERR: device_id: %ld/%d rc=%d\n",
__func__, (unsigned long)cr_device,
CGZIP_CR_VENDOR, rc);
rc = -4;
goto err_afu_free;
}
/* If we cannot get it, continue with warning ... */
mctx->errinfo = NULL;
rc = cxl_errinfo_size(mctx->afu_h, &mctx->errinfo_size);
if (0 == rc) {
mctx->errinfo = malloc(mctx->errinfo_size);
if (mctx->errinfo == NULL) {
rc = -5;
goto err_afu_free;
}
} else
VERBOSE0(" [%s] WARN: Cannot retrieve errinfo size rc=%d\n",
__func__, rc);
rc = cxl_afu_attach(mctx->afu_h, (__u64)(unsigned long)
(void *)mctx->wed);
if (0 != rc) {
rc = -6;
goto err_free_errinfo;
}
rc = cxl_mmio_map(mctx->afu_h, CXL_MMIO_BIG_ENDIAN);
if (rc != 0) {
rc = -7;
goto err_free_errinfo;
}
return 0;
err_free_errinfo:
if (mctx->errinfo)
free(mctx->errinfo);
mctx->errinfo = NULL;
err_afu_free:
cxl_afu_free(mctx->afu_h);
mctx->afu_h = NULL;
VERBOSE3("[%s] Exit rc=%d\n", __func__, rc);
return rc;
}
bool initialize_wwpn(char* i_afu_path, const AFU_PORT_ID i_port, const uint64_t i_wwpn)
{
//Locals
struct cxl_afu_h *afu_master_h = NULL;
int afu_master_fd = 0;
uint8_t *afu_psa_addr = NULL;
uint32_t mmio_data32 = 0;
uint64_t mmio_data = 0;
int l_fc_port_offset = 0; //todo: replace all references with "l_fc_port_offset"
bool l_rc = 0;
//Code
do
{
if(i_port >= MAX_WWPNS_PER_ADAPTER)
{
TRACEE("Port number %d is not a valid port number. We only support up to %d numbers.\n", i_port, MAX_WWPNS_PER_ADAPTER);
l_rc = false;
break;
}
if(i_wwpn == 0)
{
TRACEE("WWPN for %s port %d is zero and therefore invalid.\n", i_afu_path, i_port);
l_rc = false;
break;
}
l_fc_port_offset = FC_PORT_BASE_OFFSET + (i_port * FC_PORT_REG_SIZE);
TRACEV("Target reg bank for port %d = 0x%08x\n",i_port, l_fc_port_offset);
TRACEI("Attempting to open device '%s'\n", i_afu_path);
/* now that the AFU is started, lets set config options */
if ((afu_master_h = cxl_afu_open_dev(i_afu_path)) < 0) {
TRACEE("Unable to open AFU Master cxl device. errno = %d (%s)\n", errno, strerror(errno));
l_rc = false;
break;
}
TRACEV("Opened %p, now attempting to get FD\n", afu_master_h);
afu_master_fd = cxl_afu_fd(afu_master_h);
TRACEV("Got FD! = %d\n", afu_master_fd);
if (cxl_afu_attach(afu_master_h,
0)) //no WEQ needed
{
TRACEE("Call to cxl_afu_attach failed. errno = %d (%s)\n", errno, strerror(errno));
l_rc = false;
break;
}
afu_psa_addr = mmap_afu_wwpn_registers(afu_master_fd);
if (!afu_psa_addr) {
TRACEE("Error attempting to map AFU problem state registers. errno = %d (%s)\n", errno, strerror(errno));
l_rc = -1;
break;
}
//Take port offline
//TRACED("Bringing port offline\n");
//get status bits for debug
//TODO: maybe wrap up this print into a nice macro...
mmio_data32 = in_mmio32((__u64*)&afu_psa_addr[FC_MTIP_STATUS + l_fc_port_offset]);
TRACEV("FC_MTIP_STATUS (0x%08X): 0x%08X\n", FC_MTIP_STATUS + l_fc_port_offset, mmio_data32);
mmio_data32 = in_mmio32((__u64*)&afu_psa_addr[FC_MTIP_CMDCONFIG + l_fc_port_offset]);
TRACEV("FC_MTIP_CMDCONFIG (0x%08X): 0x%08X\n", FC_MTIP_CMDCONFIG + l_fc_port_offset, mmio_data32);
mmio_data32 &= ~0x20; // clear ON_LINE
mmio_data32 |= 0x40; // set OFF_LINE
TRACEV("FC_MTIP_CMDCONFIG: Proposed: 0x%08X\n", mmio_data32);
out_mmio32((__u64*)&afu_psa_addr[FC_MTIP_CMDCONFIG + l_fc_port_offset], mmio_data32);
//wait for the port to be offline
l_rc = wait_port_state(afu_psa_addr, l_fc_port_offset, false);
if(l_rc == false)
{
TRACEE("Port not offline in time. Aborting.\n");
l_rc = -1;
break;
}
//now we know we are offline, so write the PN...
//read out the current PN
//PN register is 64-bit as per spec
mmio_data = in_mmio64((__u64*)&afu_psa_addr[FC_PNAME + l_fc_port_offset]);
TRACEV("FC_PNAME: (0x%08X): 0x%"PRIx64"\n",FC_PNAME + l_fc_port_offset, mmio_data);
TRACEI("Current Port Name is 0x%"PRIx64"\n", mmio_data);
mmio_data = i_wwpn;
TRACEI("New Port Name will be 0x%"PRIx64"\n", mmio_data);
out_mmio64((__u64*)&afu_psa_addr[FC_PNAME + l_fc_port_offset], mmio_data);
//bring the port back online
//read control bits...
mmio_data32 = in_mmio32((__u64*)&afu_psa_addr[FC_MTIP_CMDCONFIG + l_fc_port_offset]);
TRACEV("FC_MTIP_CMDCONFIG (0x%08X): 0x%08X\n", FC_MTIP_CMDCONFIG + l_fc_port_offset, mmio_data32);
mmio_data32 |= 0x20; // set ON_LINE
mmio_data32 &= ~0x40; // clear OFF_LINE
//TODO: ask todd - should we explicitly-set other bits? I needed to force the port into FC mode (set bit 1)
//net result is we write 0x23 to the lowest byte
TRACEV("FC_MTIP_CMDCONFIG: Proposed: 0x%08X\n", mmio_data32);
out_mmio32((__u64*)&afu_psa_addr[FC_MTIP_CMDCONFIG + l_fc_port_offset], mmio_data32);
//wait for the port to be online
l_rc = wait_port_state(afu_psa_addr, l_fc_port_offset, true);
if(l_rc == false)
{
TRACEE("Port not online in time.\n");
l_rc = -1;
break;
}
//done!
munmap_afu_wwpn_registers((void *) afu_psa_addr);
cxl_afu_free(afu_master_h);
TRACEI("Successfully programmed WWPN!\n");
l_rc = true;
} while (0);
return l_rc;
}
bool loopbackDiag(char* i_afu_path, const AFU_PORT_ID i_port, uint64_t i_test_time_us)
{
//Locals
struct cxl_afu_h *afu_master_h = NULL;
int afu_master_fd = 0;
uint8_t *afu_psa_addr = NULL;
uint64_t mmio_data = 0;
int l_fc_port_offset = 0;
bool l_rc = true;
uint64_t l_init_txcount = 0;
uint64_t l_init_errcount = 0;
uint64_t l_init_passcount = 0;
uint64_t l_txcount = 0;
uint64_t l_errcount = 0;
uint64_t l_passcount = 0;
//Code
do
{
if(i_port >= MAX_WWPNS_PER_ADAPTER)
{
TRACEE("Port number %d is not a valid port number. We only support up to %d numbers.\n", i_port, MAX_WWPNS_PER_ADAPTER);
l_rc = false;
break;
}
l_fc_port_offset = FC_PORT_BASE_OFFSET + (i_port * FC_PORT_REG_SIZE);
TRACEV("Target reg bank for port %d = 0x%08x\n",i_port, l_fc_port_offset);
TRACEI("Attempting to open device '%s'\n", i_afu_path);
/* now that the AFU is started, lets set config options */
if ((afu_master_h = cxl_afu_open_dev(i_afu_path)) < 0) {
TRACEE("Unable to open AFU Master cxl device. errno = %d (%s)\n", errno, strerror(errno));
l_rc = false;
break;
}
TRACEV("Opened %p, now attempting to get FD\n", afu_master_h);
afu_master_fd = cxl_afu_fd(afu_master_h);
TRACEV("Got FD! = %d\n", afu_master_fd);
if (cxl_afu_attach(afu_master_h,
0)) //no WEQ needed
{
TRACEE("Call to cxl_afu_attach failed. errno = %d (%s)\n", errno, strerror(errno));
l_rc = false;
break;
}
afu_psa_addr = mmap_afu_wwpn_registers(afu_master_fd);
if (!afu_psa_addr) {
TRACEE("Error attempting to map AFU problem state registers. errno = %d (%s)\n", errno, strerror(errno));
l_rc = false;
break;
}
//Loopback enables a special echo mode and checks tx and rx counts to make sure
//the phy works correctly. The test is relatively short.
TRACEI("Enabling loopback mode for adapter %s, port %d\n", i_afu_path, i_port);
//first read back all test register counts. The AFU does not zero them
l_init_txcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_TXCNT + l_fc_port_offset]);
l_init_errcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_ERRCNT + l_fc_port_offset]);
l_init_passcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_PASSCNT + l_fc_port_offset]);
//enable the test
mmio_data = in_mmio64((__u64*)&afu_psa_addr[FC_CONFIG2 + l_fc_port_offset]);
mmio_data |= (uint64_t)0x01 << 40; //set ECHO generator for this port (bit 40)
out_mmio64((__u64*)&afu_psa_addr[FC_CONFIG2 + l_fc_port_offset], mmio_data);
TRACEI("Waiting %"PRIu64" microseconds for the test to complete...\n", i_test_time_us);
usleep(i_test_time_us);
TRACEI("Disabling loopback mode.\n");
mmio_data = in_mmio64((__u64*)&afu_psa_addr[FC_CONFIG2 + l_fc_port_offset]);
mmio_data &= ~((uint64_t)0x01 << 40); //clear ECHO generator for this port (bit 40)
out_mmio64((__u64*)&afu_psa_addr[FC_CONFIG2 + l_fc_port_offset], mmio_data);
TRACEI("Waiting for the link to quiesce...\n");
usleep(LOOPBACK_TEST_QUIET_PERIOD);//wait for quiesce and any final timeouts
//check test results from HW - subtract out initial readings
l_txcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_TXCNT + l_fc_port_offset]) - l_init_txcount;
l_errcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_ERRCNT + l_fc_port_offset]) - l_init_errcount;
l_passcount = in_mmio64((__u64*)&afu_psa_addr[FC_LOOPBACK_PASSCNT + l_fc_port_offset]) - l_init_passcount;
if((l_txcount == 0) ||
(l_errcount != 0) ||
(l_txcount != l_passcount))
{
TRACED("Loopback diagnostic failure detected. AFU: %s, port: %d, Tx Count: %"PRIu64", Pass Count: %"PRIu64", Error Count: %"PRIu64"\n", i_afu_path, i_port, l_txcount, l_passcount, l_errcount);
l_rc = false;
}
else
{
TRACEV("Loopback test passed. AFU: %s, port: %d, Tx Count: %"PRIu64", Pass Count: %"PRIu64", Error Count: %"PRIu64"\n", i_afu_path, i_port, l_txcount, l_passcount, l_errcount);
l_rc = true;
}
//done!
munmap_afu_wwpn_registers((void *) afu_psa_addr);
cxl_afu_free(afu_master_h);
} while (0);
return l_rc;
}
int main (int argc, char **argv)
{
int ret;
int i, count, data_size;
uint64_t mmio_state = 0;
struct wed * capi_wed = NULL;
struct wed_tx * p_wed_tx = NULL;
char cxl_device [64];
__u8 * source_buf, * result_buf;
int16_t * ptr = NULL;
struct cxl_afu_h * afu_h = NULL;
FILE *fp;
int send_num, received_num, loops;
struct timeval start;
struct timeval end;
double interval;
if(argc != 2)
{
printf("uasge: %s imagepath \n", argv[0]);
exit(1);
}
// Malloc buffer for the work element descriptor
// posix_memalign is used to keep alignment requirement and make DMA engine simple
ret = posix_memalign ((void **) &capi_wed, CACHELINE_BYTES, sizeof(struct wed));
if (ret) {
printf ("Error. Can not malloc buffer for wed.\n");
return -1;
}
ret = posix_memalign ((void **) &p_wed_tx, CACHELINE_BYTES, sizeof(struct wed_tx));
if (ret) {
printf ("Error. Can not malloc buffer for wed.\n");
return -1;
}
// Malloc buffer for the source data
// posix_memalign is used to keep alignment requirement and make DMA engine simple
ret = posix_memalign ((void **) &source_buf, CACHELINE_BYTES, DATA_SIZE);
if (ret) {
printf ("Error. Can not malloc buffer for source buffer.\n");
free (capi_wed);
return -1;
}
// Malloc buffer for the result
// posix_memalign is used to keep alignment requirement and make DMA engine simple
ret = posix_memalign ((void **) &result_buf, CACHELINE_BYTES, DATA_SIZE);
if (ret) {
printf ("Error. Can not malloc buffer for result buffer.\n");
free (capi_wed);
free (source_buf);
return -1;
}
if ((fp = fopen(argv[1], "r")) == NULL)
{
printf ("Image file can not be opened.\n");
exit(1);
}
//Read the input data from the file, the data should be 16bits,
//You can modify here to for your own data.
ptr = (int16_t*) source_buf;
count = 0;
while(fscanf(fp, "%hd", ptr++) > 0)
{
count ++;
if (count >= DATA_SIZE / sizeof(int16_t))
{
printf ("Buff overflow.\n");
exit(1);
}
}
data_size = count * sizeof(int16_t);
//The input data buffer should be aligined with cache line (128bytes)
if(data_size % CACHELINE_BYTES)
{
data_size = (data_size / CACHELINE_BYTES + 1) * CACHELINE_BYTES;
}
capi_wed->data_size = data_size;
capi_wed->param_s0 = 0x11223344;
capi_wed->param_s1 = 0x55667788;
capi_wed->source = source_buf;
capi_wed->result = result_buf;
capi_wed->p_wed_tx = (__u8 *)p_wed_tx;
//Detect which device is avaliable
strncpy (cxl_device, DEVICE, 64);
for (ret = 0; ret < 8; ret ++) {
if (access(cxl_device, W_OK) == 0)
break;
cxl_device [12] ++;
}
if (ret == 8) {
printf ("Can not find available CAPI device.\n");
return -1;
}
//Open CAPI device
afu_h = cxl_afu_open_dev (cxl_device);
if (!afu_h) {
printf ("Error. Can not open CAPI device : %s\n", DEVICE);
return -1;
}
printf ("Open CAPI device %s\n", cxl_device);
// Create the CAPI hardware thread, AFU is enabled
// capi_wed pointer is sent to the AFU
// The DMA dose not run at this moment, it will only read the capi_wed struction
// to the DAM engine
cxl_afu_attach (afu_h, (__u64) capi_wed);
printf ("Attach AFU to current application.\n");
// Map some register from AFU to user space to do runtime control
// This is an optinal step for CAPI application development dependent on AFU
if ((cxl_mmio_map (afu_h, CXL_MMIO_BIG_ENDIAN)) < 0) {
printf ("Error. Can not map registers\n");
return -1;
}
// We clear the status to Zero
capi_wed->status = 0;
capi_wed->jcounter = 0;
capi_wed->ret_size = 0;
// We use a register to show whether the AFU is ready
// This is an optinal step for CAPI application development dependent on AFU
do {
cxl_mmio_read64 (afu_h, MMIO_TRACE_ADDR, &mmio_state);
} while ((mmio_state & 0xf) != 0x1);
gettimeofday(&start, NULL);
//Start the data receive DMA
cxl_mmio_write64 (afu_h, MMIO_TRACE_ADDR, 0xf0);
//Start the data send DMA
cxl_mmio_write64 (afu_h, MMIO_TRACE_ADDR, 0x0f);
loops = 100;
send_num = 1;
received_num = 0;
while(1)
{
if(capi_wed->status)
{
received_num ++;
if (received_num >= loops)break;
//Clear the status bit
capi_wed->status = 0;
cxl_mmio_write64 (afu_h, MMIO_TRACE_ADDR, 0xf0);
}
if (p_wed_tx -> status)
{
p_wed_tx->status = 0;
if (send_num < loops)
{
//clear the status bit
p_wed_tx->status = 0;
cxl_mmio_write64 (afu_h, MMIO_TRACE_ADDR, 0x0f);
send_num ++;
}
}
}
gettimeofday(&end, NULL);
interval = ((end.tv_sec * 1000000 + end.tv_usec) - (start.tv_sec * 1000000 + start.tv_usec)) / 1000000.0;
printf ("The %dst job finish. Return Size = 0x%x\n", capi_wed->jcounter, capi_wed->ret_size);
printf ("The total time cost (%d loops) is %.3fs, each loop cost is %.3fms.\n", loops, interval, interval/loops * 1000.0);
ptr = (int16_t*) result_buf;
printf ("Recognition Results (without softmax):\n");
for (i = 0; i < 10; i++)
{
printf("%d \t", *ptr ++);
}
printf ("\n");
free (capi_wed);
free (source_buf);
free (result_buf);
// We use register write to trigger the DAM to finish all jobs
// This is an optinal step for CAPI application development dependent on AFU
cxl_mmio_write64 (afu_h, MMIO_TRACE_ADDR, 0xf1);
// We do register unmap because we map register above
cxl_mmio_unmap (afu_h);
// Close the CAPI device
cxl_afu_free (afu_h);
return 0;
}
