/* Allocates two pointers of given sizes and random characteristics (alignment,
overlapping, relative-position). */
static void get_random_two_pointers (uword size1, uword size2,
void ** ptr1, void ** ptr2,
void ** to_free1, void ** to_free2)
{
uword do_align1, do_align2;
uword align1, align2;
uword overlap, two_less_that_one;
uword min;
do_align1 = bounded_random_u32 (&g_seed, 0, 1);
align1 = (do_align1) ? bounded_random_u32 (&g_seed, 0, MAX_LOG2_ALIGN) : (0);
do_align2 = bounded_random_u32 (&g_seed, 0, 1);
align2 = (do_align2) ? bounded_random_u32 (&g_seed, 0, MAX_LOG2_ALIGN) : (0);
overlap = bounded_random_u32 (&g_seed, 0, 1);
two_less_that_one = (overlap) ? bounded_random_u32 (&g_seed, 0, 1) : (0);
/* Pick random distance between pointers. */
if (overlap)
{
min = clib_min (size1, size2);
overlap = bounded_random_u32 (&g_seed, 0, (min) ? (min - 1) : (0));
}
get_two_pointers (size1, size2, align1, align2,
overlap, two_less_that_one,
ptr1, ptr2, to_free1, to_free2);
VERBOSE3 ("size1 %u, size2 %u, align1 %u, align2 %u, overlap %u, "
"two_less_that_one %u\n",
size1, size2, align1, align2, overlap, two_less_that_one);
VERBOSE3 ("p1 %U (%p)\n", format_u32_binary, *ptr1, *ptr1);
VERBOSE3 ("p2 %U (%p)\n", format_u32_binary, *ptr2, *ptr2);
}
static int mmio_write(struct cxl_afu_h *afu_h, int ctx, uint32_t offset,
uint64_t data)
{
int rc = -1;
uint32_t offs = (ctx * MMIO_CTX_OFFSET) + offset;
VERBOSE3("[%s] Enter, Offset: 0x%x data: 0x%016llx\n",
__func__, offs, (long long)data);
rc = cxl_mmio_write64(afu_h, offs, data);
VERBOSE3("[%s] Exit, rc = %d\n", __func__, rc);
return rc;
}
static int mmio_read(struct cxl_afu_h *afu_h, int ctx, uint32_t offset,
uint64_t *data)
{
int rc = -1;
uint32_t offs = (ctx * MMIO_CTX_OFFSET) + offset;
VERBOSE3("[%s] Enter, CTX: %d Offset: 0x%x\n", __func__, ctx, offs);
rc = cxl_mmio_read64(afu_h, offs, data);
VERBOSE3("[%s] Exit, rc = %d data: 0x%016llx\n",
__func__, rc, (long long)*data);
return rc;
}
static int afu_m_close(struct mdev_ctx *mctx)
{
VERBOSE3("[%s] Enter\n", __func__);
if (NULL == mctx->afu_h)
return -1;
cxl_mmio_unmap(mctx->afu_h);
cxl_afu_free(mctx->afu_h);
mctx->afu_h = NULL;
if (mctx->errinfo)
free(mctx->errinfo);
mctx->errinfo = NULL;
VERBOSE3("[%s] Exit\n", __func__);
return 0;
}
/* Allocates randomly-aligned pointer to memory of given size. */
static void get_random_pointer (uword size, void ** ptr, void ** to_free)
{
uword do_align = bounded_random_u32 (&g_seed, 0, 1);
uword align, offset;
if (do_align)
{
align = bounded_random_u32 (&g_seed, 0, MAX_LOG2_ALIGN);
*ptr = alloc_aligned (size, align, to_free);
VERBOSE3 ("size %u, align %u\n", size, align);
}
else
{
offset = bounded_random_u32 (&g_seed, 0, MAX_UNALIGN_OFFSET);
*ptr = alloc_unaligned (size, offset, to_free);
VERBOSE3 ("size %u, offset %u\n", size, offset);
}
VERBOSE3 ("ptr %U (%p)\n", format_u32_binary, *ptr, *ptr);
}
int
xfreelist_release_item(xfreelist_t* list,xfreelist_item_t* item){
int fstatus=XERROR;
char* function_name="xfreelist_release_item";
INIT_DEBUG3_MARK();
/* check that this item is linked with this freelist */
if( list->items <= item && item < list->items + list->item_nb ) {
if(item->free){
fstatus=XERROR_FREELIST_ITEM_ALREADY_FREE;
}
else{
item->free=1;
item->previous=NULL;
if(list->head==NULL){
/* freelist empty */
item->next=NULL;
list->head=item;
list->tail=item;
}
else{
/* freelist not empty */
item->next=list->head;
list->head->previous=item;
list->head=item;
}
VERBOSE3("item '%x' successfully released to freelist '%x'",item,list);
fstatus=XSUCCESS;
}
}
/* this item is not linked with this list, we have to try sub lists */
else {
if ( list->next != NULL ) {
VERBOSE("item '%x' is not linked to list '%x', releasing using sublist '%x'",item,list,list->next);
fstatus=xfreelist_release_item(list->next,item);
}
else {
VERBOSE("item '%x' is not linked to list '%x'",item,list);
fstatus=XERROR_FREELIST_ITEM_NOT_FOUND;
}
}
EXIT_DEBUG3_MARK(fstatus);
return fstatus;
}
int
xfreelist_extract_item(xfreelist_t* list,xfreelist_item_t** pitem){
int fstatus=-1;
char* function_name="xfreelist_extract_item";
INIT_DEBUG3_MARK();
xfreelist_item_t* item;
/* if list is empty, just try to extract from a next list */
if(list->head==NULL){
VERBOSE("no more items in list '%x'",list);
if ( list->next == NULL)
xfreelist_extend(list);
if ( list->next != NULL) {
fstatus=xfreelist_extract_item(list->next,pitem);
}
else
fstatus=XERROR_FREELIST_IS_EMPTY;
}
else {
/* get item from head */
item=list->head;
/* shift freelist */
list->head=item->next;
if(list->head!=NULL)
list->head->previous=NULL;
else
list->tail=NULL;
item->next=NULL;
item->previous=NULL;
item->free=0;
*pitem=item;
VERBOSE3("item '%x' successfully extracted from freelist '%x'",item,list);
fstatus=XSUCCESS;
}
EXIT_DEBUG3_MARK(fstatus);
return fstatus;
}
static int afu_check_stime(struct mdev_ctx *mctx)
{
int gsel, bsel = 0, ctx = 0;
uint64_t gmask = 0, qstat_reg, err_reg, mstat_reg;
uint64_t wtime;
uint64_t cid_reg;
int n_act = 0;
uint64_t s_time = 0;
char s[32];
for (gsel = 0; gsel < MMIO_CASV_REG_NUM; gsel++) {
mmio_read(mctx->afu_h, MMIO_MASTER_CTX_NUMBER,
MMIO_CASV_REG + (gsel*8), &gmask);
if (0 == gmask)
continue; /* No bit set, Skip */
for (bsel = 0; bsel < MMIO_CASV_REG_CTX; bsel++) {
if (0 == (gmask & (1ull << bsel)))
continue; /* Skip */
ctx = (gsel * MMIO_CASV_REG_CTX) + bsel; /* Active */
mmio_read(mctx->afu_h, ctx+1, MMIO_DDCBQ_STATUS_REG, &qstat_reg);
if (0 == (qstat_reg & 0xffffffff00000000ull)) {
VERBOSE3("AFU[%d:%03d] master skip\n",
mctx->card, ctx);
continue; /* Skip Master */
}
mmio_read(mctx->afu_h, ctx+1, MMIO_DDCBQ_WT_REG, &wtime);
wtime = wtime / 250; /* makes time in usec */
mmio_read(mctx->afu_h, ctx+1, MMIO_DDCBQ_CID_REG, &cid_reg);
uint16_t cur_cid = (uint16_t)(cid_reg >> 16); /* Currect Context id */
uint16_t my_cid = (uint16_t)(cid_reg & 0xffff); /* My Context id */
mmio_read(mctx->afu_h, ctx+1, MMIO_DDCBQ_DMAE_REG, &err_reg);
uint16_t cseq = (uint16_t)(qstat_reg >> 48ull); /* Currect sequence */
uint16_t lseq = (uint16_t)(qstat_reg >> 32ull); /* Last sequence */
uint8_t qidx = (uint8_t)(qstat_reg >> 24); /* Q Index */
uint16_t qnfe = (uint16_t)(qstat_reg >> 8); /* Context Non Fatal Error Bits */
uint8_t qstat = (uint8_t)(qstat_reg & 0xff); /* Context Status */
/* Generate W for Waiting, I for Idle and R for Running */
char flag = 'W'; /* Default Context is Waiting to get executed */
if ((lseq + 1 ) == cseq)
flag = 'I'; /* Context is Idle, nothing to do */
else if (0x30 == qstat) /* if Bits 4 + 5 on ? */
flag = 'R'; /* Context is Running */
if (qnfe) {
VERBOSE0("AFU[%d:%03d] ERR: CurrentCtx: %03d MyCtx: %03d CS: %04X LS: %04X ",
mctx->card, ctx, cur_cid, my_cid, cseq, lseq);
VERBOSE0("[%c] IDX: %02d QNFE: %04x QSTAT: %02x Time: %lld usec",
flag, qidx, qnfe, qstat, (long long)wtime);
if (0 != err_reg)
VERBOSE0("DMA Err: 0x%016llx", (long long)err_reg);
VERBOSE0("\n");
} else {
VERBOSE0("AFU[%d:%03d] CurrentCtx: %03d MyCtx: %03d CS: %04X LS: %04X ",
mctx->card, ctx, cur_cid, my_cid, cseq, lseq);
VERBOSE0("[%c] IDX: %02d QNFE: %04x QSTAT: %02x Time: %lld usec",
flag, qidx, qnfe, qstat, (long long)wtime);
if (0 != err_reg)
VERBOSE0("DMA Err: 0x%016llx", (long long)err_reg);
VERBOSE0("\n");
}
n_act++;
s_time += wtime;
}
}
if (n_act) {
time_t result = time(NULL);
struct tm * p = localtime(&result);
strftime(s, 32, "%T", p);
VERBOSE0("AFU[%d:XXX] at %s Running %d Active Contexts total %lld msec",
mctx->card, s, n_act, (long long)s_time/1000);
mmio_read(mctx->afu_h, MMIO_MASTER_CTX_NUMBER,
MMIO_AFU_STATUS_REG, &mstat_reg);
if (0 != mstat_reg)
VERBOSE0(" Status: 0x%016llx", (long long)mstat_reg);
VERBOSE0("\n");
}
return mctx->dt;
}
/*
* Open AFU Master Device
*/
static int afu_m_open(struct mdev_ctx *mctx)
{
int rc = 0;
char device[64];
long api_version, cr_device, cr_vendor;
sprintf(device, "/dev/cxl/afu%d.0m", mctx->card);
VERBOSE3("[%s] Enter, Open Device: %s\n", __func__, device);
mctx->afu_h = cxl_afu_open_dev(device);
if (NULL == mctx->afu_h) {
VERBOSE0("[%s] Exit, Card Open error rc: %d\n", __func__, rc);
return -1;
}
/* Check if the compiled in API version is compatible with the
one reported by the kernel driver */
rc = cxl_get_api_version_compatible(mctx->afu_h, &api_version);
if ((rc != 0) || (api_version != CXL_KERNEL_API_VERSION)) {
VERBOSE0(" [%s] ERR: incompatible API version: %ld/%d rc=%d\n",
__func__, api_version, CXL_KERNEL_API_VERSION, rc);
rc = -2;
goto err_afu_free;
}
/* Check vendor id */
rc = cxl_get_cr_vendor(mctx->afu_h, 0, &cr_vendor);
if ((rc != 0) || (cr_vendor != CGZIP_CR_VENDOR)) {
VERBOSE0(" [%s] ERR: vendor_id: %ld/%d rc=%d\n",
__func__, (unsigned long)cr_vendor,
CGZIP_CR_VENDOR, rc);
rc = -3;
goto err_afu_free;
}
/* Check device id */
rc = cxl_get_cr_device(mctx->afu_h, 0, &cr_device);
if ((rc != 0) || (cr_device != CGZIP_CR_DEVICE)) {
VERBOSE0(" [%s] ERR: device_id: %ld/%d rc=%d\n",
__func__, (unsigned long)cr_device,
CGZIP_CR_VENDOR, rc);
rc = -4;
goto err_afu_free;
}
/* If we cannot get it, continue with warning ... */
mctx->errinfo = NULL;
rc = cxl_errinfo_size(mctx->afu_h, &mctx->errinfo_size);
if (0 == rc) {
mctx->errinfo = malloc(mctx->errinfo_size);
if (mctx->errinfo == NULL) {
rc = -5;
goto err_afu_free;
}
} else
VERBOSE0(" [%s] WARN: Cannot retrieve errinfo size rc=%d\n",
__func__, rc);
rc = cxl_afu_attach(mctx->afu_h, (__u64)(unsigned long)
(void *)mctx->wed);
if (0 != rc) {
rc = -6;
goto err_free_errinfo;
}
rc = cxl_mmio_map(mctx->afu_h, CXL_MMIO_BIG_ENDIAN);
if (rc != 0) {
rc = -7;
goto err_free_errinfo;
}
return 0;
err_free_errinfo:
if (mctx->errinfo)
free(mctx->errinfo);
mctx->errinfo = NULL;
err_afu_free:
cxl_afu_free(mctx->afu_h);
mctx->afu_h = NULL;
VERBOSE3("[%s] Exit rc=%d\n", __func__, rc);
return rc;
}
int xstream_receive_timeout(int socket,char* buffer,size_t length,int timeout){
int fstatus=-1;
int rc;
size_t read_bytes;
int sock_flags;
int nonblock=0;
struct pollfd ufds;
struct timeval start_time;
struct timeval current_time;
int timeleft;
/* set non block mode if required */
if(timeout!=0){
sock_flags=fcntl(socket,F_GETFL);
if(fcntl(socket,F_SETFL, sock_flags | O_NONBLOCK)){
ERROR("unable to set socket non-blocking flag : %s",strerror(errno));
return XERROR_STREAM_SETSOCKOPT_FAILED;
}
else{
VERBOSE("socket non-blocking flag is now set");
nonblock=1;
ufds.fd=socket;
ufds.events=POLLIN;
}
}
/* get start time */
gettimeofday(&start_time,NULL);
/* send data */
read_bytes=0;
while(read_bytes<length){
/* attempt polling if non block mode is activated */
if(nonblock){
VERBOSE3("looking for POLLIN events on socket %d",socket);
gettimeofday(¤t_time,NULL);
timeleft=timeout
-(current_time.tv_sec-start_time.tv_sec)*1000
-(current_time.tv_sec-start_time.tv_sec)/1000;
if(timeleft<=0){
ERROR("receive at %d of %d bytes : timeout",
read_bytes,length);
fstatus=XERROR_STREAM_TIMEOUT;
break;
}
if((rc=poll(&ufds,1,timeleft))<=0){
if(rc<0 && (errno==EINTR || errno==EAGAIN)){
continue;
}
else if(rc==0){
continue;
}
else if(rc<0){
ERROR("receive at %d of %d bytes : poll error : %s",
read_bytes,length,strerror(errno));
fstatus=XERROR_STREAM_POLL_ERROR;
break;
}
}
/* read data from socket */
rc=read(socket,buffer+read_bytes,length-read_bytes);
VERBOSE3("read return code is %d (errno=%d)",rc,errno);
}
else {
/* read data from socket */
do{
rc=read(socket,buffer+read_bytes,length-read_bytes);
VERBOSE3("read return code is %d (errno=%d)",rc,errno);
}
while(rc<0 && (errno==EINTR || errno==EAGAIN));
}
/*_*/ /* attempt polling if required */
/* process read return code */
if(rc>0)
read_bytes+=rc;
else if (rc==0) {
ERROR("receive at %d of %d bytes : 0 bytes received during read op",
read_bytes,length);
fstatus=XERROR_STREAM_SOCKET_CLOSED;
break;
}
else {
ERROR("receive at %d of %d bytes : bad return code on read op : %d",
read_bytes,length,rc);
fstatus=rc;
break;
}
}
if(read_bytes==length){
fstatus=XSUCCESS;
}
return fstatus;
}
int xstream_send_timeout(int socket,char* buffer,size_t length,int timeout){
int fstatus=XERROR;
int rc;
size_t written_bytes;
char test;
int sock_flags=0;
int nonblock=0;
struct pollfd ufds;
struct timeval start_time;
struct timeval current_time;
int timeleft;
/* set non block mode if required */
if(timeout!=0){
sock_flags=fcntl(socket,F_GETFL);
if(fcntl(socket,F_SETFL, sock_flags | O_NONBLOCK)){
ERROR("unable to set socket non-blocking flag : %s",strerror(errno));
return XERROR_STREAM_SETSOCKOPT_FAILED;
}
else{
VERBOSE("socket non-blocking flag is now set");
nonblock=1;
ufds.fd=socket;
ufds.events=POLLOUT;
}
}
/* get start time */
gettimeofday(&start_time,NULL);
/* send data */
written_bytes=0;
while(written_bytes<length){
/* attempt polling if non block mode is activated */
if(nonblock){
VERBOSE3("looking for POLLOUT events on socket %d",socket);
gettimeofday(¤t_time,NULL);
timeleft=timeout
-(current_time.tv_sec-start_time.tv_sec)*1000
-(current_time.tv_sec-start_time.tv_sec)/1000;
if(timeleft<=0){
ERROR("send at %d/%d bytes transmitted : timeout",
written_bytes,length);
fstatus=XERROR_STREAM_TIMEOUT;
break;
}
if((rc=poll(&ufds,1,timeleft))<=0){
if(rc<0 && (errno==EINTR || errno==EAGAIN)){
continue;
}
else if(rc==0){
continue;
}
else if(rc<0){
ERROR("send at %d/%d bytes transmitted : poll error : %s",
written_bytes,length,strerror(errno));
fstatus=XERROR_STREAM_POLL_ERROR;
break;
}
else{
/* we just check that the socket is still here */
/* read from a closed nonblocking socket should return 0 */
do{
rc=read(socket,&test,1);
}
while(rc<0 && errno==EINTR);
if(rc==0){
ERROR("send at %d/%d bytes transmitted : socket is gone",
written_bytes,length);
fstatus=XERROR_STREAM_SOCKET_CLOSED;
break;
}
}
}
/* send data */
rc=write(socket,buffer+written_bytes,length-written_bytes);
VERBOSE3("write return code is %d (errno=%d)",rc,errno);
}
else {
/* send data */
do{
rc=write(socket,buffer+written_bytes,length-written_bytes);
VERBOSE3("write return code is %d (errno=%d)",rc,errno);
}
while(rc<0 && (errno==EINTR || errno==EAGAIN));
}
/* process write return code */
if(rc>0)
written_bytes+=rc;
else if(rc) {
fstatus=rc;
break;
}
else
break;
}
/* reverse socket flags */
if(timeout!=0){
fcntl(socket,F_SETFL,sock_flags);
}
if(written_bytes==length){
fstatus=XSUCCESS;
}
return fstatus;
}
