/*
* rpmem_fip_read -- perform read operation
*/
int
rpmem_fip_read(struct rpmem_fip *fip, void *buff, size_t len, size_t off)
{
RPMEM_ASSERT(!rpmem_fip_lane_busy(&fip->rd_lane.lane));
int ret = 0;
size_t rd = 0;
uint8_t *cbuff = buff;
while (rd < len) {
rpmem_fip_lane_begin(&fip->rd_lane.lane, FI_READ);
size_t rd_len = len - rd < RPMEM_RD_BUFF_SIZE ?
len - rd : RPMEM_RD_BUFF_SIZE;
size_t rd_off = off + rd;
uint64_t raddr = fip->raddr + rd_off;
ret = rpmem_fip_readmsg(fip->ep, &fip->rd_lane.read,
fip->rd_buff, rd_len, raddr);
ret = rpmem_fip_lane_wait(&fip->rd_lane.lane, FI_READ);
if (ret)
return ret;
memcpy(&cbuff[rd], fip->rd_buff, rd_len);
rd += rd_len;
}
return ret;
}
/*
* rpmem_fip_read -- perform read operation
*/
int
rpmem_fip_read(struct rpmem_fip *fip, void *buff, size_t len, size_t off)
{
RPMEM_ASSERT(!rpmem_fip_lane_busy(&fip->rd_lane.lane));
int ret;
size_t rd = 0;
uint8_t *cbuff = buff;
while (rd < len) {
rpmem_fip_lane_begin(&fip->rd_lane.lane, FI_READ);
size_t rd_len = len - rd < RPMEM_RD_BUFF_SIZE ?
len - rd : RPMEM_RD_BUFF_SIZE;
size_t rd_off = off + rd;
uint64_t raddr = fip->raddr + rd_off;
ret = rpmem_fip_readmsg(fip->ep, &fip->rd_lane.read,
fip->rd_buff, rd_len, raddr);
VALGRIND_DO_MAKE_MEM_DEFINED(fip->rd_buff, rd_len);
ret = rpmem_fip_lane_wait(&fip->rd_lane.lane, FI_READ);
if (ret) {
ERR("error when processing read request");
errno = ret;
return -1;
}
memcpy(&cbuff[rd], fip->rd_buff, rd_len);
rd += rd_len;
}
return 0;
}
/*
* rpmem_fip_persist_gpspm -- (internal) perform persist operation for GPSPM
*/
static int
rpmem_fip_persist_gpspm(struct rpmem_fip *fip, size_t offset,
size_t len, unsigned lane)
{
int ret;
struct rpmem_fip_plane_gpspm *lanep = &fip->lanes.gpspm[lane];
ret = rpmem_fip_lane_wait(&lanep->lane, FI_SEND);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "waiting for SEND buffer");
return ret;
}
RPMEM_ASSERT(!rpmem_fip_lane_busy(&lanep->lane));
rpmem_fip_lane_begin(&lanep->lane, FI_SEND | FI_RECV);
void *laddr = (void *)((uintptr_t)fip->laddr + offset);
uint64_t raddr = fip->raddr + offset;
struct rpmem_msg_persist *msg;
struct rpmem_fip_plane_gpspm *gpspm = (void *)lanep;
/* WRITE for requested memory region */
ret = rpmem_fip_writemsg(fip->ep, &gpspm->write, laddr, len, raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR((int)ret, "RMA write");
return ret;
}
/* SEND persist message */
msg = rpmem_fip_msg_get_pmsg(&gpspm->send);
msg->lane = lane;
msg->addr = raddr;
msg->size = len;
ret = rpmem_fip_sendmsg(fip->ep, &gpspm->send);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "MSG send");
return ret;
}
/* wait for persist operation completion */
ret = rpmem_fip_lane_wait(&lanep->lane, FI_RECV);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "persist operation failed");
return ret;
}
return ret;
}
/*
* rpmem_fip_persist_apm -- (internal) perform persist operation for APM
*/
static int
rpmem_fip_persist_apm(struct rpmem_fip *fip, size_t offset,
size_t len, unsigned lane)
{
struct rpmem_fip_plane_apm *lanep = &fip->lanes.apm[lane];
RPMEM_ASSERT(!rpmem_fip_lane_busy(&lanep->lane));
rpmem_fip_lane_begin(&lanep->lane, FI_READ);
int ret;
void *laddr = (void *)((uintptr_t)fip->laddr + offset);
uint64_t raddr = fip->raddr + offset;
/* WRITE for requested memory region */
ret = rpmem_fip_writemsg(fip->ep, &lanep->write, laddr, len, raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "RMA write");
return ret;
}
/* READ to read-after-write buffer */
ret = rpmem_fip_readmsg(fip->ep, &lanep->read, &fip->raw_buff,
sizeof(fip->raw_buff), raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "RMA read");
return ret;
}
/* wait for READ completion */
ret = rpmem_fip_lane_wait(&lanep->lane, FI_READ);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "waiting for READ completion failed");
return ret;
}
return ret;
}
