/*
* rpmem_fip_persist_gpspm -- (internal) perform persist operation for GPSPM
*/
static int
rpmem_fip_persist_gpspm(struct rpmem_fip *fip, size_t offset,
size_t len, unsigned lane)
{
int ret;
struct rpmem_fip_plane_gpspm *lanep = &fip->lanes.gpspm[lane];
ret = rpmem_fip_lane_wait(&lanep->lane, FI_SEND);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "waiting for SEND buffer");
return ret;
}
RPMEM_ASSERT(!rpmem_fip_lane_busy(&lanep->lane));
rpmem_fip_lane_begin(&lanep->lane, FI_SEND | FI_RECV);
void *laddr = (void *)((uintptr_t)fip->laddr + offset);
uint64_t raddr = fip->raddr + offset;
struct rpmem_msg_persist *msg;
struct rpmem_fip_plane_gpspm *gpspm = (void *)lanep;
/* WRITE for requested memory region */
ret = rpmem_fip_writemsg(fip->ep, &gpspm->write, laddr, len, raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR((int)ret, "RMA write");
return ret;
}
/* SEND persist message */
msg = rpmem_fip_msg_get_pmsg(&gpspm->send);
msg->lane = lane;
msg->addr = raddr;
msg->size = len;
ret = rpmem_fip_sendmsg(fip->ep, &gpspm->send);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "MSG send");
return ret;
}
/* wait for persist operation completion */
ret = rpmem_fip_lane_wait(&lanep->lane, FI_RECV);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "persist operation failed");
return ret;
}
return ret;
}
/*
* rpmem_fip_read -- perform read operation
*/
int
rpmem_fip_read(struct rpmem_fip *fip, void *buff, size_t len, size_t off)
{
RPMEM_ASSERT(!rpmem_fip_lane_busy(&fip->rd_lane.lane));
int ret = 0;
size_t rd = 0;
uint8_t *cbuff = buff;
while (rd < len) {
rpmem_fip_lane_begin(&fip->rd_lane.lane, FI_READ);
size_t rd_len = len - rd < RPMEM_RD_BUFF_SIZE ?
len - rd : RPMEM_RD_BUFF_SIZE;
size_t rd_off = off + rd;
uint64_t raddr = fip->raddr + rd_off;
ret = rpmem_fip_readmsg(fip->ep, &fip->rd_lane.read,
fip->rd_buff, rd_len, raddr);
ret = rpmem_fip_lane_wait(&fip->rd_lane.lane, FI_READ);
if (ret)
return ret;
memcpy(&cbuff[rd], fip->rd_buff, rd_len);
rd += rd_len;
}
return ret;
}
/*
* rpmem_fip_read -- perform read operation
*/
int
rpmem_fip_read(struct rpmem_fip *fip, void *buff, size_t len, size_t off)
{
RPMEM_ASSERT(!rpmem_fip_lane_busy(&fip->rd_lane.lane));
int ret;
size_t rd = 0;
uint8_t *cbuff = buff;
while (rd < len) {
rpmem_fip_lane_begin(&fip->rd_lane.lane, FI_READ);
size_t rd_len = len - rd < RPMEM_RD_BUFF_SIZE ?
len - rd : RPMEM_RD_BUFF_SIZE;
size_t rd_off = off + rd;
uint64_t raddr = fip->raddr + rd_off;
ret = rpmem_fip_readmsg(fip->ep, &fip->rd_lane.read,
fip->rd_buff, rd_len, raddr);
VALGRIND_DO_MAKE_MEM_DEFINED(fip->rd_buff, rd_len);
ret = rpmem_fip_lane_wait(&fip->rd_lane.lane, FI_READ);
if (ret) {
ERR("error when processing read request");
errno = ret;
return -1;
}
memcpy(&cbuff[rd], fip->rd_buff, rd_len);
rd += rd_len;
}
return 0;
}
/*
* rpmemd_fip_worker -- worker callback which processes persist
* operation in GPSPM
*/
static int
rpmemd_fip_worker(void *arg, void *data)
{
struct rpmemd_fip *fip = arg;
struct rpmemd_fip_lane *lanep = data;
int ret = 0;
/* wait until last SEND message has been processed */
rpmem_fip_lane_wait(&lanep->lane, FI_SEND);
/*
* Get persist message and persist message response from appropriate
* buffers. The persist message is in lane's RECV buffer and the
* persist response message in lane's SEND buffer.
*/
struct rpmem_msg_persist *pmsg = rpmem_fip_msg_get_pmsg(&lanep->recv);
struct rpmem_msg_persist_resp *pres =
rpmem_fip_msg_get_pres(&lanep->send);
/* verify persist message */
ret = rpmemd_fip_check_pmsg(fip, pmsg);
if (unlikely(ret))
goto err;
/* return back the lane id */
pres->lane = pmsg->lane;
/*
* Perform the persist operation.
*
* XXX
*
* Maybe the persist could be divided into flush + drain.
* We could issue flush operation, do some other work like
* posting RECV buffer and then call drain. Need to consider this.
*/
fip->persist((void *)pmsg->addr, pmsg->size);
/* post lane's RECV buffer */
ret = rpmemd_fip_gpspm_post_msg(fip, &lanep->recv);
if (unlikely(ret))
goto err;
/* initialize lane for waiting for SEND completion */
rpmem_fip_lane_begin(&lanep->lane, FI_SEND);
/* post lane's SEND buffer */
ret = rpmemd_fip_gpspm_post_resp(fip, &lanep->send);
if (unlikely(ret))
goto err;
err:
return ret;
}
/*
* rpmem_fip_persist_apm -- (internal) perform persist operation for APM
*/
static int
rpmem_fip_persist_apm(struct rpmem_fip *fip, size_t offset,
size_t len, unsigned lane)
{
struct rpmem_fip_plane_apm *lanep = &fip->lanes.apm[lane];
RPMEM_ASSERT(!rpmem_fip_lane_busy(&lanep->lane));
rpmem_fip_lane_begin(&lanep->lane, FI_READ);
int ret;
void *laddr = (void *)((uintptr_t)fip->laddr + offset);
uint64_t raddr = fip->raddr + offset;
/* WRITE for requested memory region */
ret = rpmem_fip_writemsg(fip->ep, &lanep->write, laddr, len, raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "RMA write");
return ret;
}
/* READ to read-after-write buffer */
ret = rpmem_fip_readmsg(fip->ep, &lanep->read, &fip->raw_buff,
sizeof(fip->raw_buff), raddr);
if (unlikely(ret)) {
RPMEM_FI_ERR(ret, "RMA read");
return ret;
}
/* wait for READ completion */
ret = rpmem_fip_lane_wait(&lanep->lane, FI_READ);
if (unlikely(ret)) {
RPMEM_LOG(ERR, "waiting for READ completion failed");
return ret;
}
return ret;
}