void
pci_iov_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
{
struct pcicfg_iov *iov;
iov = dinfo->cfg.iov;
IOV_WRITE(dinfo, PCIR_SRIOV_PAGE_SIZE, iov->iov_page_size, 4);
IOV_WRITE(dinfo, PCIR_SRIOV_NUM_VFS, iov->iov_num_vfs, 2);
IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov->iov_ctl, 2);
}
static int
pci_iov_config_page_size(struct pci_devinfo *dinfo)
{
uint32_t page_cap, page_size;
page_cap = IOV_READ(dinfo, PCIR_SRIOV_PAGE_CAP, 4);
/*
* If the system page size is less than the smallest SR-IOV page size
* then round up to the smallest SR-IOV page size.
*/
if (PAGE_SHIFT < PCI_SRIOV_BASE_PAGE_SHIFT)
page_size = (1 << 0);
else
page_size = (1 << (PAGE_SHIFT - PCI_SRIOV_BASE_PAGE_SHIFT));
/* Check that the device supports the system page size. */
if (!(page_size & page_cap))
return (ENXIO);
IOV_WRITE(dinfo, PCIR_SRIOV_PAGE_SIZE, page_size, 4);
return (0);
}
size_t mc_ascii_response_write_iovs(mc_ascii_response_buf_t* buf,
const mc_msg_t* req,
const mc_msg_t* reply,
struct iovec* iovs,
size_t max_niovs) {
size_t niovs = 0;
buf->offset = 0;
if (mc_res_is_err(reply->result)) {
if (reply->value.len > 0) {
if (reply->result == mc_res_client_error) {
IOV_WRITE_CONST_STR("CLIENT_ERROR ");
} else {
IOV_WRITE_CONST_STR("SERVER_ERROR ");
}
if (reply->err_code != 0) {
IOV_FORMAT(buf, "%" PRIu32 " ", reply->err_code);
}
IOV_WRITE_NSTRING(reply->value);
IOV_WRITE_CONST_STR("\r\n");
} else {
IOV_WRITE_STR(mc_res_to_response_string(reply->result));
}
return niovs;
}
switch (req->op) {
case mc_op_incr:
case mc_op_decr:
switch (reply->result) {
case mc_res_stored:
IOV_FORMAT(buf, "%" PRIu64 "\r\n", reply->delta);
break;
case mc_res_notfound:
IOV_WRITE_CONST_STR("NOT_FOUND\r\n");
break;
default:
goto UNEXPECTED;
}
break;
case mc_op_set:
case mc_op_lease_set:
case mc_op_add:
case mc_op_replace:
case mc_op_append:
case mc_op_prepend:
case mc_op_cas:
switch (reply->result) {
case mc_res_ok:
IOV_WRITE_STR(mc_res_to_response_string(mc_res_stored));
break;
case mc_res_stored:
case mc_res_stalestored:
case mc_res_found:
case mc_res_notstored:
case mc_res_notfound:
case mc_res_exists:
IOV_WRITE_STR(mc_res_to_response_string(reply->result));
break;
default:
goto UNEXPECTED;
}
break;
case mc_op_delete:
switch (reply->result) {
case mc_res_deleted:
case mc_res_notfound:
IOV_WRITE_STR(mc_res_to_response_string(reply->result));
break;
default:
goto UNEXPECTED;
}
break;
case mc_op_get:
case mc_op_lease_get:
case mc_op_gets:
switch (reply->result) {
case mc_res_found:
IOV_WRITE_CONST_STR("VALUE ");
IOV_WRITE_NSTRING(req->key);
IOV_FORMAT(buf, " %" PRIu64 " %lu", reply->flags,
reply->value.len);
if (req->op == mc_op_gets) {
IOV_FORMAT(buf, " %" PRIu64, reply->cas);
}
IOV_WRITE_CONST_STR("\r\n");
IOV_WRITE_NSTRING(reply->value);
IOV_WRITE_CONST_STR("\r\n");
break;
case mc_res_notfound:
if (req->op != mc_op_lease_get) {
// misses should have been suppressed!
goto UNEXPECTED;
}
// but lease-get always has a response
IOV_WRITE_CONST_STR("LVALUE ");
IOV_WRITE_NSTRING(req->key);
IOV_FORMAT(buf, " %" PRIu64 " %"PRIu64 " %zu\r\n",
reply->lease_id,
reply->flags,
reply->value.len);
IOV_WRITE_NSTRING(reply->value);
IOV_WRITE_CONST_STR("\r\n");
break;
default:
goto UNEXPECTED;
}
break;
case mc_op_metaget:
switch (reply->result) {
case mc_res_found:
/* (META key age: (unknown|\d+); exptime: \d+;
from: (\d+\.\d+\.\d+\.\d+|unknown); is_transient: (1|0)\r\n) */
IOV_WRITE_CONST_STR("META ");
IOV_WRITE_NSTRING(req->key);
IOV_WRITE_CONST_STR(" age: ");
if (reply->number == (uint32_t) -1) {
IOV_WRITE_CONST_STR("unknown");
}
else {
IOV_FORMAT(buf, "%d", reply->number);
}
IOV_WRITE_CONST_STR("; exptime: ");
IOV_FORMAT(buf, "%d", reply->exptime);
IOV_WRITE_CONST_STR("; from: ");
if (reply->ipv == 0) {
IOV_WRITE_CONST_STR("unknown");
}
else {
IOV_WRITE_IP(buf, reply->ipv, &(reply->ip_addr));
}
IOV_WRITE_CONST_STR("; is_transient: ");
IOV_FORMAT(buf, "%" PRIu64, reply->flags);
IOV_WRITE_CONST_STR("\r\n");
break;
case mc_res_notfound:
break;
default:
goto UNEXPECTED;
}
break;
case mc_op_end:
if (reply->result == mc_res_found) {
IOV_WRITE_CONST_STR("END\r\n");
}
else {
IOV_WRITE_STR(mc_res_to_response_string(reply->result));
}
break;
case mc_op_stats:
switch (reply->result) {
case mc_res_ok:
{
size_t length = 0;
char* stats;
if (reply->stats) {
/* TODO(agartrell) assert(!reply->value.str)
*
* The assert here can't be turned on until
* mcrouter/stats.c:560 has been fixed to not set both
* value and stats on the libmc reply
*/
stats = stats_reply_to_string(reply->stats, reply->number, &length);
buf->stats = stats;
} else {
stats = reply->value.str;
length = reply->value.len;
}
if (!stats) {
return 0;
}
IOV_WRITE(stats, length);
break;
}
default:
goto UNEXPECTED;
}
break;
case mc_op_flushall:
case mc_op_flushre:
IOV_WRITE_CONST_STR("OK\r\n");
break;
case mc_op_version:
IOV_WRITE_CONST_STR("VERSION ");
IOV_WRITE_NSTRING(reply->value);
IOV_WRITE_CONST_STR("\r\n");
break;
case mc_op_shutdown:
if (reply->result == mc_res_ok) {
IOV_WRITE_CONST_STR("OK\r\n");
}
else {
goto UNEXPECTED;
}
break;
case mc_op_exec:
switch (reply->result) {
case mc_res_ok:
IOV_WRITE_NSTRING(reply->value);
IOV_WRITE_CONST_STR("\r\n");
break;
default:
goto UNEXPECTED;
}
break;
default:
IOV_WRITE_CONST_STR("SERVER_ERROR unhandled token ");
IOV_WRITE_STR(mc_op_to_string(req->op));
IOV_FORMAT(buf, " (%d)\r\n", (int)req->op);
break;
}
return niovs;
UNEXPECTED:
FBI_ASSERT(niovs == 0);
IOV_WRITE_CONST_STR("SERVER_ERROR unexpected result ");
IOV_WRITE_STR(mc_res_to_string(reply->result));
IOV_FORMAT(buf, " (%d) for ", (int)reply->result);
IOV_WRITE_STR(mc_op_to_string(req->op));
IOV_FORMAT(buf, " (%d)\r\n", (int)req->op);
return niovs;
}
static int
pci_iov_delete(struct cdev *cdev)
{
device_t bus, dev, vf, *devlist;
struct pci_devinfo *dinfo;
struct pcicfg_iov *iov;
int i, error, devcount;
uint32_t iov_ctl;
mtx_lock(&Giant);
dinfo = cdev->si_drv1;
iov = dinfo->cfg.iov;
dev = dinfo->cfg.dev;
bus = device_get_parent(dev);
devlist = NULL;
if (iov->iov_flags & IOV_BUSY) {
mtx_unlock(&Giant);
return (EBUSY);
}
if (iov->iov_num_vfs == 0) {
mtx_unlock(&Giant);
return (ECHILD);
}
iov->iov_flags |= IOV_BUSY;
error = device_get_children(bus, &devlist, &devcount);
if (error != 0)
goto out;
for (i = 0; i < devcount; i++) {
vf = devlist[i];
if (!pci_iov_is_child_vf(iov, vf))
continue;
error = device_detach(vf);
if (error != 0) {
device_printf(dev,
"Could not disable SR-IOV: failed to detach VF %s\n",
device_get_nameunit(vf));
goto out;
}
}
for (i = 0; i < devcount; i++) {
vf = devlist[i];
if (pci_iov_is_child_vf(iov, vf))
device_delete_child(bus, vf);
}
PCI_IOV_UNINIT(dev);
iov_ctl = IOV_READ(dinfo, PCIR_SRIOV_CTL, 2);
iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov_ctl, 2);
IOV_WRITE(dinfo, PCIR_SRIOV_NUM_VFS, 0, 2);
iov->iov_num_vfs = 0;
for (i = 0; i <= PCIR_MAX_BAR_0; i++) {
if (iov->iov_bar[i].res != NULL) {
pci_release_resource(bus, dev, SYS_RES_MEMORY,
iov->iov_pos + PCIR_SRIOV_BAR(i),
iov->iov_bar[i].res);
pci_delete_resource(bus, dev, SYS_RES_MEMORY,
iov->iov_pos + PCIR_SRIOV_BAR(i));
iov->iov_bar[i].res = NULL;
}
}
if (iov->iov_flags & IOV_RMAN_INITED) {
rman_fini(&iov->rman);
iov->iov_flags &= ~IOV_RMAN_INITED;
}
error = 0;
out:
free(devlist, M_TEMP);
iov->iov_flags &= ~IOV_BUSY;
mtx_unlock(&Giant);
return (error);
}
static int
pci_iov_config(struct cdev *cdev, struct pci_iov_arg *arg)
{
device_t bus, dev;
struct pci_devinfo *dinfo;
struct pcicfg_iov *iov;
nvlist_t *config;
int i, error;
uint16_t rid_off, rid_stride;
uint16_t first_rid, last_rid;
uint16_t iov_ctl;
uint16_t num_vfs, total_vfs;
int iov_inited;
mtx_lock(&Giant);
dinfo = cdev->si_drv1;
iov = dinfo->cfg.iov;
dev = dinfo->cfg.dev;
bus = device_get_parent(dev);
iov_inited = 0;
config = NULL;
if ((iov->iov_flags & IOV_BUSY) || iov->iov_num_vfs != 0) {
mtx_unlock(&Giant);
return (EBUSY);
}
iov->iov_flags |= IOV_BUSY;
error = pci_iov_parse_config(iov, arg, &config);
if (error != 0)
goto out;
num_vfs = pci_iov_config_get_num_vfs(config);
total_vfs = IOV_READ(dinfo, PCIR_SRIOV_TOTAL_VFS, 2);
if (num_vfs > total_vfs) {
error = EINVAL;
goto out;
}
error = pci_iov_config_page_size(dinfo);
if (error != 0)
goto out;
error = pci_iov_set_ari(bus);
if (error != 0)
goto out;
error = pci_iov_init(dev, num_vfs, config);
if (error != 0)
goto out;
iov_inited = 1;
IOV_WRITE(dinfo, PCIR_SRIOV_NUM_VFS, num_vfs, 2);
rid_off = IOV_READ(dinfo, PCIR_SRIOV_VF_OFF, 2);
rid_stride = IOV_READ(dinfo, PCIR_SRIOV_VF_STRIDE, 2);
first_rid = pci_get_rid(dev) + rid_off;
last_rid = first_rid + (num_vfs - 1) * rid_stride;
/* We don't yet support allocating extra bus numbers for VFs. */
if (pci_get_bus(dev) != PCI_RID2BUS(last_rid)) {
error = ENOSPC;
goto out;
}
iov_ctl = IOV_READ(dinfo, PCIR_SRIOV_CTL, 2);
iov_ctl &= ~(PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE);
IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov_ctl, 2);
error = pci_iov_init_rman(dev, iov);
if (error != 0)
goto out;
iov->iov_num_vfs = num_vfs;
error = pci_iov_setup_bars(dinfo);
if (error != 0)
goto out;
iov_ctl = IOV_READ(dinfo, PCIR_SRIOV_CTL, 2);
iov_ctl |= PCIM_SRIOV_VF_EN | PCIM_SRIOV_VF_MSE;
IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov_ctl, 2);
/* Per specification, we must wait 100ms before accessing VFs. */
pause("iov", roundup(hz, 10));
pci_iov_enumerate_vfs(dinfo, config, first_rid, rid_stride);
nvlist_destroy(config);
iov->iov_flags &= ~IOV_BUSY;
mtx_unlock(&Giant);
return (0);
out:
if (iov_inited)
PCI_IOV_UNINIT(dev);
for (i = 0; i <= PCIR_MAX_BAR_0; i++) {
if (iov->iov_bar[i].res != NULL) {
pci_release_resource(bus, dev, SYS_RES_MEMORY,
iov->iov_pos + PCIR_SRIOV_BAR(i),
iov->iov_bar[i].res);
pci_delete_resource(bus, dev, SYS_RES_MEMORY,
iov->iov_pos + PCIR_SRIOV_BAR(i));
iov->iov_bar[i].res = NULL;
}
}
if (iov->iov_flags & IOV_RMAN_INITED) {
rman_fini(&iov->rman);
iov->iov_flags &= ~IOV_RMAN_INITED;
}
nvlist_destroy(config);
iov->iov_num_vfs = 0;
iov->iov_flags &= ~IOV_BUSY;
mtx_unlock(&Giant);
return (error);
}
