/**
* fsl_mc_allocator_probe - callback invoked when an allocatable device is
* being added to the system
*/
static int fsl_mc_allocator_probe(struct fsl_mc_device *mc_dev)
{
enum fsl_mc_pool_type pool_type;
struct fsl_mc_device *mc_bus_dev;
struct fsl_mc_bus *mc_bus;
int error = -EINVAL;
if (WARN_ON(!FSL_MC_IS_ALLOCATABLE(mc_dev->obj_desc.type)))
goto error;
mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
if (WARN_ON(mc_bus_dev->dev.bus != &fsl_mc_bus_type))
goto error;
mc_bus = to_fsl_mc_bus(mc_bus_dev);
error = object_type_to_pool_type(mc_dev->obj_desc.type, &pool_type);
if (error < 0)
goto error;
error = fsl_mc_resource_pool_add_device(mc_bus, pool_type, mc_dev);
if (error < 0)
goto error;
dev_info(&mc_dev->dev,
"Allocatable MC object device bound to fsl_mc_allocator driver");
return 0;
error:
return error;
}
static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg,
struct msi_desc *desc)
{
arg->desc = desc;
arg->hwirq = fsl_mc_domain_calc_hwirq(to_fsl_mc_device(desc->dev),
desc);
}
static int its_fsl_mc_msi_prepare(struct irq_domain *msi_domain,
struct device *dev,
int nvec, msi_alloc_info_t *info)
{
struct fsl_mc_device *mc_bus_dev;
struct msi_domain_info *msi_info;
if (!dev_is_fsl_mc(dev))
return -EINVAL;
mc_bus_dev = to_fsl_mc_device(dev);
if (!(mc_bus_dev->flags & FSL_MC_IS_DPRC))
return -EINVAL;
/*
* Set the device Id to be passed to the GIC-ITS:
*
* NOTE: This device id corresponds to the IOMMU stream ID
* associated with the DPRC object (ICID).
*/
#ifdef GENERIC_MSI_DOMAIN_OPS
info->scratchpad[0].ul = mc_bus_dev->icid;
#endif
msi_info = msi_get_domain_info(msi_domain->parent);
return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
}
/**
* fsl_mc_bus_match - device to driver matching callback
* @dev: the MC object device structure to match against
* @drv: the device driver to search for matching MC object device id
* structures
*
* Returns 1 on success, 0 otherwise.
*/
static int fsl_mc_bus_match(struct device *dev, struct device_driver *drv)
{
const struct fsl_mc_device_match_id *id;
struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(drv);
bool found = false;
bool major_version_mismatch = false;
bool minor_version_mismatch = false;
if (WARN_ON(!fsl_mc_bus_exists()))
goto out;
if (!mc_drv->match_id_table)
goto out;
/*
* If the object is not 'plugged' don't match.
* Only exception is the root DPRC, which is a special case.
*/
if ((mc_dev->obj_desc.state & DPRC_OBJ_STATE_PLUGGED) == 0 &&
!fsl_mc_is_root_dprc(&mc_dev->dev))
goto out;
/*
* Traverse the match_id table of the given driver, trying to find
* a matching for the given MC object device.
*/
for (id = mc_drv->match_id_table; id->vendor != 0x0; id++) {
if (id->vendor == mc_dev->obj_desc.vendor &&
strcmp(id->obj_type, mc_dev->obj_desc.type) == 0) {
if (id->ver_major == mc_dev->obj_desc.ver_major) {
found = true;
if (id->ver_minor != mc_dev->obj_desc.ver_minor)
minor_version_mismatch = true;
} else {
major_version_mismatch = true;
}
break;
}
}
if (major_version_mismatch) {
dev_warn(dev,
"Major version mismatch: driver version %u.%u, MC object version %u.%u\n",
id->ver_major, id->ver_minor,
mc_dev->obj_desc.ver_major,
mc_dev->obj_desc.ver_minor);
} else if (minor_version_mismatch) {
dev_warn(dev,
"Minor version mismatch: driver version %u.%u, MC object version %u.%u\n",
id->ver_major, id->ver_minor,
mc_dev->obj_desc.ver_major,
mc_dev->obj_desc.ver_minor);
}
out:
dev_dbg(dev, "%smatched\n", found ? "" : "not ");
return found;
}
static int __fsl_mc_device_remove(struct device *dev, void *data)
{
WARN_ON(!dev);
WARN_ON(data);
fsl_mc_device_remove(to_fsl_mc_device(dev));
return 0;
}
static int __fsl_mc_device_match(struct device *dev, void *data)
{
struct dprc_obj_desc *obj_desc = data;
struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
return FSL_MC_DEVICE_MATCH(mc_dev, obj_desc);
}
static void fsl_mc_driver_shutdown(struct device *dev)
{
struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
mc_drv->shutdown(mc_dev);
}
/**
* fsl_mc_resource_pool_remove_device - remove an allocatable device from a
* resource pool
*
* @mc_dev: Pointer to allocatable MC object device
*
* It permanently removes an allocatable MC object device from the resource
* pool, the device is currently in, as long as it is in the pool's free list.
*/
static int __must_check fsl_mc_resource_pool_remove_device(struct fsl_mc_device
*mc_dev)
{
struct fsl_mc_device *mc_bus_dev;
struct fsl_mc_bus *mc_bus;
struct fsl_mc_resource_pool *res_pool;
struct fsl_mc_resource *resource;
int error = -EINVAL;
bool mutex_locked = false;
if (WARN_ON(!FSL_MC_IS_ALLOCATABLE(mc_dev->obj_desc.type)))
goto out;
resource = mc_dev->resource;
if (WARN_ON(resource->data != mc_dev))
goto out;
mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
mc_bus = to_fsl_mc_bus(mc_bus_dev);
res_pool = resource->parent_pool;
if (WARN_ON(res_pool != &mc_bus->resource_pools[resource->type]))
goto out;
mutex_lock(&res_pool->mutex);
mutex_locked = true;
if (WARN_ON(res_pool->max_count <= 0))
goto out;
if (WARN_ON(res_pool->free_count <= 0 ||
res_pool->free_count > res_pool->max_count))
goto out;
/*
* If the device is currently allocated, its resource is not
* in the free list and thus, the device cannot be removed.
*/
if (list_empty(&resource->node)) {
error = -EBUSY;
dev_err(&mc_bus_dev->dev,
"Device %s cannot be removed from resource pool\n",
dev_name(&mc_dev->dev));
goto out;
}
list_del(&resource->node);
INIT_LIST_HEAD(&resource->node);
res_pool->free_count--;
res_pool->max_count--;
devm_kfree(&mc_bus_dev->dev, resource);
mc_dev->resource = NULL;
error = 0;
out:
if (mutex_locked)
mutex_unlock(&res_pool->mutex);
return error;
}
/**
* fsl_mc_portal_allocate - Allocates an MC portal
*
* @mc_dev: MC device for which the MC portal is to be allocated
* @mc_io_flags: Flags for the fsl_mc_io object that wraps the allocated
* MC portal.
* @new_mc_io: Pointer to area where the pointer to the fsl_mc_io object
* that wraps the allocated MC portal is to be returned
*
* This function allocates an MC portal from the device's parent DPRC,
* from the corresponding MC bus' pool of MC portals and wraps
* it in a new fsl_mc_io object. If 'mc_dev' is a DPRC itself, the
* portal is allocated from its own MC bus.
*/
int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
uint16_t mc_io_flags,
struct fsl_mc_io **new_mc_io)
{
struct fsl_mc_device *mc_bus_dev;
struct fsl_mc_bus *mc_bus;
phys_addr_t mc_portal_phys_addr;
size_t mc_portal_size;
struct fsl_mc_device *mc_adev;
int error = -EINVAL;
struct fsl_mc_resource *resource = NULL;
struct fsl_mc_io *mc_io = NULL;
if (mc_dev->flags & FSL_MC_IS_DPRC) {
mc_bus_dev = mc_dev;
} else {
if (WARN_ON(mc_dev->dev.parent->bus != &fsl_mc_bus_type))
return error;
mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
}
mc_bus = to_fsl_mc_bus(mc_bus_dev);
*new_mc_io = NULL;
error = fsl_mc_resource_allocate(mc_bus, FSL_MC_POOL_DPMCP, &resource);
if (error < 0)
return error;
mc_adev = resource->data;
if (WARN_ON(!mc_adev))
goto error_cleanup_resource;
if (WARN_ON(mc_adev->obj_desc.region_count == 0))
goto error_cleanup_resource;
mc_portal_phys_addr = mc_adev->regions[0].start;
mc_portal_size = mc_adev->regions[0].end -
mc_adev->regions[0].start + 1;
if (WARN_ON(mc_portal_size != mc_bus_dev->mc_io->portal_size))
goto error_cleanup_resource;
error = fsl_create_mc_io(&mc_bus_dev->dev,
mc_portal_phys_addr,
mc_portal_size, resource,
mc_io_flags, &mc_io);
if (error < 0)
goto error_cleanup_resource;
*new_mc_io = mc_io;
return 0;
error_cleanup_resource:
fsl_mc_resource_free(resource);
return error;
}
static struct fsl_mc_device *fsl_mc_device_lookup(struct dprc_obj_desc
*obj_desc,
struct fsl_mc_device
*mc_bus_dev)
{
struct device *dev;
dev = device_find_child(&mc_bus_dev->dev, obj_desc,
__fsl_mc_device_match);
return dev ? to_fsl_mc_device(dev) : NULL;
}
/*
* NOTE: This function is invoked with interrupts disabled
*/
static void fsl_mc_msi_write_msg(struct irq_data *irq_data,
struct msi_msg *msg)
{
struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev);
struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
struct fsl_mc_device_irq *mc_dev_irq =
&mc_bus->irq_resources[msi_desc->fsl_mc.msi_index];
msi_desc->msg = *msg;
/*
* Program the MSI (paddr, value) pair in the device:
*/
__fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq);
}
static int fsl_mc_driver_remove(struct device *dev)
{
struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(dev->driver);
struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
int error;
if (WARN_ON(!dev->driver))
return -EINVAL;
error = mc_drv->remove(mc_dev);
if (error < 0) {
dev_err(dev,
"MC object device remove callback failed: %d\n",
error);
return error;
}
return 0;
}
/**
* fsl_mc_object_allocate - Allocates a MC object device of the given
* pool type from a given MC bus
*
* @mc_dev: MC device for which the MC object device is to be allocated
* @pool_type: MC bus resource pool type
* @new_mc_dev: Pointer to area where the pointer to the allocated
* MC object device is to be returned
*
* This function allocates a MC object device from the device's parent DPRC,
* from the corresponding MC bus' pool of allocatable MC object devices of
* the given resource type. mc_dev cannot be a DPRC itself.
*
* NOTE: pool_type must be different from FSL_MC_POOL_MCP, since MC
* portals are allocated using fsl_mc_portal_allocate(), instead of
* this function.
*/
int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev,
enum fsl_mc_pool_type pool_type,
struct fsl_mc_device **new_mc_adev)
{
struct fsl_mc_device *mc_bus_dev;
struct fsl_mc_bus *mc_bus;
struct fsl_mc_device *mc_adev;
int error = -EINVAL;
struct fsl_mc_resource *resource = NULL;
*new_mc_adev = NULL;
if (WARN_ON(mc_dev->flags & FSL_MC_IS_DPRC))
goto error;
if (WARN_ON(mc_dev->dev.parent->bus != &fsl_mc_bus_type))
goto error;
if (WARN_ON(pool_type == FSL_MC_POOL_DPMCP))
goto error;
mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
mc_bus = to_fsl_mc_bus(mc_bus_dev);
error = fsl_mc_resource_allocate(mc_bus, pool_type, &resource);
if (error < 0)
goto error;
mc_adev = resource->data;
if (WARN_ON(!mc_adev))
goto error;
*new_mc_adev = mc_adev;
return 0;
error:
if (resource)
fsl_mc_resource_free(resource);
return error;
}
static int __fsl_mc_device_remove_if_not_in_mc(struct device *dev, void *data)
{
int i;
struct dprc_child_objs *objs;
struct fsl_mc_device *mc_dev;
WARN_ON(!dev);
WARN_ON(!data);
mc_dev = to_fsl_mc_device(dev);
objs = data;
for (i = 0; i < objs->child_count; i++) {
struct dprc_obj_desc *obj_desc = &objs->child_array[i];
if (strlen(obj_desc->type) != 0 &&
FSL_MC_DEVICE_MATCH(mc_dev, obj_desc))
break;
}
if (i == objs->child_count)
fsl_mc_device_remove(mc_dev);
return 0;
}
/**
* dprc_irq0_handler_thread - Handler thread function for DPRC interrupt 0
*
* @irq: IRQ number of the interrupt being handled
* @arg: Pointer to device structure
*/
static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg)
{
int error;
u32 status;
struct device *dev = arg;
struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev);
struct fsl_mc_io *mc_io = mc_dev->mc_io;
struct msi_desc *msi_desc = mc_dev->irqs[0]->msi_desc;
dev_dbg(dev, "DPRC IRQ %d triggered on CPU %u\n",
irq_num, smp_processor_id());
if (WARN_ON(!(mc_dev->flags & FSL_MC_IS_DPRC)))
return IRQ_HANDLED;
mutex_lock(&mc_bus->scan_mutex);
if (WARN_ON(!msi_desc || msi_desc->irq != (u32)irq_num))
goto out;
status = 0;
error = dprc_get_irq_status(mc_io, 0, mc_dev->mc_handle, 0,
&status);
if (error < 0) {
dev_err(dev,
"dprc_get_irq_status() failed: %d\n", error);
goto out;
}
error = dprc_clear_irq_status(mc_io, 0, mc_dev->mc_handle, 0,
status);
if (error < 0) {
dev_err(dev,
"dprc_clear_irq_status() failed: %d\n", error);
goto out;
}
if (status & (DPRC_IRQ_EVENT_OBJ_ADDED |
DPRC_IRQ_EVENT_OBJ_REMOVED |
DPRC_IRQ_EVENT_CONTAINER_DESTROYED |
DPRC_IRQ_EVENT_OBJ_DESTROYED |
DPRC_IRQ_EVENT_OBJ_CREATED)) {
unsigned int irq_count;
error = dprc_scan_objects(mc_dev, &irq_count);
if (error < 0) {
/*
* If the error is -ENXIO, we ignore it, as it indicates
* that the object scan was aborted, as we detected that
* an object was removed from the DPRC in the MC, while
* we were scanning the DPRC.
*/
if (error != -ENXIO) {
dev_err(dev, "dprc_scan_objects() failed: %d\n",
error);
}
goto out;
}
if (irq_count > FSL_MC_IRQ_POOL_MAX_TOTAL_IRQS) {
dev_warn(dev,
"IRQs needed (%u) exceed IRQs preallocated (%u)\n",
irq_count, FSL_MC_IRQ_POOL_MAX_TOTAL_IRQS);
}
}
out:
mutex_unlock(&mc_bus->scan_mutex);
return IRQ_HANDLED;
}
/**
* fsl_mc_portal_allocate - Allocates an MC portal
*
* @mc_dev: MC device for which the MC portal is to be allocated
* @mc_io_flags: Flags for the fsl_mc_io object that wraps the allocated
* MC portal.
* @new_mc_io: Pointer to area where the pointer to the fsl_mc_io object
* that wraps the allocated MC portal is to be returned
*
* This function allocates an MC portal from the device's parent DPRC,
* from the corresponding MC bus' pool of MC portals and wraps
* it in a new fsl_mc_io object. If 'mc_dev' is a DPRC itself, the
* portal is allocated from its own MC bus.
*/
int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
u16 mc_io_flags,
struct fsl_mc_io **new_mc_io)
{
struct fsl_mc_device *mc_bus_dev;
struct fsl_mc_bus *mc_bus;
phys_addr_t mc_portal_phys_addr;
size_t mc_portal_size;
struct fsl_mc_device *dpmcp_dev;
int error = -EINVAL;
struct fsl_mc_resource *resource = NULL;
struct fsl_mc_io *mc_io = NULL;
if (mc_dev->flags & FSL_MC_IS_DPRC) {
mc_bus_dev = mc_dev;
} else {
if (WARN_ON(!dev_is_fsl_mc(mc_dev->dev.parent)))
return error;
mc_bus_dev = to_fsl_mc_device(mc_dev->dev.parent);
}
mc_bus = to_fsl_mc_bus(mc_bus_dev);
*new_mc_io = NULL;
error = fsl_mc_resource_allocate(mc_bus, FSL_MC_POOL_DPMCP, &resource);
if (error < 0)
return error;
error = -EINVAL;
dpmcp_dev = resource->data;
if (WARN_ON(!dpmcp_dev))
goto error_cleanup_resource;
if (dpmcp_dev->obj_desc.ver_major < DPMCP_MIN_VER_MAJOR ||
(dpmcp_dev->obj_desc.ver_major == DPMCP_MIN_VER_MAJOR &&
dpmcp_dev->obj_desc.ver_minor < DPMCP_MIN_VER_MINOR)) {
dev_err(&dpmcp_dev->dev,
"ERROR: Version %d.%d of DPMCP not supported.\n",
dpmcp_dev->obj_desc.ver_major,
dpmcp_dev->obj_desc.ver_minor);
error = -ENOTSUPP;
goto error_cleanup_resource;
}
if (WARN_ON(dpmcp_dev->obj_desc.region_count == 0))
goto error_cleanup_resource;
mc_portal_phys_addr = dpmcp_dev->regions[0].start;
mc_portal_size = dpmcp_dev->regions[0].end -
dpmcp_dev->regions[0].start + 1;
if (WARN_ON(mc_portal_size != mc_bus_dev->mc_io->portal_size))
goto error_cleanup_resource;
error = fsl_create_mc_io(&mc_bus_dev->dev,
mc_portal_phys_addr,
mc_portal_size, dpmcp_dev,
mc_io_flags, &mc_io);
if (error < 0)
goto error_cleanup_resource;
*new_mc_io = mc_io;
return 0;
error_cleanup_resource:
fsl_mc_resource_free(resource);
return error;
}
/**
* Add a newly discovered MC object device to be visible in Linux
*/
int fsl_mc_device_add(struct dprc_obj_desc *obj_desc,
struct fsl_mc_io *mc_io,
struct device *parent_dev,
struct fsl_mc_device **new_mc_dev)
{
int error;
struct fsl_mc_device *mc_dev = NULL;
struct fsl_mc_bus *mc_bus = NULL;
struct fsl_mc_device *parent_mc_dev;
if (parent_dev->bus == &fsl_mc_bus_type)
parent_mc_dev = to_fsl_mc_device(parent_dev);
else
parent_mc_dev = NULL;
if (strcmp(obj_desc->type, "dprc") == 0) {
/*
* Allocate an MC bus device object:
*/
mc_bus = devm_kzalloc(parent_dev, sizeof(*mc_bus), GFP_KERNEL);
if (!mc_bus)
return -ENOMEM;
mc_dev = &mc_bus->mc_dev;
} else {
/*
* Allocate a regular fsl_mc_device object:
*/
mc_dev = kmem_cache_zalloc(mc_dev_cache, GFP_KERNEL);
if (!mc_dev)
return -ENOMEM;
}
mc_dev->obj_desc = *obj_desc;
mc_dev->mc_io = mc_io;
device_initialize(&mc_dev->dev);
mc_dev->dev.parent = parent_dev;
mc_dev->dev.bus = &fsl_mc_bus_type;
dev_set_name(&mc_dev->dev, "%s.%d", obj_desc->type, obj_desc->id);
if (strcmp(obj_desc->type, "dprc") == 0) {
struct fsl_mc_io *mc_io2;
mc_dev->flags |= FSL_MC_IS_DPRC;
/*
* To get the DPRC's ICID, we need to open the DPRC
* in get_dprc_icid(). For child DPRCs, we do so using the
* parent DPRC's MC portal instead of the child DPRC's MC
* portal, in case the child DPRC is already opened with
* its own portal (e.g., the DPRC used by AIOP).
*
* NOTE: There cannot be more than one active open for a
* given MC object, using the same MC portal.
*/
if (parent_mc_dev) {
/*
* device being added is a child DPRC device
*/
mc_io2 = parent_mc_dev->mc_io;
} else {
/*
* device being added is the root DPRC device
*/
if (WARN_ON(!mc_io)) {
error = -EINVAL;
goto error_cleanup_dev;
}
mc_io2 = mc_io;
atomic_inc(&root_dprc_count);
}
error = get_dprc_icid(mc_io2, obj_desc->id, &mc_dev->icid);
if (error < 0)
goto error_cleanup_dev;
} else {
/*
* A non-DPRC MC object device has to be a child of another
* MC object (specifically a DPRC object)
*/
mc_dev->icid = parent_mc_dev->icid;
mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
mc_dev->dev.dma_mask = &mc_dev->dma_mask;
dev_set_msi_domain(&mc_dev->dev,
dev_get_msi_domain(&parent_mc_dev->dev));
}
/*
* Get MMIO regions for the device from the MC:
*
* NOTE: the root DPRC is a special case as its MMIO region is
* obtained from the device tree
*/
if (parent_mc_dev && obj_desc->region_count != 0) {
error = fsl_mc_device_get_mmio_regions(mc_dev,
parent_mc_dev);
if (error < 0)
goto error_cleanup_dev;
}
/*
* The device-specific probe callback will get invoked by device_add()
*/
error = device_add(&mc_dev->dev);
if (error < 0) {
dev_err(parent_dev,
"device_add() failed for device %s: %d\n",
dev_name(&mc_dev->dev), error);
goto error_cleanup_dev;
}
(void)get_device(&mc_dev->dev);
dev_dbg(parent_dev, "Added MC object device %s\n",
dev_name(&mc_dev->dev));
*new_mc_dev = mc_dev;
return 0;
error_cleanup_dev:
kfree(mc_dev->regions);
if (mc_bus)
devm_kfree(parent_dev, mc_bus);
else
kmem_cache_free(mc_dev_cache, mc_dev);
return error;
}
